diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/AdjacencyMatrixIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/AdjacencyMatrixIT.java index 44e708e00d4d5..abb5e367a92ac 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/AdjacencyMatrixIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/AdjacencyMatrixIT.java @@ -93,6 +93,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java index d775a1bae8b91..8f22d399b3e6b 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.TransportSearchAction; @@ -79,8 +80,9 @@ public void testCancellationDuringTimeSeriesAggregation() throws Exception { // Make sure we sometimes have a few segments try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int j = 0; j < numberOfDocsPerRefresh; j++) { - bulkRequestBuilder.add( - prepareIndex("test").setOpType(DocWriteRequest.OpType.CREATE) + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); + try { + indexRequestBuilder.setOpType(DocWriteRequest.OpType.CREATE) .setSource( "@timestamp", now + (long) i * numberOfDocsPerRefresh + j, @@ -88,8 +90,11 @@ public void testCancellationDuringTimeSeriesAggregation() throws Exception { (double) j, "dim", String.valueOf(j % 100) - ) - ); + ); + bulkRequestBuilder.add(indexRequestBuilder); + } finally { + indexRequestBuilder.request().decRef(); + } } assertNoFailures(bulkRequestBuilder.get()); } diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java index c115b04a6b052..2e0f2bfc7be09 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java @@ -19,6 +19,7 @@ import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -174,6 +175,9 @@ public void setupSuiteScopeCluster() throws Exception { docs.add(prepareIndex("index" + findIndex(timestamp)).setOpType(DocWriteRequest.OpType.CREATE).setSource(docSource)); } indexRandom(true, false, docs); + for (IndexRequestBuilder doc : docs) { + doc.request().decRef(); + } } public void testStandAloneTimeSeriesAgg() { @@ -508,12 +512,22 @@ public void testGetHitsFailure() throws Exception { .add(prepareIndex("test").setId("1").setSource("key", "bar", "val", 10, "@timestamp", "2021-01-01T00:00:00Z")) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); + for (DocWriteRequest request : bulkRequestBuilder.request().requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } } try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { bulkRequestBuilder.add(prepareIndex("test").setId("4").setSource("key", "bar", "val", 50, "@timestamp", "2021-01-01T00:00:30Z")) .add(prepareIndex("test").setId("3").setSource("key", "bar", "val", 40, "@timestamp", "2021-01-01T00:00:20Z")) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); + for (DocWriteRequest request : bulkRequestBuilder.request().requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } } try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { bulkRequestBuilder.add(prepareIndex("test").setId("7").setSource("key", "foo", "val", 20, "@timestamp", "2021-01-01T00:00:00Z")) @@ -522,6 +536,11 @@ public void testGetHitsFailure() throws Exception { .add(prepareIndex("test").setId("6").setSource("key", "baz", "val", 30, "@timestamp", "2021-01-01T00:10:00Z")) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); + for (DocWriteRequest request : bulkRequestBuilder.request().requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } } QueryBuilder queryBuilder = QueryBuilders.rangeQuery("@timestamp").lte("2021-01-01T00:10:00Z"); diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java index cd69526c76481..96267d8d82524 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.aggregations.AggregationIntegTestCase; import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; @@ -67,7 +68,12 @@ public void setup() throws Exception { try (BulkRequestBuilder bulkIndexRequest = client().prepareBulk()) { for (int docId = 0; docId < numberOfDocuments; docId++) { final XContentBuilder document = timeSeriesDocument(FOO_DIM_VALUE, BAR_DIM_VALUE, BAZ_DIM_VALUE, docId, timestamps::next); - bulkIndexRequest.add(prepareIndex("index").setOpType(DocWriteRequest.OpType.CREATE).setSource(document)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index"); + try { + bulkIndexRequest.add(indexRequestBuilder.setOpType(DocWriteRequest.OpType.CREATE).setSource(document)); + } finally { + indexRequestBuilder.request().decRef(); + } } final BulkResponse bulkIndexResponse = bulkIndexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java index ce7e4c63dc69c..b67a2e959e1dd 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java @@ -103,6 +103,9 @@ public void setupSuiteScopeCluster() throws Exception { ) ); // date: Mar 23, dates: Mar 23, Apr 24 indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -217,6 +220,9 @@ public void testSingleValuedFieldNormalised_timeZone_CET_DstStart() throws Excep addNTimes(3, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-26T01:00:00")), builders); addNTimes(4, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-27T01:00:00")), builders); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); assertNoFailuresAndResponse( @@ -274,6 +280,9 @@ public void testSingleValuedFieldNormalised_timeZone_CET_DstEnd() throws Excepti addNTimes(3, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-29T01:00:00")), builders); addNTimes(4, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-30T01:00:00")), builders); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); assertNoFailuresAndResponse( @@ -333,6 +342,9 @@ public void testSingleValuedFieldNormalised_timeZone_AsiaKathmandu() throws Exce addNTimes(3, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1986-01-01T01:30:00")), builders); addNTimes(4, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1986-01-01T02:30:00")), builders); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); assertNoFailuresAndResponse( diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/SerialDiffIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/SerialDiffIT.java index 7cbb298f49931..54e06a31fcbcb 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/SerialDiffIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/SerialDiffIT.java @@ -150,6 +150,9 @@ public void setupSuiteScopeCluster() throws Exception { } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java index bb450f1cc43ee..953478eebb394 100644 --- a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java +++ b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java @@ -49,7 +49,7 @@ public void testCustomWordDelimiterQueryString() { .setMapping("field1", "type=text,analyzer=my_analyzer", "field2", "type=text,analyzer=my_analyzer") ); - prepareIndex("test").setId("1").setSource("field1", "foo bar baz", "field2", "not needed").get(); + indexDoc("test", "1", "field1", "foo bar baz", "field2", "not needed"); refresh(); assertHitCount( diff --git a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java index e8164bfbb8f36..fa104284b5fbb 100644 --- a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java +++ b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java @@ -70,7 +70,7 @@ private void testSynonymsUpdate(boolean preview) throws FileNotFoundException, I .setMapping("field", "type=text,analyzer=standard,search_analyzer=my_synonym_analyzer") ); - prepareIndex("test").setId("1").setSource("field", "foo").get(); + indexDoc("test", "1", "field", "foo"); assertNoFailures(indicesAdmin().prepareRefresh("test").get()); assertHitCount(prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")), 1L); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java index 2693245ac2757..7a9980c816665 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.analysis.common; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.Operator; @@ -109,7 +110,9 @@ public void testNgramHighlightingWithBrokenPositions() throws IOException { .putList("analysis.analyzer.search_autocomplete.filter", "lowercase", "wordDelimiter") ) ); - prepareIndex("test").setId("1").setSource("name", "ARCOTEL Hotels Deutschland").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("name", "ARCOTEL Hotels Deutschland"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); assertResponse( prepareSearch("test").setQuery(matchQuery("name.autocomplete", "deut tel").operator(Operator.OR)) @@ -151,7 +154,7 @@ public void testMultiPhraseCutoff() throws IOException { ); ensureGreen(); - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource( "body", "Test: http://www.facebook.com http://elasticsearch.org " @@ -159,8 +162,9 @@ public void testMultiPhraseCutoff() throws IOException { + "a test for highlighting feature Test: http://www.facebook.com " + "http://elasticsearch.org http://xing.com http://cnn.com http://quora.com " + "http://twitter.com this is a test for highlighting feature" - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); assertResponse( prepareSearch().setQuery(matchPhraseQuery("body", "Test: http://www.facebook.com ")) @@ -211,7 +215,10 @@ public void testSynonyms() throws IOException { ); ensureGreen(); - prepareIndex("test").setId("0").setSource("field1", "The quick brown fox jumps over the lazy dog").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("0") + .setSource("field1", "The quick brown fox jumps over the lazy dog"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); for (String highlighterType : new String[] { "plain", "fvh", "unified" }) { logger.info("--> highlighting (type=" + highlighterType + ") and searching on field1"); @@ -248,10 +255,23 @@ public void testPhrasePrefix() throws IOException { ensureGreen(); - prepareIndex("first_test_index").setId("0") - .setSource("field0", "The quick brown fox jumps over the lazy dog", "field1", "The quick brown fox jumps over the lazy dog") - .get(); - prepareIndex("first_test_index").setId("1").setSource("field1", "The quick browse button is a fancy thing, right bro?").get(); + { + IndexRequestBuilder indexRequestBuilder = prepareIndex("first_test_index").setId("0") + .setSource( + "field0", + "The quick brown fox jumps over the lazy dog", + "field1", + "The quick brown fox jumps over the lazy dog" + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } + { + IndexRequestBuilder indexRequestBuilder = prepareIndex("first_test_index").setId("1") + .setSource("field1", "The quick browse button is a fancy thing, right bro?"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } refresh(); logger.info("--> highlighting and searching on field0"); @@ -324,20 +344,31 @@ public void testPhrasePrefix() throws IOException { "type=text,analyzer=synonym" ) ); - prepareIndex("second_test_index").setId("0") - .setSource( - "type", - "type2", - "field4", - "The quick brown fox jumps over the lazy dog", - "field3", - "The quick brown fox jumps over the lazy dog" - ) - .get(); - prepareIndex("second_test_index").setId("1") - .setSource("type", "type2", "field4", "The quick browse button is a fancy thing, right bro?") - .get(); - prepareIndex("second_test_index").setId("2").setSource("type", "type2", "field4", "a quick fast blue car").get(); + { + IndexRequestBuilder indexRequestBuilder = prepareIndex("second_test_index").setId("0") + .setSource( + "type", + "type2", + "field4", + "The quick brown fox jumps over the lazy dog", + "field3", + "The quick brown fox jumps over the lazy dog" + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } + { + IndexRequestBuilder indexRequestBuilder = prepareIndex("second_test_index").setId("1") + .setSource("type", "type2", "field4", "The quick browse button is a fancy thing, right bro?"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } + { + IndexRequestBuilder indexRequestBuilder = prepareIndex("second_test_index").setId("2") + .setSource("type", "type2", "field4", "a quick fast blue car"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } refresh(); assertResponse( diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 4d45f3027af5c..c31537160c759 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -51,6 +51,7 @@ import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -76,6 +77,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; @@ -249,9 +251,8 @@ public void testOtherWriteOps() throws Exception { client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); try ( - BulkRequest bulkRequest = new BulkRequest().add( - new IndexRequest(dataStreamName).source("{\"@timestamp1\": \"2020-12-12\"}", XContentType.JSON) - ) + IndexRequest indexRequest = new IndexRequest(dataStreamName); + BulkRequest bulkRequest = new BulkRequest().add(indexRequest.source("{\"@timestamp1\": \"2020-12-12\"}", XContentType.JSON)) ) { BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems(), arrayWithSize(1)); @@ -268,11 +269,10 @@ public void testOtherWriteOps() throws Exception { containsString("only write ops with an op_type of create are allowed in data streams") ); } - try ( - BulkRequest bulkRequest = new BulkRequest().add( - new UpdateRequest(dataStreamName, "_id").doc("{\"@timestamp1\": \"2020-12-12\"}", XContentType.JSON) - ) - ) { + try (BulkRequest bulkRequest = new BulkRequest()) { + UpdateRequest updateRequest = new UpdateRequest(dataStreamName, "_id"); + bulkRequest.add(updateRequest.doc("{\"@timestamp1\": \"2020-12-12\"}", XContentType.JSON)); + updateRequest.decRef(); BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems(), arrayWithSize(1)); assertThat( @@ -282,13 +282,21 @@ public void testOtherWriteOps() throws Exception { } { IndexRequest indexRequest = new IndexRequest(dataStreamName).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON); - Exception e = expectThrows(IllegalArgumentException.class, client().index(indexRequest)); - assertThat(e.getMessage(), equalTo("only write ops with an op_type of create are allowed in data streams")); + try { + Exception e = expectThrows(IllegalArgumentException.class, client().index(indexRequest)); + assertThat(e.getMessage(), equalTo("only write ops with an op_type of create are allowed in data streams")); + } finally { + indexRequest.decRef(); + } } { UpdateRequest updateRequest = new UpdateRequest(dataStreamName, "_id").doc("{}", XContentType.JSON); - Exception e = expectThrows(IllegalArgumentException.class, client().update(updateRequest)); - assertThat(e.getMessage(), equalTo("only write ops with an op_type of create are allowed in data streams")); + try { + Exception e = expectThrows(IllegalArgumentException.class, client().update(updateRequest)); + assertThat(e.getMessage(), equalTo("only write ops with an op_type of create are allowed in data streams")); + } finally { + updateRequest.decRef(); + } } { DeleteRequest deleteRequest = new DeleteRequest(dataStreamName, "_id"); @@ -299,14 +307,18 @@ public void testOtherWriteOps() throws Exception { IndexRequest indexRequest = new IndexRequest(dataStreamName).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) .opType(DocWriteRequest.OpType.CREATE); DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertThat(indexResponse.getIndex(), backingIndexEqualTo(dataStreamName, 1)); } - try ( - BulkRequest bulkRequest = new BulkRequest().add( - new IndexRequest(dataStreamName).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) - .opType(DocWriteRequest.OpType.CREATE) - ) - ) { + try (BulkRequest bulkRequest = new BulkRequest()) { + IndexRequest indexRequest = new IndexRequest(dataStreamName); + try { + bulkRequest.add( + indexRequest.source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON).opType(DocWriteRequest.OpType.CREATE) + ); + } finally { + indexRequest.decRef(); + } BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet(); assertThat(bulkItemResponses.getItems()[0].getIndex(), backingIndexEqualTo(dataStreamName, 1)); } @@ -340,6 +352,11 @@ public void testOtherWriteOps() throws Exception { .setIfSeqNo(1) .setIfPrimaryTerm(1) ); + for (DocWriteRequest request : bulkRequest.requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems(), arrayWithSize(11)); { @@ -547,12 +564,17 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { aliasesRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).alias(aliasToDataStream).index("logs-foobar")); assertAcked(indicesAdmin().aliases(aliasesRequest).actionGet()); - verifyResolvability( - dataStreamName, - prepareIndex(dataStreamName).setSource("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) - .setOpType(DocWriteRequest.OpType.CREATE), - false - ); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex(dataStreamName); + try { + verifyResolvability( + dataStreamName, + indexRequestBuilder1.setSource("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) + .setOpType(DocWriteRequest.OpType.CREATE), + false + ); + } finally { + indexRequestBuilder1.request().decRef(); + } verifyResolvability(dataStreamName, indicesAdmin().prepareRefresh(dataStreamName), false); verifyResolvability(dataStreamName, prepareSearch(dataStreamName), false, 1); verifyResolvability( @@ -590,12 +612,17 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { request = new CreateDataStreamAction.Request("logs-barbaz"); client().execute(CreateDataStreamAction.INSTANCE, request).actionGet(); - verifyResolvability( - "logs-barbaz", - prepareIndex("logs-barbaz").setSource("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) - .setOpType(DocWriteRequest.OpType.CREATE), - false - ); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("logs-barbaz"); + try { + verifyResolvability( + "logs-barbaz", + indexRequestBuilder2.setSource("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) + .setOpType(DocWriteRequest.OpType.CREATE), + false + ); + } finally { + indexRequestBuilder2.request().decRef(); + } String wildcardExpression = "logs*"; verifyResolvability(wildcardExpression, indicesAdmin().prepareRefresh(wildcardExpression), false); @@ -708,14 +735,20 @@ public void testAliasActionsOnDataStreams() throws Exception { public void testDataSteamAliasWithFilter() throws Exception { putComposableIndexTemplate("id1", List.of("logs-*")); String dataStreamName = "logs-foobar"; - prepareIndex(dataStreamName).setId("1") - .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"x\"}", XContentType.JSON) - .setOpType(DocWriteRequest.OpType.CREATE) - .get(); - prepareIndex(dataStreamName).setId("2") - .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"y\"}", XContentType.JSON) - .setOpType(DocWriteRequest.OpType.CREATE) - .get(); + { + IndexRequestBuilder indexRequestBuilder = prepareIndex(dataStreamName).setId("1") + .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"x\"}", XContentType.JSON) + .setOpType(DocWriteRequest.OpType.CREATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } + { + IndexRequestBuilder indexRequestBuilder = prepareIndex(dataStreamName).setId("2") + .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"y\"}", XContentType.JSON) + .setOpType(DocWriteRequest.OpType.CREATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } refresh(dataStreamName); AliasActions addAction = new AliasActions(AliasActions.Type.ADD).index(dataStreamName) @@ -781,14 +814,20 @@ public void testDataSteamAliasWithFilter() throws Exception { public void testSearchFilteredAndUnfilteredAlias() throws Exception { putComposableIndexTemplate("id1", List.of("logs-*")); String dataStreamName = "logs-foobar"; - prepareIndex(dataStreamName).setId("1") - .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"x\"}", XContentType.JSON) - .setOpType(DocWriteRequest.OpType.CREATE) - .get(); - prepareIndex(dataStreamName).setId("2") - .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"y\"}", XContentType.JSON) - .setOpType(DocWriteRequest.OpType.CREATE) - .get(); + { + IndexRequestBuilder indexRequestBuilder = prepareIndex(dataStreamName).setId("1") + .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"x\"}", XContentType.JSON) + .setOpType(DocWriteRequest.OpType.CREATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } + { + IndexRequestBuilder indexRequestBuilder = prepareIndex(dataStreamName).setId("2") + .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"y\"}", XContentType.JSON) + .setOpType(DocWriteRequest.OpType.CREATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } refresh(dataStreamName); AliasActions addFilteredAliasAction = new AliasActions(AliasActions.Type.ADD).index(dataStreamName) @@ -1153,34 +1192,41 @@ public void testUpdateIndexSettingsViaDataStream() throws Exception { public void testIndexDocsWithCustomRoutingTargetingDataStreamIsNotAllowed() throws Exception { putComposableIndexTemplate("id1", List.of("logs-foo*")); - // Index doc that triggers creation of a data stream String dataStream = "logs-foobar"; - IndexRequest indexRequest = new IndexRequest(dataStream).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) - .opType(DocWriteRequest.OpType.CREATE); - DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); - assertThat(indexResponse.getIndex(), backingIndexEqualTo(dataStream, 1)); + { + // Index doc that triggers creation of a data stream + IndexRequest indexRequest = new IndexRequest(dataStream).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) + .opType(DocWriteRequest.OpType.CREATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertThat(indexResponse.getIndex(), backingIndexEqualTo(dataStream, 1)); + } // Index doc with custom routing that targets the data stream IndexRequest indexRequestWithRouting = new IndexRequest(dataStream).source("@timestamp", System.currentTimeMillis()) .opType(DocWriteRequest.OpType.CREATE) .routing("custom"); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, client().index(indexRequestWithRouting)); - assertThat( - exception.getMessage(), - is( - "index request targeting data stream [logs-foobar] specifies a custom routing " - + "but the [allow_custom_routing] setting was not enabled in the data stream's template." - ) - ); + try { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, client().index(indexRequestWithRouting)); + assertThat( + exception.getMessage(), + is( + "index request targeting data stream [logs-foobar] specifies a custom routing " + + "but the [allow_custom_routing] setting was not enabled in the data stream's template." + ) + ); + } finally { + indexRequestWithRouting.decRef(); + } // Bulk indexing with custom routing targeting the data stream is also prohibited try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < 10; i++) { + IndexRequest indexRequest = new IndexRequest(dataStream); bulkRequest.add( - new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) - .routing("bulk-request-routing") - .source("{}", XContentType.JSON) + indexRequest.opType(DocWriteRequest.OpType.CREATE).routing("bulk-request-routing").source("{}", XContentType.JSON) ); + indexRequest.decRef(); } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); @@ -1210,24 +1256,30 @@ public void testIndexDocsWithCustomRoutingAllowed() throws Exception { ).actionGet(); // Index doc that triggers creation of a data stream String dataStream = "logs-foobar"; - IndexRequest indexRequest = new IndexRequest(dataStream).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) - .opType(DocWriteRequest.OpType.CREATE) - .routing("custom"); - DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); - assertThat(indexResponse.getIndex(), backingIndexEqualTo(dataStream, 1)); + { + IndexRequest indexRequest = new IndexRequest(dataStream).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) + .opType(DocWriteRequest.OpType.CREATE) + .routing("custom"); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertThat(indexResponse.getIndex(), backingIndexEqualTo(dataStream, 1)); + } // Index doc with custom routing that targets the data stream IndexRequest indexRequestWithRouting = new IndexRequest(dataStream).source("@timestamp", System.currentTimeMillis()) .opType(DocWriteRequest.OpType.CREATE) .routing("custom"); client().index(indexRequestWithRouting).actionGet(); + indexRequestWithRouting.decRef(); // Bulk indexing with custom routing targeting the data stream try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < 10; i++) { + IndexRequest indexRequest = new IndexRequest(dataStream); bulkRequest.add( - new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) + indexRequest.opType(DocWriteRequest.OpType.CREATE) .source("@timestamp", System.currentTimeMillis()) .routing("bulk-request-routing") ); + indexRequest.decRef(); } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); for (BulkItemResponse responseItem : bulkResponse.getItems()) { @@ -1243,6 +1295,7 @@ public void testIndexDocsWithCustomRoutingTargetingBackingIndex() throws Excepti IndexRequest indexRequest = new IndexRequest("logs-foobar").source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) .opType(DocWriteRequest.OpType.CREATE); DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertThat(indexResponse.getIndex(), backingIndexEqualTo("logs-foobar", 1)); String backingIndex = indexResponse.getIndex(); @@ -1254,6 +1307,7 @@ public void testIndexDocsWithCustomRoutingTargetingBackingIndex() throws Excepti .setIfPrimaryTerm(indexResponse.getPrimaryTerm()) .setIfSeqNo(indexResponse.getSeqNo()); DocWriteResponse response = client().index(indexRequestWithRouting).actionGet(); + indexRequestWithRouting.decRef(); assertThat(response.getIndex(), equalTo(backingIndex)); } @@ -1323,8 +1377,12 @@ public void testNoTimestampInDocument() throws Exception { client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create").source("{}", XContentType.JSON); - Exception e = expectThrows(Exception.class, client().index(indexRequest)); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); + try { + Exception e = expectThrows(Exception.class, client().index(indexRequest)); + assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); + } finally { + indexRequest.decRef(); + } } public void testMultipleTimestampValuesInDocument() throws Exception { @@ -1335,8 +1393,12 @@ public void testMultipleTimestampValuesInDocument() throws Exception { IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create") .source("{\"@timestamp\": [\"2020-12-12\",\"2022-12-12\"]}", XContentType.JSON); - Exception e = expectThrows(Exception.class, client().index(indexRequest)); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] encountered multiple values")); + try { + Exception e = expectThrows(Exception.class, client().index(indexRequest)); + assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] encountered multiple values")); + } finally { + indexRequest.decRef(); + } } public void testMixedAutoCreate() throws Exception { @@ -1357,6 +1419,11 @@ public void testMixedAutoCreate() throws Exception { bulkRequest.add(new IndexRequest("logs-foobaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); bulkRequest.add(new IndexRequest("logs-barfoo").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + for (DocWriteRequest request : bulkRequest.requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); } @@ -1366,6 +1433,11 @@ public void testMixedAutoCreate() throws Exception { bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + for (DocWriteRequest request : bulkRequest.requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); } @@ -1377,6 +1449,11 @@ public void testMixedAutoCreate() throws Exception { bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); bulkRequest.add(new IndexRequest("logs-barfoo3").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + for (DocWriteRequest request : bulkRequest.requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); } @@ -1415,7 +1492,9 @@ public void testAutoCreateV1TemplateNoDataStream() { indicesAdmin().putTemplate(v1Request).actionGet(); try (BulkRequest bulkRequest = new BulkRequest()) { - bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest("logs-foobar"); + bulkRequest.add(indexRequest.opType(CREATE).source("{}", XContentType.JSON)); + indexRequest.decRef(); BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); } @@ -1887,10 +1966,15 @@ static void indexDocs(String dataStream, int numDocs) { try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < numDocs; i++) { String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); - bulkRequest.add( - new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) - .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) - ); + IndexRequest indexRequest = new IndexRequest(dataStream); + try { + bulkRequest.add( + indexRequest.opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems().length, equalTo(numDocs)); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java index f35ce66acbd7f..f21730166615c 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java @@ -238,13 +238,18 @@ static void indexDocs(String index, int numDocs, String fieldPrefix) { try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < numDocs; i++) { String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); - bulkRequest.add( - new IndexRequest(index).opType(DocWriteRequest.OpType.CREATE) - .source( - String.format(Locale.ROOT, "{\"%s\":\"%s\"}", fieldPrefix + DEFAULT_TIMESTAMP_FIELD, value), - XContentType.JSON - ) - ); + IndexRequest indexRequest = new IndexRequest(index); + try { + bulkRequest.add( + indexRequest.opType(DocWriteRequest.OpType.CREATE) + .source( + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", fieldPrefix + DEFAULT_TIMESTAMP_FIELD, value), + XContentType.JSON + ) + ); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems().length, equalTo(numDocs)); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index da782cfd86ce2..237021887ae51 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; @@ -116,10 +117,9 @@ public void setup() throws Exception { ds2BackingIndexName = dsBackingIndexName.replace("-ds-", "-ds2-"); otherDs2BackingIndexName = otherDsBackingIndexName.replace("-other-ds-", "-other-ds2-"); - DocWriteResponse indexResponse = client.prepareIndex("ds") - .setOpType(DocWriteRequest.OpType.CREATE) - .setSource(DOCUMENT_SOURCE) - .get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex("ds"); + DocWriteResponse indexResponse = indexRequestBuilder.setOpType(DocWriteRequest.OpType.CREATE).setSource(DOCUMENT_SOURCE).get(); + indexRequestBuilder.request().decRef(); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); id = indexResponse.getId(); @@ -303,10 +303,9 @@ public void testSnapshotAndRestoreInPlace() { } public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exception { - DocWriteResponse indexResponse = client.prepareIndex("other-ds") - .setOpType(DocWriteRequest.OpType.CREATE) - .setSource(DOCUMENT_SOURCE) - .get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex("other-ds"); + DocWriteResponse indexResponse = indexRequestBuilder.setOpType(DocWriteRequest.OpType.CREATE).setSource(DOCUMENT_SOURCE).get(); + indexRequestBuilder.request().decRef(); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); String id2 = indexResponse.getId(); @@ -857,11 +856,12 @@ public void testDeleteDataStreamDuringSnapshot() throws Exception { logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { - client1.prepareIndex(dataStream) - .setOpType(DocWriteRequest.OpType.CREATE) + IndexRequestBuilder indexRequestBuilder = client1.prepareIndex(dataStream); + indexRequestBuilder.setOpType(DocWriteRequest.OpType.CREATE) .setId(Integer.toString(i)) .setSource(Collections.singletonMap("@timestamp", "2020-12-12")) .get(); + indexRequestBuilder.request().decRef(); } refresh(); assertDocCount(dataStream, 100L); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java index 698656dfa7406..dedbce6d4eb2d 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.index.IndexNotFoundException; @@ -64,10 +65,12 @@ public void testSystemDataStreamInGlobalState() throws Exception { } // Index a doc so that a concrete backing index will be created - DocWriteResponse indexRepsonse = prepareIndex(SYSTEM_DATA_STREAM_NAME).setId("42") + IndexRequestBuilder indexRequestBuilder = prepareIndex(SYSTEM_DATA_STREAM_NAME); + DocWriteResponse indexRepsonse = indexRequestBuilder.setId("42") .setSource("{ \"@timestamp\": \"2099-03-08T11:06:07.000Z\", \"name\": \"my-name\" }", XContentType.JSON) .setOpType(DocWriteRequest.OpType.CREATE) .get(); + indexRequestBuilder.request().decRef(); assertThat(indexRepsonse.status().getStatus(), oneOf(200, 201)); { @@ -160,19 +163,27 @@ public void testSystemDataStreamInFeatureState() throws Exception { assertTrue(response.isAcknowledged()); } - // Index a doc so that a concrete backing index will be created - DocWriteResponse indexToDataStreamResponse = prepareIndex(SYSTEM_DATA_STREAM_NAME).setId("42") - .setSource("{ \"@timestamp\": \"2099-03-08T11:06:07.000Z\", \"name\": \"my-name\" }", XContentType.JSON) - .setOpType(DocWriteRequest.OpType.CREATE) - .get(); - assertThat(indexToDataStreamResponse.status().getStatus(), oneOf(200, 201)); + { + // Index a doc so that a concrete backing index will be created + IndexRequestBuilder indexRequestBuilder = prepareIndex(SYSTEM_DATA_STREAM_NAME).setId("42"); + DocWriteResponse indexToDataStreamResponse = indexRequestBuilder.setSource( + "{ \"@timestamp\": \"2099-03-08T11:06:07.000Z\", \"name\": \"my-name\" }", + XContentType.JSON + ).setOpType(DocWriteRequest.OpType.CREATE).get(); + indexRequestBuilder.request().decRef(); + assertThat(indexToDataStreamResponse.status().getStatus(), oneOf(200, 201)); + } - // Index a doc so that a concrete backing index will be created - DocWriteResponse indexResponse = prepareIndex("my-index").setId("42") - .setSource("{ \"name\": \"my-name\" }", XContentType.JSON) - .setOpType(DocWriteRequest.OpType.CREATE) - .get(); - assertThat(indexResponse.status().getStatus(), oneOf(200, 201)); + { + // Index a doc so that a concrete backing index will be created + IndexRequestBuilder indexRequestBuilder = prepareIndex("my-index"); + DocWriteResponse indexResponse = indexRequestBuilder.setId("42") + .setSource("{ \"name\": \"my-name\" }", XContentType.JSON) + .setOpType(DocWriteRequest.OpType.CREATE) + .get(); + indexRequestBuilder.request().decRef(); + assertThat(indexResponse.status().getStatus(), oneOf(200, 201)); + } { GetDataStreamAction.Request request = new GetDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME }); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index 0fc86d4126d18..9237206923609 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -147,6 +147,7 @@ public void testTimeRanges() throws Exception { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); var indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); backingIndexName = indexResponse.getIndex(); } @@ -159,15 +160,20 @@ public void testTimeRanges() throws Exception { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); indexRequest.source(DOC.replace("$time", formatInstant(endTime.minusSeconds(1))), XContentType.JSON); var indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertThat(indexResponse.getIndex(), equalTo(backingIndexName)); } // index doc beyond range and check failure { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); - time = randomBoolean() ? endTime : endTime.plusSeconds(randomIntBetween(1, 99)); - indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); - expectThrows(IllegalArgumentException.class, () -> client().index(indexRequest).actionGet()); + try { + time = randomBoolean() ? endTime : endTime.plusSeconds(randomIntBetween(1, 99)); + indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); + expectThrows(IllegalArgumentException.class, () -> client().index(indexRequest).actionGet()); + } finally { + indexRequest.decRef(); + } } // Fetch UpdateTimeSeriesRangeService and increment time range of latest backing index: @@ -181,6 +187,7 @@ public void testTimeRanges() throws Exception { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); var indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertThat(indexResponse.getIndex(), equalTo(backingIndexName)); } @@ -200,6 +207,7 @@ public void testTimeRanges() throws Exception { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); var indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertThat(indexResponse.getIndex(), equalTo(newBackingIndexName)); } time = Instant.ofEpochMilli(randomLongBetween(newStartTime.toEpochMilli(), newEndTime.toEpochMilli() - 1)); @@ -207,6 +215,7 @@ public void testTimeRanges() throws Exception { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); var indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertThat(indexResponse.getIndex(), equalTo(newBackingIndexName)); } time = Instant.ofEpochMilli(newEndTime.toEpochMilli() - 1); @@ -214,6 +223,7 @@ public void testTimeRanges() throws Exception { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); var indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertThat(indexResponse.getIndex(), equalTo(newBackingIndexName)); } @@ -223,6 +233,7 @@ public void testTimeRanges() throws Exception { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); var indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertThat(indexResponse.getIndex(), equalTo(backingIndexName)); } } @@ -379,6 +390,7 @@ public void testSkippingShards() throws Exception { var indexRequest = new IndexRequest("pattern-1").opType(DocWriteRequest.OpType.CREATE).setRefreshPolicy("true"); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); client().index(indexRequest).actionGet(); + indexRequest.decRef(); } { var request = new TransportPutComposableIndexTemplateAction.Request("id2"); @@ -393,6 +405,7 @@ public void testSkippingShards() throws Exception { var indexRequest = new IndexRequest("pattern-2").opType(DocWriteRequest.OpType.CREATE).setRefreshPolicy("true"); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); client().index(indexRequest).actionGet(); + indexRequest.decRef(); } { var matchingRange = new SearchSourceBuilder().query( @@ -461,6 +474,7 @@ public void testTrimId() throws Exception { var indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); bulkRequest.add(indexRequest); + indexRequest.decRef(); time = time.plusMillis(1); } var bulkResponse = client().bulk(bulkRequest).actionGet(); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index 507b05182eb19..d4d9ccc053072 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -837,10 +837,15 @@ static void indexDocs(String dataStream, int numDocs) { try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < numDocs; i++) { String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); - bulkRequest.add( - new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) - .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) - ); + IndexRequest indexRequest = new IndexRequest(dataStream); + try { + bulkRequest.add( + indexRequest.opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems().length, equalTo(numDocs)); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java index dcdc8f7452254..bbfbc4f892c3c 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java @@ -325,10 +325,15 @@ static void indexDocs(String dataStream, int numDocs) { try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < numDocs; i++) { String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); - bulkRequest.add( - new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) - .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) - ); + IndexRequest indexRequest = new IndexRequest(dataStream); + try { + bulkRequest.add( + indexRequest.opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems().length, equalTo(numDocs)); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java index 0ef4ea5ef451a..1ee3ad149f164 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java @@ -246,16 +246,16 @@ private long createDocument(String dataStreamName) throws Exception { // Get some randomized but reasonable timestamps on the data since not all of it is guaranteed to arrive in order. long timeSeed = System.currentTimeMillis(); long timestamp = randomLongBetween(timeSeed - TimeUnit.HOURS.toMillis(5), timeSeed); - client().index( - new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE) - .source( - JsonXContent.contentBuilder() - .startObject() - .field("@timestamp", timestamp) - .field("data", randomAlphaOfLength(25)) - .endObject() - ) - ).get(); + IndexRequest indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE) + .source( + JsonXContent.contentBuilder() + .startObject() + .field("@timestamp", timestamp) + .field("data", randomAlphaOfLength(25)) + .endObject() + ); + client().index(indexRequest).get(); + indexRequest.decRef(); indicesAdmin().refresh(new RefreshRequest(".ds-" + dataStreamName + "*").indicesOptions(IndicesOptions.lenientExpandOpenHidden())) .get(); return timestamp; diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java index 0ff34cf687500..98ab5707be9ec 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -86,14 +87,7 @@ public void testFailureInConditionalProcessor() { ] }""", MockScriptEngine.NAME)), XContentType.JSON).get(); - Exception e = expectThrows( - Exception.class, - () -> prepareIndex("index").setId("1") - .setSource("x", 0) - .setPipeline(pipelineId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get() - ); + Exception e = expectThrows(Exception.class, () -> indexDocImmediate("index", "1", pipelineId, "x", 0)); assertTrue(e.getMessage().contains("this script always fails")); NodesStatsResponse r = clusterAdmin().prepareNodesStats(internalCluster().getNodeNames()).setIngest(true).get(); @@ -143,19 +137,11 @@ public Settings onNodeStopped(String nodeName) { checkPipelineExists.accept(pipelineIdWithoutScript); checkPipelineExists.accept(pipelineIdWithScript); - prepareIndex("index").setId("1") - .setSource("x", 0) - .setPipeline(pipelineIdWithoutScript) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + indexDocImmediate("index", "1", pipelineIdWithoutScript, "x", 0); IllegalStateException exception = expectThrows( IllegalStateException.class, - () -> prepareIndex("index").setId("2") - .setSource("x", 0) - .setPipeline(pipelineIdWithScript) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get() + () -> indexDocImmediate("index", "2", pipelineIdWithScript, "x", 0) ); assertThat( exception.getMessage(), @@ -191,7 +177,7 @@ public void testPipelineWithScriptProcessorThatHasStoredScript() throws Exceptio }"""); clusterAdmin().preparePutPipeline("_id", pipeline, XContentType.JSON).get(); - prepareIndex("index").setId("1").setSource("x", 0).setPipeline("_id").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("index", "1", "_id", "x", 0); Map source = client().prepareGet("index", "1").get().getSource(); assertThat(source.get("x"), equalTo(0)); @@ -205,7 +191,7 @@ public void testPipelineWithScriptProcessorThatHasStoredScript() throws Exceptio internalCluster().fullRestart(); ensureYellow("index"); - prepareIndex("index").setId("2").setSource("x", 0).setPipeline("_id").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("index", "2", "_id", "x", 0); source = client().prepareGet("index", "2").get().getSource(); assertThat(source.get("x"), equalTo(0)); @@ -225,7 +211,7 @@ public void testWithDedicatedIngestNode() throws Exception { }"""); clusterAdmin().preparePutPipeline("_id", pipeline, XContentType.JSON).get(); - prepareIndex("index").setId("1").setSource("x", 0).setPipeline("_id").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("index", "1", "_id", "x", 0); Map source = client().prepareGet("index", "1").get().getSource(); assertThat(source.get("x"), equalTo(0)); @@ -234,12 +220,16 @@ public void testWithDedicatedIngestNode() throws Exception { logger.info("Stopping"); internalCluster().restartNode(node, new InternalTestCluster.RestartCallback()); - client(ingestNode).prepareIndex("index") - .setId("2") - .setSource("x", 0) - .setPipeline("_id") - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + IndexRequestBuilder indexRequestBuilder = client(ingestNode).prepareIndex("index"); + try { + indexRequestBuilder.setId("2") + .setSource("x", 0) + .setPipeline("_id") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + } finally { + indexRequestBuilder.request().decRef(); + } source = client(ingestNode).prepareGet("index", "2").get().getSource(); assertThat(source.get("x"), equalTo(0)); @@ -283,35 +273,45 @@ public boolean validateClusterForming() { }); // this one should fail - assertThat( - expectThrows( - ClusterBlockException.class, - () -> prepareIndex("index").setId("fails") - .setSource("x", 1) - .setTimeout(TimeValue.timeValueMillis(100)) // 100ms, to fail quickly - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(timeout) - ).getMessage(), - equalTo("blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];") - ); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("index"); + try { + assertThat( + expectThrows( + ClusterBlockException.class, + () -> indexRequestBuilder1.setId("fails") + .setSource("x", 1) + .setTimeout(TimeValue.timeValueMillis(100)) // 100ms, to fail quickly + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(timeout) + ).getMessage(), + equalTo("blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];") + ); + } finally { + indexRequestBuilder1.request().decRef(); + } // but this one should pass since it has a longer timeout final PlainActionFuture future = new PlainActionFuture<>(); - prepareIndex("index").setId("passes1") - .setSource("x", 2) - .setTimeout(TimeValue.timeValueSeconds(60)) // wait for second node to start in below - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .execute(future); - - // so the cluster state can be recovered - internalCluster().startNode(Settings.builder().put(GatewayService.RECOVER_AFTER_DATA_NODES_SETTING.getKey(), "1")); - ensureYellow("index"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("index"); + try { + indexRequestBuilder2.setId("passes1") + .setSource("x", 2) + .setTimeout(TimeValue.timeValueSeconds(60)) // wait for second node to start in below + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute(future); - final DocWriteResponse indexResponse = future.actionGet(timeout); - assertThat(indexResponse.status(), equalTo(RestStatus.CREATED)); - assertThat(indexResponse.getResult(), equalTo(DocWriteResponse.Result.CREATED)); + // so the cluster state can be recovered + internalCluster().startNode(Settings.builder().put(GatewayService.RECOVER_AFTER_DATA_NODES_SETTING.getKey(), "1")); + ensureYellow("index"); + + final DocWriteResponse indexResponse = future.actionGet(timeout); + assertThat(indexResponse.status(), equalTo(RestStatus.CREATED)); + assertThat(indexResponse.getResult(), equalTo(DocWriteResponse.Result.CREATED)); + } finally { + indexRequestBuilder2.request().decRef(); + } - prepareIndex("index").setId("passes2").setSource("x", 3).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("index", "passes2", null, "x", 3); // successfully indexed documents should have the value field set by the pipeline Map source = client().prepareGet("index", "passes1").get(timeout).getSource(); @@ -326,4 +326,17 @@ public boolean validateClusterForming() { source = client().prepareGet("index", "fails").get(timeout).getSource(); assertNull(source); } + + private void indexDocImmediate(String index, String id, String pipeline, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + if (pipeline != null) { + indexRequestBuilder.setPipeline(pipeline); + } + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverWithPipelinesIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverWithPipelinesIT.java index c7954eec58ef2..660c26a0e71d5 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverWithPipelinesIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverWithPipelinesIT.java @@ -60,11 +60,14 @@ public void testDocumentIsReportedWithPipelines() throws IOException { """); clusterAdmin().putPipeline(new PutPipelineRequest("pipeline", pipelineBody, XContentType.JSON)).actionGet(); - client().index( - new IndexRequest(TEST_INDEX_NAME).setPipeline("pipeline") - .id("1") - .source(jsonBuilder().startObject().field("test", "I am sam i am").endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest(TEST_INDEX_NAME); + try { + client().index( + indexRequest.setPipeline("pipeline").id("1").source(jsonBuilder().startObject().field("test", "I am sam i am").endObject()) + ).actionGet(); + } finally { + indexRequest.decRef(); + } assertTrue(hasWrappedParser); // there are more assertions in a TestDocumentParsingObserver } diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java index 8490b17d535c6..ebfb2f9d4098e 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java @@ -102,6 +102,7 @@ public void testLazyLoading() throws IOException { indexRequest.setPipeline("geoip"); indexRequest.source(Collections.singletonMap("ip", "1.1.1.1")); final DocWriteResponse indexResponse = client(ingestNode).index(indexRequest).actionGet(); + indexRequest.decRef(); assertThat(indexResponse.status(), equalTo(RestStatus.CREATED)); // now the geo-IP database should be loaded on the ingest node assertDatabaseLoadStatus(ingestNode, true); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java index 3e04f7bfea2de..279b4e2db91ff 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -225,10 +225,15 @@ int indexChunks(String name, InputStream is, int chunk, String expectedMd5, long MessageDigest md = MessageDigests.md5(); for (byte[] buf = getChunk(is); buf.length != 0; buf = getChunk(is)) { md.update(buf); - IndexRequest indexRequest = new IndexRequest(DATABASES_INDEX).id(name + "_" + chunk + "_" + timestamp) - .create(true) - .source(XContentType.SMILE, "name", name, "chunk", chunk, "data", buf); - client.index(indexRequest).actionGet(); + IndexRequest indexRequest = new IndexRequest(DATABASES_INDEX); + try { + indexRequest.id(name + "_" + chunk + "_" + timestamp) + .create(true) + .source(XContentType.SMILE, "name", name, "chunk", chunk, "data", buf); + client.index(indexRequest).actionGet(); + } finally { + indexRequest.decRef(); + } chunk++; } diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java index 69e33863b0f2b..876bfba7a691c 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java @@ -8,6 +8,8 @@ package org.elasticsearch.script.expression; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchType; @@ -78,7 +80,7 @@ private SearchRequestBuilder buildRequest(String script, Object... params) { public void testBasic() throws Exception { createIndex("test"); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("foo", 4).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "foo", 4); assertResponse(buildRequest("doc['foo'] + 1"), rsp -> { assertEquals(1, rsp.getHits().getTotalHits().value); assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D); @@ -88,7 +90,7 @@ public void testBasic() throws Exception { public void testFunction() throws Exception { createIndex("test"); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("foo", 4).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "foo", 4); assertNoFailuresAndResponse(buildRequest("doc['foo'] + abs(1)"), rsp -> { assertEquals(1, rsp.getHits().getTotalHits().value); assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D); @@ -99,7 +101,7 @@ public void testBasicUsingDotValue() throws Exception { createIndex("test"); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("foo", 4).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "foo", 4); assertResponse(buildRequest("doc['foo'].value + 1"), rsp -> { assertEquals(1, rsp.getHits().getTotalHits().value); assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D); @@ -109,7 +111,7 @@ public void testBasicUsingDotValue() throws Exception { public void testScore() throws Exception { createIndex("test"); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("text", "hello goodbye"), prepareIndex("test").setId("2").setSource("text", "hello hello hello goodbye"), @@ -141,7 +143,7 @@ public void testScore() throws Exception { public void testDateMethods() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").setMapping("date0", "type=date", "date1", "type=date")); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), prepareIndex("test").setId("2").setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z") @@ -175,7 +177,7 @@ public void testDateMethods() throws Exception { public void testDateObjectMethods() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").setMapping("date0", "type=date", "date1", "type=date")); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), prepareIndex("test").setId("2").setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z") @@ -228,7 +230,7 @@ public void testMultiValueMethods() throws Exception { doc3.put("double0", new Double[] { 5.0d, 1.0d, 1.5d, -1.5d }); doc3.put("double1", 4.0d); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource(doc1), prepareIndex("test").setId("2").setSource(doc2), @@ -313,7 +315,7 @@ public void testMultiValueMethods() throws Exception { public void testInvalidDateMethodCall() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").setMapping("double", "type=double")); ensureGreen("test"); - indexRandom(true, prepareIndex("test").setId("1").setSource("double", "178000000.0")); + indexRandomAndDecRefRequests(true, prepareIndex("test").setId("1").setSource("double", "178000000.0")); try { buildRequest("doc['double'].getYear()").get(); fail(); @@ -334,7 +336,7 @@ public void testInvalidDateMethodCall() throws Exception { public void testSparseField() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").setMapping("x", "type=long", "y", "type=long")); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("id", 1, "x", 4), prepareIndex("test").setId("2").setSource("id", 2, "y", 2) @@ -350,7 +352,7 @@ public void testSparseField() throws Exception { public void testMissingField() throws Exception { createIndex("test"); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("x", 4).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "x", 4); try { buildRequest("doc['bogus']").get(); fail("Expected missing field to cause failure"); @@ -367,7 +369,7 @@ public void testMissingField() throws Exception { public void testParams() throws Exception { createIndex("test"); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("id", 1, "x", 10), prepareIndex("test").setId("2").setSource("id", 2, "x", 3), @@ -385,7 +387,7 @@ public void testParams() throws Exception { } public void testCompileFailure() { - prepareIndex("test").setId("1").setSource("x", 1).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "x", 1); try { buildRequest("garbage%@#%@").get(); fail("Expected expression compilation failure"); @@ -396,7 +398,7 @@ public void testCompileFailure() { } public void testNonNumericParam() { - prepareIndex("test").setId("1").setSource("x", 1).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "x", 1); try { buildRequest("a", "a", "astring").get(); fail("Expected string parameter to cause failure"); @@ -411,7 +413,7 @@ public void testNonNumericParam() { } public void testNonNumericField() { - prepareIndex("test").setId("1").setSource("text", "this is not a number").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "text", "this is not a number"); try { buildRequest("doc['text.keyword']").get(); fail("Expected text field to cause execution failure"); @@ -426,7 +428,7 @@ public void testNonNumericField() { } public void testInvalidGlobalVariable() { - prepareIndex("test").setId("1").setSource("foo", 5).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "foo", 5); try { buildRequest("bogus").get(); fail("Expected bogus variable to cause execution failure"); @@ -441,7 +443,7 @@ public void testInvalidGlobalVariable() { } public void testDocWithoutField() { - prepareIndex("test").setId("1").setSource("foo", 5).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "foo", 5); try { buildRequest("doc").get(); fail("Expected doc variable without field to cause execution failure"); @@ -456,7 +458,7 @@ public void testDocWithoutField() { } public void testInvalidFieldMember() { - prepareIndex("test").setId("1").setSource("foo", 5).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "foo", 5); try { buildRequest("doc['foo'].bogus").get(); fail("Expected bogus field member to cause execution failure"); @@ -474,7 +476,7 @@ public void testSpecialValueVariable() throws Exception { // i.e. _value for aggregations createIndex("test"); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("x", 5, "y", 1.2), prepareIndex("test").setId("2").setSource("x", 10, "y", 1.4), @@ -521,7 +523,7 @@ public void testStringSpecialValueVariable() throws Exception { // i.e. expression script for term aggregations, which is not allowed assertAcked(indicesAdmin().prepareCreate("test").setMapping("text", "type=keyword").get()); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("text", "hello"), prepareIndex("test").setId("2").setSource("text", "goodbye"), @@ -553,11 +555,11 @@ public void testStringSpecialValueVariable() throws Exception { // test to make sure expressions are not allowed to be used as update scripts public void testInvalidUpdateScript() throws Exception { + UpdateRequestBuilder urb = client().prepareUpdate().setIndex("test_index"); try { createIndex("test_index"); ensureGreen("test_index"); - indexRandom(true, prepareIndex("test_index").setId("1").setSource("text_field", "text")); - UpdateRequestBuilder urb = client().prepareUpdate().setIndex("test_index"); + indexRandomAndDecRefRequests(true, prepareIndex("test_index").setId("1").setSource("text_field", "text")); urb.setId("1"); urb.setScript(new Script(ScriptType.INLINE, ExpressionScriptEngine.NAME, "0", Collections.emptyMap())); urb.get(); @@ -567,6 +569,8 @@ public void testInvalidUpdateScript() throws Exception { assertThat(message + " should have contained failed to execute", message.contains("failed to execute"), equalTo(true)); message = e.getCause().getMessage(); assertThat(message, equalTo("Failed to compile inline script [0] using lang [expression]")); + } finally { + urb.request().decRef(); } } @@ -574,7 +578,7 @@ public void testInvalidUpdateScript() throws Exception { public void testPipelineAggregationScript() throws Exception { createIndex("agg_index"); ensureGreen("agg_index"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("agg_index").setId("1").setSource("one", 1.0, "two", 2.0, "three", 3.0, "four", 4.0), prepareIndex("agg_index").setId("2").setSource("one", 2.0, "two", 2.0, "three", 3.0, "four", 4.0), @@ -640,17 +644,17 @@ public void testGeo() throws Exception { xContentBuilder.endObject().endObject().endObject().endObject(); assertAcked(prepareCreate("test").setMapping(xContentBuilder)); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("name", "test") - .startObject("location") - .field("lat", 61.5240) - .field("lon", 105.3188) - .endObject() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("name", "test") + .startObject("location") + .field("lat", 61.5240) + .field("lon", 105.3188) + .endObject() + .endObject() + ); refresh(); // access .lat assertNoFailuresAndResponse(buildRequest("doc['location'].lat"), rsp -> { @@ -684,7 +688,7 @@ public void testBoolean() throws Exception { xContentBuilder.endObject().endObject().endObject().endObject(); assertAcked(prepareCreate("test").setMapping(xContentBuilder)); ensureGreen(); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("id", 1, "price", 1.0, "vip", true), prepareIndex("test").setId("2").setSource("id", 2, "price", 2.0, "vip", false), @@ -717,7 +721,7 @@ public void testBoolean() throws Exception { public void testFilterScript() throws Exception { createIndex("test"); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("id", 1, "foo", 1.0), prepareIndex("test").setId("2").setSource("id", 2, "foo", 0.0) @@ -730,4 +734,23 @@ public void testFilterScript() throws Exception { assertEquals(1.0D, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D); }); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private DocWriteResponse indexDocImmediate(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java index 121a6b01ea792..00f57ad4c4eeb 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java @@ -8,6 +8,8 @@ package org.elasticsearch.script.expression; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; @@ -41,13 +43,18 @@ protected Collection> nodePlugins() { public void testAllOpsDisabledIndexedScripts() throws IOException { clusterAdmin().preparePutStoredScript().setId("script1").setContent(new BytesArray(""" {"script": {"lang": "expression", "source": "2"} }"""), XContentType.JSON).get(); - prepareIndex("test").setId("1").setSource("{\"theField\":\"foo\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("{\"theField\":\"foo\"}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", "1"); try { - client().prepareUpdate("test", "1").setScript(new Script(ScriptType.STORED, null, "script1", Collections.emptyMap())).get(); + updateRequestBuilder.setScript(new Script(ScriptType.STORED, null, "script1", Collections.emptyMap())).get(); fail("update script should have been rejected"); } catch (Exception e) { assertThat(e.getMessage(), containsString("failed to execute script")); assertThat(e.getCause().getMessage(), containsString("Failed to compile stored script [script1] using lang [expression]")); + } finally { + updateRequestBuilder.request().decRef(); } try { prepareSearch().setSource( diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java index 0c3376c9c8a90..dc8d13b39cef1 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java @@ -63,6 +63,9 @@ public void testBasic() throws Exception { indexRequestBuilders[i] = prepareIndex("msearch").setId(String.valueOf(i)).setSource("odd", (i % 2 == 0), "group", (i % 3)); } indexRandom(true, indexRequestBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } final String template = Strings.toString( jsonBuilder().startObject() diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index 9684defa19080..67f959932eae4 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -8,17 +8,22 @@ package org.elasticsearch.script.mustache; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.DummyQueryParserPlugin; import org.elasticsearch.search.SearchService; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; @@ -56,8 +61,8 @@ protected Settings nodeSettings() { @Before public void setup() throws IOException { createIndex("test"); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("text", "value1").endObject()).get(); - prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("text", "value2").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("text", "value1").endObject()); + index("test", "2", jsonBuilder().startObject().field("text", "value2").endObject()); indicesAdmin().prepareRefresh().get(); } @@ -178,6 +183,11 @@ public void testIndexedTemplateClient() throws Exception { bulkRequestBuilder.add(prepareIndex("test").setId("3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON)); bulkRequestBuilder.add(prepareIndex("test").setId("4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON)); bulkRequestBuilder.add(prepareIndex("test").setId("5").setSource("{\"theField\":\"bar\"}", XContentType.JSON)); + for (DocWriteRequest request : bulkRequestBuilder.request().requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } bulkRequestBuilder.get(); indicesAdmin().prepareRefresh().get(); } @@ -277,6 +287,11 @@ public void testIndexedTemplate() throws Exception { bulkRequestBuilder.add(prepareIndex("test").setId("3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON)); bulkRequestBuilder.add(prepareIndex("test").setId("4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON)); bulkRequestBuilder.add(prepareIndex("test").setId("5").setSource("{\"theField\":\"bar\"}", XContentType.JSON)); + for (DocWriteRequest request : bulkRequestBuilder.request().requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } bulkRequestBuilder.get(); indicesAdmin().prepareRefresh().get(); } @@ -315,7 +330,7 @@ public void testIndexedTemplateOverwrite() throws Exception { createIndex("testindex"); ensureGreen("testindex"); - prepareIndex("testindex").setId("1").setSource(jsonBuilder().startObject().field("searchtext", "dev1").endObject()).get(); + index("testindex", "1", jsonBuilder().startObject().field("searchtext", "dev1").endObject()); indicesAdmin().prepareRefresh().get(); int iterations = randomIntBetween(2, 11); @@ -399,6 +414,11 @@ public void testIndexedTemplateWithArray() throws Exception { bulkRequestBuilder.add(prepareIndex("test").setId("3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON)); bulkRequestBuilder.add(prepareIndex("test").setId("4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON)); bulkRequestBuilder.add(prepareIndex("test").setId("5").setSource("{\"theField\":\"bar\"}", XContentType.JSON)); + for (DocWriteRequest request : bulkRequestBuilder.request().requests()) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } bulkRequestBuilder.get(); indicesAdmin().prepareRefresh().get(); } @@ -453,4 +473,13 @@ public void testCCSCheckCompatibility() throws Exception { public static void assertHitCount(SearchTemplateRequestBuilder requestBuilder, long expectedHitCount) { assertResponse(requestBuilder, response -> ElasticsearchAssertions.assertHitCount(response.getResponse(), expectedHitCount)); } + + private DocWriteResponse index(String index, String id, XContentBuilder source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java index 1bd6468c562f8..03cc835ae0892 100644 --- a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java +++ b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.painless.action; import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -187,7 +188,12 @@ private void setupTwoClusters() throws Exception { private int indexDocs(Client client, String index) { int numDocs = between(1, 10); for (int i = 0; i < numDocs; i++) { - client.prepareIndex(index).setSource(KEYWORD_FIELD, "my_value").get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); + try { + indexRequestBuilder.setSource(KEYWORD_FIELD, "my_value").get(); + } finally { + indexRequestBuilder.request().decRef(); + } } client.admin().indices().prepareRefresh(index).get(); return numDocs; diff --git a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/search/SyntheticSourceIT.java b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/search/SyntheticSourceIT.java index 0383999d6f7e5..b67ae08c971ae 100644 --- a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/search/SyntheticSourceIT.java +++ b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/search/SyntheticSourceIT.java @@ -40,6 +40,7 @@ public void testSearchUsingRuntimeField() throws Exception { indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); } indexRequest.get(); + indexRequest.request().decRef(); } client().admin().indices().prepareRefresh("test").get(); assertNoFailures(client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("long_id").from(0))); diff --git a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java index 2b9c35429c328..00d8207291fb4 100644 --- a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java +++ b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.legacygeo.search; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.geometry.Circle; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.legacygeo.test.TestLegacyGeoShapeFieldMapperPlugin; @@ -59,7 +60,7 @@ public void testLegacyCircle() throws Exception { ); ensureGreen(); - indexRandom(true, prepareIndex("test").setId("0").setSource("shape", (ToXContent) (builder, params) -> { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("0").setSource("shape", (ToXContent) (builder, params) -> { builder.startObject() .field("type", "circle") .startArray("coordinates") @@ -69,7 +70,9 @@ public void testLegacyCircle() throws Exception { .field("radius", "77km") .endObject(); return builder; - })); + }); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); // test self crossing of circles assertHitCount(prepareSearch("test").setQuery(geoShapeQuery("shape", new Circle(30, 50, 77000))), 1L); diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeQueryTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeQueryTests.java index 4e06a37ec7f20..966efc97da383 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeQueryTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeQueryTests.java @@ -8,6 +8,8 @@ package org.elasticsearch.legacygeo.search; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoJson; import org.elasticsearch.common.settings.Settings; @@ -86,17 +88,19 @@ public void testPointsOnlyExplicit() throws Exception { // MULTIPOINT MultiPoint multiPoint = GeometryTestUtils.randomMultiPoint(false); - prepareIndex("geo_points_only").setId("1") - .setSource(GeoJson.toXContent(multiPoint, jsonBuilder().startObject().field(defaultFieldName), null).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + indexImmediate( + "geo_points_only", + "1", + GeoJson.toXContent(multiPoint, jsonBuilder().startObject().field(defaultFieldName), null).endObject() + ); // POINT Point point = GeometryTestUtils.randomPoint(false); - prepareIndex("geo_points_only").setId("2") - .setSource(GeoJson.toXContent(point, jsonBuilder().startObject().field(defaultFieldName), null).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + indexImmediate( + "geo_points_only", + "2", + GeoJson.toXContent(point, jsonBuilder().startObject().field(defaultFieldName), null).endObject() + ); // test that point was inserted assertHitCount(client().prepareSearch("geo_points_only").setQuery(matchAllQuery()), 2L); @@ -123,10 +127,11 @@ public void testPointsOnly() throws Exception { Geometry geometry = GeometryTestUtils.randomGeometry(false); try { - prepareIndex("geo_points_only").setId("1") - .setSource(GeoJson.toXContent(geometry, jsonBuilder().startObject().field(defaultFieldName), null).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + indexImmediate( + "geo_points_only", + "1", + GeoJson.toXContent(geometry, jsonBuilder().startObject().field(defaultFieldName), null).endObject() + ); } catch (Exception e) { // Random geometry generator created something other than a POINT type, verify the correct exception is thrown assertThat(e.getMessage(), containsString("is configured for points only")); @@ -158,10 +163,11 @@ public void testFieldAlias() throws IOException { ensureGreen(); MultiPoint multiPoint = GeometryTestUtils.randomMultiPoint(false); - prepareIndex(defaultIndexName).setId("1") - .setSource(GeoJson.toXContent(multiPoint, jsonBuilder().startObject().field(defaultFieldName), null).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + indexImmediate( + defaultIndexName, + "1", + GeoJson.toXContent(multiPoint, jsonBuilder().startObject().field(defaultFieldName), null).endObject() + ); assertHitCount(client().prepareSearch(defaultIndexName).setQuery(geoShapeQuery("alias", multiPoint)), 1L); } @@ -175,4 +181,13 @@ public void testFieldAlias() throws IOException { protected boolean ignoreLons(double[] lons) { return Arrays.stream(lons).anyMatch(v -> v == 180); } + + private DocWriteResponse indexImmediate(String index, String id, XContentBuilder source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextMapperIT.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextMapperIT.java index 4a43b6afe3054..d8ce4146754b4 100644 --- a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextMapperIT.java +++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextMapperIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.index.query.QueryBuilders; @@ -48,9 +49,10 @@ public void testHighlightingWithMatchOnlyTextFieldMatchPhrase() throws IOExcepti assertAcked(prepareCreate("test").setMapping(mappings)); try (BulkRequestBuilder bulk = client().prepareBulk("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int i = 0; i < 2000; i++) { - bulk.add( - client().prepareIndex() - .setSource( + IndexRequestBuilder indexRequestBuilder = client().prepareIndex(); + try { + bulk.add( + indexRequestBuilder.setSource( XContentFactory.jsonBuilder() .startObject() .field( @@ -61,7 +63,10 @@ public void testHighlightingWithMatchOnlyTextFieldMatchPhrase() throws IOExcepti ) .endObject() ) - ); + ); + } finally { + indexRequestBuilder.request().decRef(); + } } BulkResponse bulkItemResponses = bulk.get(); assertNoFailures(bulkItemResponses); @@ -98,9 +103,10 @@ public void testHighlightingWithMatchOnlyTextFieldSyntheticSource() throws IOExc assertAcked(prepareCreate("test").setMapping(mappings)); try (BulkRequestBuilder bulk = client().prepareBulk("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int i = 0; i < 2000; i++) { - bulk.add( - client().prepareIndex() - .setSource( + IndexRequestBuilder indexRequestBuilder = client().prepareIndex(); + try { + bulk.add( + indexRequestBuilder.setSource( XContentFactory.jsonBuilder() .startObject() .field( @@ -111,7 +117,10 @@ public void testHighlightingWithMatchOnlyTextFieldSyntheticSource() throws IOExc ) .endObject() ) - ); + ); + } finally { + indexRequestBuilder.request().decRef(); + } } BulkResponse bulkItemResponses = bulk.get(); assertNoFailures(bulkItemResponses); diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java index c2b8554c072d8..02fc7acc09172 100644 --- a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java +++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.index.query.QueryBuilders; @@ -20,6 +21,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse; @@ -106,13 +108,17 @@ private void init() throws IOException { ensureGreen(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulk = bulkRequestBuilder.add( + List indexRequestBuilders = List.of( prepareIndex(INDEX_NAME).setId("all") - .setSource(Map.of("all_rank_features", Map.of(LOWER_RANKED_FEATURE, 10, HIGHER_RANKED_FEATURE, 20))) - ) - .add(prepareIndex(INDEX_NAME).setId("lower").setSource(Map.of("all_rank_features", Map.of(LOWER_RANKED_FEATURE, 10)))) - .add(prepareIndex(INDEX_NAME).setId("higher").setSource(Map.of("all_rank_features", Map.of(HIGHER_RANKED_FEATURE, 20)))) - .get(); + .setSource(Map.of("all_rank_features", Map.of(LOWER_RANKED_FEATURE, 10, HIGHER_RANKED_FEATURE, 20))), + prepareIndex(INDEX_NAME).setId("lower").setSource(Map.of("all_rank_features", Map.of(LOWER_RANKED_FEATURE, 10))), + prepareIndex(INDEX_NAME).setId("higher").setSource(Map.of("all_rank_features", Map.of(HIGHER_RANKED_FEATURE, 20))) + ); + for (IndexRequestBuilder builder : indexRequestBuilders) { + bulkRequestBuilder.add(builder); + builder.request().decRef(); + } + BulkResponse bulk = bulkRequestBuilder.get(); assertFalse(bulk.buildFailureMessage(), bulk.hasFailures()); assertThat(refresh().getFailedShards(), equalTo(0)); } diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java index f1914c43ba282..de28f80870333 100644 --- a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java +++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java @@ -157,21 +157,30 @@ private void init() throws IOException { .get(); ensureGreen(); - assertEquals(DocWriteResponse.Result.CREATED, prepareIndex("single", "I have four terms").get().getResult()); + assertEquals(DocWriteResponse.Result.CREATED, indexDoc("test", "single", "foo", new String[] { "I have four terms" }).getResult()); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulk = bulkRequestBuilder.add(prepareIndex("bulk1", "bulk three terms")) - .add(prepareIndex("bulk2", "this has five bulk terms")) - .get(); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("bulk1", "bulk three terms"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("bulk2", "this has five bulk terms"); + BulkResponse bulk = bulkRequestBuilder.add(indexRequestBuilder1).add(indexRequestBuilder2).get(); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); assertFalse(bulk.buildFailureMessage(), bulk.hasFailures()); assertEquals( DocWriteResponse.Result.CREATED, - prepareIndex("multi", "two terms", "wow now I have seven lucky terms").get().getResult() + // prepareIndex("multi", "two terms", "wow now I have seven lucky terms").get().getResult() + indexDoc("test", "multi", "foo", new String[] { "two terms", "wow now I have seven lucky terms" }).getResult() ); } try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulk = bulkRequestBuilder.add(prepareIndex("multibulk1", "one", "oh wow now I have eight unlucky terms")) - .add(prepareIndex("multibulk2", "six is a bunch of terms", "ten! ten terms is just crazy! too many too count!")) - .get(); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("multibulk1", "one", "oh wow now I have eight unlucky terms"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex( + "multibulk2", + "six is a bunch of terms", + "ten! ten terms is just crazy! too many too count!" + ); + BulkResponse bulk = bulkRequestBuilder.add(indexRequestBuilder1).add(indexRequestBuilder2).get(); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); assertFalse(bulk.buildFailureMessage(), bulk.hasFailures()); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/BWCTemplateTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/BWCTemplateTests.java index b4ee066a0e391..a210dcdd154e4 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/BWCTemplateTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/BWCTemplateTests.java @@ -8,6 +8,8 @@ package org.elasticsearch.index.mapper.extras; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xcontent.XContentType; @@ -34,8 +36,17 @@ public void testBeatsTemplatesBWC() throws Exception { indicesAdmin().preparePutTemplate("packetbeat").setSource(packetBeat, XContentType.JSON).get(); indicesAdmin().preparePutTemplate("filebeat").setSource(fileBeat, XContentType.JSON).get(); - prepareIndex("metricbeat-foo").setId("1").setSource("message", "foo").get(); - prepareIndex("packetbeat-foo").setId("1").setSource("message", "foo").get(); - prepareIndex("filebeat-foo").setId("1").setSource("message", "foo").get(); + indexDoc("metricbeat-foo", "1", "message", "foo"); + indexDoc("packetbeat-foo", "1", "message", "foo"); + indexDoc("filebeat-foo", "1", "message", "foo"); + } + + protected final DocWriteResponse indexDoc(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } } diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java index 03c64174af2c5..c8fad6fb763c6 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java @@ -107,6 +107,9 @@ public void setupCluster() throws Exception { requests.add(createIndexRequest("test", "comment", "f", "c")); indexRandom(true, requests); + for (IndexRequestBuilder indexRequestBuilder : requests) { + indexRequestBuilder.request().decRef(); + } ensureSearchable("test"); } diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java index 595d845d40b3d..c9acfac0b48bd 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Requests; import org.elasticsearch.search.SearchHit; @@ -171,7 +172,7 @@ public void testWithDeletes() throws Exception { requests.add(createIndexRequest(indexName, "child", "3", "1", "count", 1)); requests.add(createIndexRequest(indexName, "child", "4", "1", "count", 1)); requests.add(createIndexRequest(indexName, "child", "5", "1", "count", 1)); - indexRandom(true, requests); + indexRandomAndDecRefRequests(true, requests); for (int i = 0; i < 10; i++) { assertNoFailuresAndResponse( @@ -192,11 +193,12 @@ public void testWithDeletes() throws Exception { * the updates cause that. */ UpdateResponse updateResponse; - updateResponse = client().prepareUpdate(indexName, idToUpdate) - .setRouting("1") + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexName, idToUpdate); + updateResponse = updateRequestBuilder.setRouting("1") .setDoc(Requests.INDEX_CONTENT_TYPE, "count", 1) .setDetectNoop(false) .get(); + updateRequestBuilder.request().decRef(); assertThat(updateResponse.getVersion(), greaterThan(1L)); refresh(); } @@ -252,7 +254,7 @@ public void testPostCollection() throws Exception { requests.add(createIndexRequest(indexName, childType, "14", "2", "color", "black", "size", "40")); requests.add(createIndexRequest(indexName, childType, "15", "2", "color", "orange", "size", "36")); requests.add(createIndexRequest(indexName, childType, "16", "2", "color", "green", "size", "44")); - indexRandom(true, requests); + indexRandomAndDecRefRequests(true, requests); assertNoFailuresAndResponse( prepareSearch(indexName).setQuery(hasChildQuery(childType, termQuery("color", "orange"), ScoreMode.None)) @@ -300,9 +302,9 @@ public void testHierarchicalChildrenAggs() { ) ); - createIndexRequest(indexName, grandParentType, "1", null, "name", "europe").get(); - createIndexRequest(indexName, parentType, "2", "1", "name", "belgium").get(); - createIndexRequest(indexName, childType, "3", "2", "name", "brussels").setRouting("1").get(); + indexData(indexName, grandParentType, "1", null, null, "name", "europe"); + indexData(indexName, parentType, "2", "1", null, "name", "belgium"); + indexData(indexName, childType, "3", "2", "1", "name", "brussels"); refresh(); assertNoFailuresAndResponse( @@ -357,7 +359,7 @@ public void testPostCollectAllLeafReaders() throws Exception { requests.add(createIndexRequest("index", "childType", "6", "2", "name", "John", "age", 2)); requests.add(createIndexRequest("index", "childType", "7", "3", "name", "Betty", "age", 6)); requests.add(createIndexRequest("index", "childType", "8", "3", "name", "Dan", "age", 1)); - indexRandom(true, requests); + indexRandomAndDecRefRequests(true, requests); assertNoFailuresAndResponse( prepareSearch("index").setSize(0) @@ -397,4 +399,27 @@ public void testPostCollectAllLeafReaders() throws Exception { } ); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexData(String index, String type, String id, String parentId, String routing, Object... fields) { + IndexRequestBuilder indexRequestBuilder = createIndexRequest(index, type, id, parentId, fields); + try { + if (routing != null) { + indexRequestBuilder.setRouting(routing); + } + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + } diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java index 02776eb277020..cfa04714c674f 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java @@ -38,6 +38,7 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xcontent.XContentBuilder; import org.hamcrest.Matchers; import java.io.IOException; @@ -88,9 +89,9 @@ public void testMultiLevelChild() throws Exception { ); ensureGreen(); - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "c_value1").get(); - createIndexRequest("test", "grandchild", "gc1", "c1", "gc_field", "gc_value1").setRouting("p1").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "child", "c1", "p1", null, "c_field", "c_value1"); + indexData("test", "grandchild", "gc1", "c1", "p1", "gc_field", "gc_value1"); refresh(); assertNoFailuresAndResponse( @@ -154,8 +155,8 @@ public void test2744() throws IOException { ensureGreen(); // index simple data - createIndexRequest("test", "foo", "1", null, "foo", 1).get(); - createIndexRequest("test", "test", "2", "1", "foo", 1).get(); + indexData("test", "foo", "1", null, null, "foo", 1); + indexData("test", "test", "2", "1", null, "foo", 1); refresh(); assertNoFailuresAndResponse( prepareSearch("test").setQuery(hasChildQuery("test", matchQuery("foo", 1), ScoreMode.None)), @@ -171,12 +172,12 @@ public void testSimpleChildQuery() throws Exception { ensureGreen(); // index simple data - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get(); - createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); - createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get(); - createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "child", "c1", "p1", null, "c_field", "red"); + indexData("test", "child", "c2", "p1", null, "c_field", "yellow"); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); + indexData("test", "child", "c3", "p2", null, "c_field", "blue"); + indexData("test", "child", "c4", "p2", null, "c_field", "red"); refresh(); // TEST FETCHING _parent from child @@ -245,7 +246,7 @@ public void testCachingBugWithFqueryFilter() throws Exception { for (int i = 0; i < 10; i++) { builders.add(createIndexRequest("test", "parent", Integer.toString(i), null, "p_field", i)); } - indexRandom(randomBoolean(), builders); + indexRandomAndDecRefRequests(randomBoolean(), builders); builders.clear(); for (int j = 0; j < 2; j++) { for (int i = 0; i < 10; i++) { @@ -259,7 +260,7 @@ public void testCachingBugWithFqueryFilter() throws Exception { break; // randomly break out and dont' have deletes / updates } } - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); for (int i = 1; i <= 10; i++) { logger.info("Round {}", i); @@ -273,7 +274,7 @@ public void testHasParentFilter() throws Exception { ensureGreen(); Map> parentToChildren = new HashMap<>(); // Childless parent - createIndexRequest("test", "parent", "p0", null, "p_field", "p0").get(); + indexData("test", "parent", "p0", null, null, "p_field", "p0"); parentToChildren.put("p0", new HashSet<>()); String previousParentId = null; @@ -296,7 +297,7 @@ public void testHasParentFilter() throws Exception { } assertThat(parentToChildren.get(previousParentId).add(childId), is(true)); } - indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()])); + indexRandomAndDecRefRequests(true, builders); assertThat(parentToChildren.isEmpty(), equalTo(false)); for (Map.Entry> parentToChildrenEntry : parentToChildren.entrySet()) { @@ -322,17 +323,17 @@ public void testSimpleChildQueryWithFlush() throws Exception { ensureGreen(); // index simple data with flushes, so we have many segments - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); indicesAdmin().prepareFlush().get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get(); + indexData("test", "child", "c1", "p1", null, "c_field", "red"); indicesAdmin().prepareFlush().get(); - createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get(); + indexData("test", "child", "c2", "p1", null, "c_field", "yellow"); indicesAdmin().prepareFlush().get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); indicesAdmin().prepareFlush().get(); - createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get(); + indexData("test", "child", "c3", "p2", null, "c_field", "blue"); indicesAdmin().prepareFlush().get(); - createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get(); + indexData("test", "child", "c4", "p2", null, "c_field", "red"); indicesAdmin().prepareFlush().get(); refresh(); @@ -398,12 +399,12 @@ public void testScopedFacet() throws Exception { ensureGreen(); // index simple data - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get(); - createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); - createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get(); - createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "child", "c1", "p1", null, "c_field", "red"); + indexData("test", "child", "c2", "p1", null, "c_field", "yellow"); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); + indexData("test", "child", "c3", "p2", null, "c_field", "blue"); + indexData("test", "child", "c4", "p2", null, "c_field", "red"); refresh(); @@ -445,12 +446,12 @@ public void testDeletedParent() throws Exception { assertAcked(prepareCreate("test").setMapping(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"))); ensureGreen(); // index simple data - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get(); - createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); - createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get(); - createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "child", "c1", "p1", null, "c_field", "red"); + indexData("test", "child", "c2", "p1", null, "c_field", "yellow"); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); + indexData("test", "child", "c3", "p2", null, "c_field", "blue"); + indexData("test", "child", "c4", "p2", null, "c_field", "red"); refresh(); @@ -465,7 +466,7 @@ public void testDeletedParent() throws Exception { // update p1 and see what that we get updated values... - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1_updated").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1_updated"); indicesAdmin().prepareRefresh().get(); assertNoFailuresAndResponse( @@ -483,12 +484,12 @@ public void testDfsSearchType() throws Exception { ensureGreen(); // index simple data - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get(); - createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); - createIndexRequest("test", "child", "c3", "p3", "c_field", "blue").get(); - createIndexRequest("test", "child", "c4", "p2", "c_field", "red").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "child", "c1", "p1", null, "c_field", "red"); + indexData("test", "child", "c2", "p1", null, "c_field", "yellow"); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); + indexData("test", "child", "c3", "p3", null, "c_field", "blue"); + indexData("test", "child", "c4", "p2", null, "c_field", "red"); refresh(); @@ -507,10 +508,10 @@ public void testHasChildAndHasParentFailWhenSomeSegmentsDontContainAnyParentOrCh assertAcked(prepareCreate("test").setMapping(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"))); ensureGreen(); - createIndexRequest("test", "parent", "1", null, "p_field", 1).get(); - createIndexRequest("test", "child", "2", "1", "c_field", 1).get(); + indexData("test", "parent", "1", null, null, "p_field", 1); + indexData("test", "child", "2", "1", null, "c_field", 1); - prepareIndex("test").setId("3").setSource("p_field", 1).get(); + indexDoc("test", "3", "p_field", 1); refresh(); assertHitCountAndNoFailures( @@ -531,8 +532,8 @@ public void testCountApiUsage() throws Exception { ensureGreen(); String parentId = "p1"; - createIndexRequest("test", "parent", parentId, null, "p_field", "1").get(); - createIndexRequest("test", "child", "c1", parentId, "c_field", "1").get(); + indexData("test", "parent", parentId, null, null, "p_field", "1"); + indexData("test", "child", "c1", parentId, null, "c_field", "1"); refresh(); assertHitCount(prepareSearch("test").setSize(0).setQuery(hasChildQuery("child", termQuery("c_field", "1"), ScoreMode.Max)), 1L); @@ -556,8 +557,8 @@ public void testExplainUsage() throws Exception { ensureGreen(); String parentId = "p1"; - createIndexRequest("test", "parent", parentId, null, "p_field", "1").get(); - createIndexRequest("test", "child", "c1", parentId, "c_field", "1").get(); + indexData("test", "parent", parentId, null, null, "p_field", "1"); + indexData("test", "child", "c1", parentId, null, "c_field", "1"); refresh(); assertResponse( @@ -635,7 +636,7 @@ public void testScoreForParentChildQueriesWithFunctionScore() throws Exception { ); ensureGreen(); - indexRandom(true, createDocBuilders().toArray(new IndexRequestBuilder[0])); + indexRandomAndDecRefRequests(true, createDocBuilders()); assertResponse( prepareSearch("test").setQuery( hasChildQuery( @@ -735,9 +736,11 @@ public void testParentChildQueriesCanHandleNoRelevantTypesInIndex() throws Excep 0L ); - prepareIndex("test").setSource(jsonBuilder().startObject().field("text", "value").endObject()) - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource( + jsonBuilder().startObject().field("text", "value").endObject() + ).setRefreshPolicy(RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCountAndNoFailures( prepareSearch("test").setQuery(hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)), @@ -755,11 +758,11 @@ public void testHasChildAndHasParentFilter_withFilter() throws Exception { assertAcked(prepareCreate("test").setMapping(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"))); ensureGreen(); - createIndexRequest("test", "parent", "1", null, "p_field", 1).get(); - createIndexRequest("test", "child", "2", "1", "c_field", 1).get(); + indexData("test", "parent", "1", null, null, "p_field", 1); + indexData("test", "child", "2", "1", null, "c_field", 1); indicesAdmin().prepareFlush("test").get(); - prepareIndex("test").setId("3").setSource("p_field", 2).get(); + indexDoc("test", "3", "p_field", 2); refresh(); assertNoFailuresAndResponse( @@ -787,8 +790,8 @@ public void testHasChildInnerHitsHighlighting() throws Exception { assertAcked(prepareCreate("test").setMapping(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"))); ensureGreen(); - createIndexRequest("test", "parent", "1", null, "p_field", 1).get(); - createIndexRequest("test", "child", "2", "1", "c_field", "foo bar").get(); + indexData("test", "parent", "1", null, null, "p_field", 1); + indexData("test", "child", "2", "1", null, "c_field", "foo bar"); refresh(); assertNoFailuresAndResponse( @@ -817,9 +820,9 @@ public void testHasChildAndHasParentWrappedInAQueryFilter() throws Exception { ensureGreen(); // query filter in case for p/c shouldn't execute per segment, but rather - createIndexRequest("test", "parent", "1", null, "p_field", 1).get(); + indexData("test", "parent", "1", null, null, "p_field", 1); indicesAdmin().prepareFlush("test").setForce(true).get(); - createIndexRequest("test", "child", "2", "1", "c_field", 1).get(); + indexData("test", "child", "2", "1", null, "c_field", 1); refresh(); assertResponse( @@ -869,11 +872,11 @@ public void testSimpleQueryRewrite() throws Exception { int childId = 0; for (int i = 0; i < 10; i++) { String parentId = Strings.format("p%03d", i); - createIndexRequest("test", "parent", parentId, null, "p_field", parentId).get(); + indexData("test", "parent", parentId, null, null, "p_field", parentId); int j = childId; for (; j < childId + 50; j++) { String childUid = Strings.format("c%03d", j); - createIndexRequest("test", "child", childUid, parentId, "c_field", childUid).get(); + indexData("test", "child", childUid, parentId, null, "c_field", childUid); } childId = j; } @@ -919,12 +922,12 @@ public void testReIndexingParentAndChildDocuments() throws Exception { ensureGreen(); // index simple data - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "red").get(); - createIndexRequest("test", "child", "c2", "p1", "c_field", "yellow").get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); - createIndexRequest("test", "child", "c3", "p2", "c_field", "x").get(); - createIndexRequest("test", "child", "c4", "p2", "c_field", "x").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "child", "c1", "p1", null, "c_field", "red"); + indexData("test", "child", "c2", "p1", null, "c_field", "yellow"); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); + indexData("test", "child", "c3", "p2", null, "c_field", "x"); + indexData("test", "child", "c4", "p2", null, "c_field", "x"); refresh(); @@ -950,10 +953,10 @@ public void testReIndexingParentAndChildDocuments() throws Exception { // re-index for (int i = 0; i < 10; i++) { - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "child", "d" + i, "p1", "c_field", "red").get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); - createIndexRequest("test", "child", "c3", "p2", "c_field", "x").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "child", "d" + i, "p1", null, "c_field", "red"); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); + indexData("test", "child", "c3", "p2", null, "c_field", "x"); indicesAdmin().prepareRefresh("test").get(); } @@ -984,12 +987,12 @@ public void testHasChildQueryWithMinimumScore() throws Exception { ensureGreen(); // index simple data - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "x").get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); - createIndexRequest("test", "child", "c3", "p2", "c_field", "x").get(); - createIndexRequest("test", "child", "c4", "p2", "c_field", "x").get(); - createIndexRequest("test", "child", "c5", "p2", "c_field", "x").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "child", "c1", "p1", null, "c_field", "x"); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); + indexData("test", "child", "c3", "p2", null, "c_field", "x"); + indexData("test", "child", "c4", "p2", null, "c_field", "x"); + indexData("test", "child", "c5", "p2", null, "c_field", "x"); refresh(); // Score needs to be 3 or above! assertNoFailuresAndResponse( @@ -1016,7 +1019,7 @@ public void testParentFieldQuery() throws Exception { 0L ); - createIndexRequest("test", "child", "c1", "p1").get(); + indexData("test", "child", "c1", "p1"); refresh(); assertHitCount( @@ -1026,7 +1029,7 @@ public void testParentFieldQuery() throws Exception { 1L ); - createIndexRequest("test", "child", "c2", "p2").get(); + indexData("test", "child", "c2", "p2"); refresh(); assertHitCount( prepareSearch("test").setQuery( @@ -1044,12 +1047,12 @@ public void testParentIdQuery() throws Exception { ); ensureGreen(); - createIndexRequest("test", "child", "c1", "p1").get(); + indexData("test", "child", "c1", "p1"); refresh(); assertHitCount(prepareSearch("test").setQuery(parentId("child", "p1")), 1L); - createIndexRequest("test", "child", "c2", "p2").get(); + indexData("test", "child", "c2", "p2"); refresh(); assertHitCount(prepareSearch("test").setQuery(boolQuery().should(parentId("child", "p1")).should(parentId("child", "p2"))), 2L); @@ -1060,17 +1063,17 @@ public void testHasChildNotBeingCached() throws IOException { ensureGreen(); // index simple data - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); - createIndexRequest("test", "parent", "p3", null, "p_field", "p_value3").get(); - createIndexRequest("test", "parent", "p4", null, "p_field", "p_value4").get(); - createIndexRequest("test", "parent", "p5", null, "p_field", "p_value5").get(); - createIndexRequest("test", "parent", "p6", null, "p_field", "p_value6").get(); - createIndexRequest("test", "parent", "p7", null, "p_field", "p_value7").get(); - createIndexRequest("test", "parent", "p8", null, "p_field", "p_value8").get(); - createIndexRequest("test", "parent", "p9", null, "p_field", "p_value9").get(); - createIndexRequest("test", "parent", "p10", null, "p_field", "p_value10").get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "blue").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); + indexData("test", "parent", "p3", null, null, "p_field", "p_value3"); + indexData("test", "parent", "p4", null, null, "p_field", "p_value4"); + indexData("test", "parent", "p5", null, null, "p_field", "p_value5"); + indexData("test", "parent", "p6", null, null, "p_field", "p_value6"); + indexData("test", "parent", "p7", null, null, "p_field", "p_value7"); + indexData("test", "parent", "p8", null, null, "p_field", "p_value8"); + indexData("test", "parent", "p9", null, null, "p_field", "p_value9"); + indexData("test", "parent", "p10", null, null, "p_field", "p_value10"); + indexData("test", "child", "c1", "p1", null, "c_field", "blue"); indicesAdmin().prepareFlush("test").get(); indicesAdmin().prepareRefresh("test").get(); @@ -1079,7 +1082,7 @@ public void testHasChildNotBeingCached() throws IOException { 1L ); - createIndexRequest("test", "child", "c2", "p2", "c_field", "blue").get(); + indexData("test", "child", "c2", "p2", null, "c_field", "blue"); indicesAdmin().prepareRefresh("test").get(); assertHitCountAndNoFailures( @@ -1132,10 +1135,10 @@ public void testHasChildQueryOnlyReturnsSingleChildType() throws Exception { ) ); - createIndexRequest("grandissue", "grandparent", "1", null, "name", "Grandpa").get(); - createIndexRequest("grandissue", "parent", "2", "1", "name", "Dana").get(); - createIndexRequest("grandissue", "child_type_one", "3", "2", "name", "William").setRouting("1").get(); - createIndexRequest("grandissue", "child_type_two", "4", "2", "name", "Kate").setRouting("1").get(); + indexData("grandissue", "grandparent", "1", null, null, "name", "Grandpa"); + indexData("grandissue", "parent", "2", "1", null, "name", "Dana"); + indexData("grandissue", "child_type_one", "3", "2", "1", "name", "William"); + indexData("grandissue", "child_type_two", "4", "2", "1", "name", "Kate"); refresh(); assertHitCount( @@ -1177,7 +1180,7 @@ public void testHasChildQueryWithNestedInnerObjects() throws Exception { ); ensureGreen(); - createIndexRequest( + indexData( "test", "parent", "p1", @@ -1205,8 +1208,8 @@ public void testHasChildQueryWithNestedInnerObjects() throws Exception { .endObject() .endArray() .endObject() - ).get(); - createIndexRequest( + ); + indexData( "test", "parent", "p2", @@ -1222,10 +1225,10 @@ public void testHasChildQueryWithNestedInnerObjects() throws Exception { .endObject() .endArray() .endObject() - ).get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "blue").get(); - createIndexRequest("test", "child", "c2", "p1", "c_field", "red").get(); - createIndexRequest("test", "child", "c3", "p2", "c_field", "red").get(); + ); + indexData("test", "child", "c1", "p1", null, "c_field", "blue"); + indexData("test", "child", "c2", "p1", null, "c_field", "red"); + indexData("test", "child", "c3", "p2", null, "c_field", "red"); refresh(); ScoreMode scoreMode = randomFrom(ScoreMode.values()); @@ -1251,8 +1254,8 @@ public void testNamedFilters() throws Exception { ensureGreen(); String parentId = "p1"; - createIndexRequest("test", "parent", parentId, null, "p_field", "1").get(); - createIndexRequest("test", "child", "c1", parentId, "c_field", "1").get(); + indexData("test", "parent", parentId, null, null, "p_field", "1"); + indexData("test", "child", "c1", parentId, null, "c_field", "1"); refresh(); assertResponse( @@ -1301,7 +1304,7 @@ public void testParentChildQueriesNoParentType() throws Exception { ensureGreen(); String parentId = "p1"; - prepareIndex("test").setId(parentId).setSource("p_field", "1").get(); + indexDoc("test", parentId, "p_field", "1"); refresh(); try { @@ -1348,17 +1351,17 @@ public void testParentChildCaching() throws Exception { ensureGreen(); // index simple data - createIndexRequest("test", "parent", "p1", null, "p_field", "p_value1").get(); - createIndexRequest("test", "parent", "p2", null, "p_field", "p_value2").get(); - createIndexRequest("test", "child", "c1", "p1", "c_field", "blue").get(); - createIndexRequest("test", "child", "c2", "p1", "c_field", "red").get(); - createIndexRequest("test", "child", "c3", "p2", "c_field", "red").get(); + indexData("test", "parent", "p1", null, null, "p_field", "p_value1"); + indexData("test", "parent", "p2", null, null, "p_field", "p_value2"); + indexData("test", "child", "c1", "p1", null, "c_field", "blue"); + indexData("test", "child", "c2", "p1", null, "c_field", "red"); + indexData("test", "child", "c3", "p2", null, "c_field", "red"); indicesAdmin().prepareForceMerge("test").setMaxNumSegments(1).setFlush(true).get(); - createIndexRequest("test", "parent", "p3", null, "p_field", "p_value3").get(); - createIndexRequest("test", "parent", "p4", null, "p_field", "p_value4").get(); - createIndexRequest("test", "child", "c4", "p3", "c_field", "green").get(); - createIndexRequest("test", "child", "c5", "p3", "c_field", "blue").get(); - createIndexRequest("test", "child", "c6", "p4", "c_field", "blue").get(); + indexData("test", "parent", "p3", null, null, "p_field", "p_value3"); + indexData("test", "parent", "p4", null, null, "p_field", "p_value4"); + indexData("test", "child", "c4", "p3", null, "c_field", "green"); + indexData("test", "child", "c5", "p3", null, "c_field", "blue"); + indexData("test", "child", "c6", "p4", null, "c_field", "blue"); indicesAdmin().prepareFlush("test").get(); indicesAdmin().prepareRefresh("test").get(); @@ -1374,7 +1377,7 @@ public void testParentChildCaching() throws Exception { ); } - createIndexRequest("test", "child", "c3", "p2", "c_field", "blue").get(); + indexData("test", "child", "c3", "p2", null, "c_field", "blue"); indicesAdmin().prepareRefresh("test").get(); assertHitCount( @@ -1390,8 +1393,8 @@ public void testParentChildQueriesViaScrollApi() throws Exception { assertAcked(prepareCreate("test").setMapping(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"))); ensureGreen(); for (int i = 0; i < 10; i++) { - createIndexRequest("test", "parent", "p" + i, null).get(); - createIndexRequest("test", "child", "c" + i, "p" + i).get(); + indexData("test", "parent", "p" + i, null); + indexData("test", "child", "c" + i, "p" + i); } refresh(); @@ -1463,7 +1466,7 @@ public void testMinMaxChildren() throws Exception { assertAcked(prepareCreate("test").setMapping(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"))); ensureGreen(); - indexRandom(true, createMinMaxDocBuilders().toArray(new IndexRequestBuilder[0])); + indexRandomAndDecRefRequests(true, createMinMaxDocBuilders()); // Score mode = NONE assertResponse(minMaxQuery(ScoreMode.None, 1, null), response -> { @@ -1722,12 +1725,12 @@ public void testMinMaxChildren() throws Exception { assertThat(e.getMessage(), equalTo("[has_child] 'max_children' is less than 'min_children'")); } - public void testHasParentInnerQueryType() { + public void testHasParentInnerQueryType() throws IOException { assertAcked( prepareCreate("test").setMapping(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent-type", "child-type")) ); - createIndexRequest("test", "child-type", "child-id", "parent-id").get(); - createIndexRequest("test", "parent-type", "parent-id", null).get(); + indexData("test", "child-type", "child-id", "parent-id"); + indexData("test", "parent-type", "parent-id", null); refresh(); // make sure that when we explicitly set a type, the inner query is executed in the context of the child type instead @@ -1762,8 +1765,8 @@ public void testHighlightersIgnoreParentChild() throws IOException { .endObject() ) ); - createIndexRequest("test", "parent-type", "parent-id", null, "searchText", "quick brown fox").get(); - createIndexRequest("test", "child-type", "child-id", "parent-id", "searchText", "quick brown fox").get(); + indexData("test", "parent-type", "parent-id", null, null, "searchText", "quick brown fox"); + indexData("test", "child-type", "child-id", "parent-id", null, "searchText", "quick brown fox"); refresh(); String[] highlightTypes = new String[] { "plain", "fvh", "unified" }; @@ -1801,8 +1804,8 @@ public void testAliasesFilterWithHasChildQuery() throws Exception { assertAcked( prepareCreate("my-index").setMapping(buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")) ); - createIndexRequest("my-index", "parent", "1", null).get(); - createIndexRequest("my-index", "child", "2", "1").get(); + indexData("my-index", "parent", "1", null); + indexData("my-index", "child", "2", "1"); refresh(); assertAcked( @@ -1820,4 +1823,44 @@ public void testAliasesFilterWithHasChildQuery() throws Exception { }); } + private void indexData(String index, String type, String id, String parentId, String routing, Object... fields) { + IndexRequestBuilder indexRequestBuilder = createIndexRequest(index, type, id, parentId, fields); + try { + if (routing != null) { + indexRequestBuilder.setRouting(routing); + } + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private void indexData(String index, String type, String id, String parentId) { + IndexRequestBuilder indexRequestBuilder = createIndexRequest(index, type, id, parentId); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private void indexData(String index, String type, String id, String parentId, XContentBuilder builder) throws IOException { + IndexRequestBuilder indexRequestBuilder = createIndexRequest(index, type, id, parentId, builder); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + } diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java index f851678b6c9d6..cd93487cb827e 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java @@ -114,7 +114,7 @@ public void testSimpleParentChild() throws Exception { requests.add(createIndexRequest("articles", "comment", "c4", "p2", "message", "elephant captured")); requests.add(createIndexRequest("articles", "comment", "c5", "p2", "message", "mice squashed by elephant x")); requests.add(createIndexRequest("articles", "comment", "c6", "p2", "message", "elephant scared by mice x y")); - indexRandom(true, requests); + indexRandomAndDecRefRequests(true, requests); assertNoFailuresAndResponse( prepareSearch("articles").setQuery( @@ -248,7 +248,7 @@ public void testRandomParentChild() throws Exception { requestBuilders.add(createIndexRequest("idx", "child2", String.format(Locale.ENGLISH, "c2_%04d", child2), parentId)); } } - indexRandom(true, requestBuilders); + indexRandomAndDecRefRequests(true, requestBuilders); int size = randomIntBetween(0, numDocs); BoolQueryBuilder boolQuery = new BoolQueryBuilder(); @@ -333,7 +333,7 @@ public void testInnerHitsOnHasParent() throws Exception { requests.add( createIndexRequest("stack", "answer", "4", "2", "body", "Denyhosts protects only ssh; Fail2Ban protects all daemons.") ); - indexRandom(true, requests); + indexRandomAndDecRefRequests(true, requests); assertNoFailuresAndResponse( prepareSearch("stack").addSort("id", SortOrder.ASC) @@ -377,7 +377,7 @@ public void testParentChildMultipleLayers() throws Exception { requests.add(createIndexRequest("articles", "article", "2", null, "title", "big gray elephant")); requests.add(createIndexRequest("articles", "comment", "4", "2", "message", "elephant captured")); requests.add(createIndexRequest("articles", "remark", "6", "4", "message", "bad").setRouting("2")); - indexRandom(true, requests); + indexRandomAndDecRefRequests(true, requests); assertNoFailuresAndResponse( prepareSearch("articles").setQuery( @@ -456,7 +456,7 @@ public void testRoyals() throws Exception { requests.add(createIndexRequest("royals", "baron", "baron2", "earl2").setRouting("king")); requests.add(createIndexRequest("royals", "baron", "baron3", "earl3").setRouting("king")); requests.add(createIndexRequest("royals", "baron", "baron4", "earl4").setRouting("king")); - indexRandom(true, requests); + indexRandomAndDecRefRequests(true, requests); assertResponse( prepareSearch("royals").setQuery( boolQuery().filter( @@ -522,7 +522,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception { requests.add(createIndexRequest("index", "child", "4", "1", "field", "value2")); requests.add(createIndexRequest("index", "parent", "2", null)); requests.add(createIndexRequest("index", "child", "5", "2", "field", "value1")); - indexRandom(true, requests); + indexRandomAndDecRefRequests(true, requests); assertResponse( prepareSearch("index").setQuery( @@ -565,7 +565,7 @@ public void testUseMaxDocInsteadOfSize() throws Exception { List requests = new ArrayList<>(); requests.add(createIndexRequest("index1", "parent", "1", null)); requests.add(createIndexRequest("index1", "child", "2", "1", "field", "value1")); - indexRandom(true, requests); + indexRandomAndDecRefRequests(true, requests); QueryBuilder query = hasChildQuery("child", matchQuery("field", "value1"), ScoreMode.None).innerHit( new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1) @@ -583,8 +583,19 @@ public void testNestedInnerHitWrappedInParentChildInnerhit() { ) ) ); - createIndexRequest("test", "parent_type", "1", null, "key", "value").get(); - createIndexRequest("test", "child_type", "2", "1", "nested_type", Collections.singletonMap("key", "value")).get(); + IndexRequestBuilder indexRequestBuilder1 = createIndexRequest("test", "parent_type", "1", null, "key", "value"); + indexRequestBuilder1.get(); + indexRequestBuilder1.request().decRef(); + IndexRequestBuilder indexRequestBuilder2 = createIndexRequest( + "test", + "child_type", + "2", + "1", + "nested_type", + Collections.singletonMap("key", "value") + ); + indexRequestBuilder2.get(); + indexRequestBuilder2.request().decRef(); refresh(); assertResponse( prepareSearch("test").setQuery( @@ -624,9 +635,20 @@ public void testInnerHitsWithIgnoreUnmapped() { ) ); assertAcked(prepareCreate("index2")); - createIndexRequest("index1", "parent_type", "1", null, "nested_type", Collections.singletonMap("key", "value")).get(); - createIndexRequest("index1", "child_type", "2", "1").get(); - prepareIndex("index2").setId("3").setSource("key", "value").get(); + IndexRequestBuilder indexRequestBuilder1 = createIndexRequest( + "index1", + "parent_type", + "1", + null, + "nested_type", + Collections.singletonMap("key", "value") + ); + indexRequestBuilder1.get(); + indexRequestBuilder1.request().decRef(); + IndexRequestBuilder indexRequestBuilder2 = createIndexRequest("index1", "child_type", "2", "1"); + indexRequestBuilder2.get(); + indexRequestBuilder2.request().decRef(); + indexDoc("index2", "3", "key", "value"); refresh(); assertSearchHitsWithoutFailures( prepareSearch("index1", "index2").setQuery( @@ -650,8 +672,19 @@ public void testTooHighResultWindow() { ) ) ); - createIndexRequest("index1", "parent_type", "1", null, "nested_type", Collections.singletonMap("key", "value")).get(); - createIndexRequest("index1", "child_type", "2", "1").get(); + IndexRequestBuilder indexRequestBuilder1 = createIndexRequest( + "index1", + "parent_type", + "1", + null, + "nested_type", + Collections.singletonMap("key", "value") + ); + indexRequestBuilder1.get(); + indexRequestBuilder1.request().decRef(); + IndexRequestBuilder indexRequestBuilder2 = createIndexRequest("index1", "child_type", "2", "1"); + indexRequestBuilder2.get(); + indexRequestBuilder2.request().decRef(); refresh(); assertHitCountAndNoFailures( prepareSearch("index1").setQuery( @@ -697,4 +730,14 @@ public void testTooHighResultWindow() { ) ); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java index 02f24a67dda02..6559f7823c637 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java @@ -56,7 +56,12 @@ protected IndexRequestBuilder createIndexRequest(String index, String type, Stri protected IndexRequestBuilder createIndexRequest(String index, String type, String id, String parentId, XContentBuilder builder) throws IOException { - Map source = XContentHelper.convertToMap(JsonXContent.jsonXContent, Strings.toString(builder), false); + Map source; + if (builder == null) { + source = null; + } else { + source = XContentHelper.convertToMap(JsonXContent.jsonXContent, Strings.toString(builder), false); + } return createIndexRequest(index, type, id, parentId, source); } @@ -101,17 +106,22 @@ private IndexRequestBuilder createIndexRequest(String index, String type, String type = "doc"; IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id); - Map joinField = new HashMap<>(); - if (parentId != null) { - joinField.put("name", name); - joinField.put("parent", parentId); - indexRequestBuilder.setRouting(parentId); - } else { - joinField.put("name", name); + try { + Map joinField = new HashMap<>(); + if (parentId != null) { + joinField.put("name", name); + joinField.put("parent", parentId); + indexRequestBuilder.setRouting(parentId); + } else { + joinField.put("name", name); + } + source.put("join_field", joinField); + indexRequestBuilder.setSource(source); + return indexRequestBuilder; + } catch (Exception e) { + indexRequestBuilder.request().decRef(); + throw e; } - source.put("join_field", joinField); - indexRequestBuilder.setSource(source); - return indexRequestBuilder; } } diff --git a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 88a39fe4aebc8..bbe4dfd8a75de 100644 --- a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.MultiSearchResponse.Item; import org.elasticsearch.action.support.WriteRequest; @@ -80,20 +81,16 @@ public void testPercolatorQuery() throws Exception { .setMapping("id", "type=keyword", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") ); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("id", "1").field("query", matchAllQuery()).endObject()) - .get(); - prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("id", "2").field("query", matchQuery("field1", "value")).endObject()) - .get(); - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .field("id", "3") - .field("query", boolQuery().must(matchQuery("field1", "value")).must(matchQuery("field2", "value"))) - .endObject() - ) - .get(); + index("test", "1", jsonBuilder().startObject().field("id", "1").field("query", matchAllQuery()).endObject()); + index("test", "2", jsonBuilder().startObject().field("id", "2").field("query", matchQuery("field1", "value")).endObject()); + index( + "test", + "3", + jsonBuilder().startObject() + .field("id", "3") + .field("query", boolQuery().must(matchQuery("field1", "value")).must(matchQuery("field2", "value"))) + .endObject() + ); indicesAdmin().prepareRefresh().get(); BytesReference source = BytesReference.bytes(jsonBuilder().startObject().endObject()); @@ -177,62 +174,58 @@ public void testPercolatorRangeQueries() throws Exception { ) ); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", rangeQuery("field1").from(10).to(12)).endObject()) - .get(); - prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", rangeQuery("field1").from(20).to(22)).endObject()) - .get(); - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .field("query", boolQuery().must(rangeQuery("field1").from(10).to(12)).must(rangeQuery("field1").from(12).to(14))) - .endObject() - ) - .get(); + index("test", "1", jsonBuilder().startObject().field("query", rangeQuery("field1").from(10).to(12)).endObject()); + index("test", "2", jsonBuilder().startObject().field("query", rangeQuery("field1").from(20).to(22)).endObject()); + index( + "test", + "3", + jsonBuilder().startObject() + .field("query", boolQuery().must(rangeQuery("field1").from(10).to(12)).must(rangeQuery("field1").from(12).to(14))) + .endObject() + ); indicesAdmin().prepareRefresh().get(); - prepareIndex("test").setId("4") - .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from(10).to(12)).endObject()) - .get(); - prepareIndex("test").setId("5") - .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from(20).to(22)).endObject()) - .get(); - prepareIndex("test").setId("6") - .setSource( - jsonBuilder().startObject() - .field("query", boolQuery().must(rangeQuery("field2").from(10).to(12)).must(rangeQuery("field2").from(12).to(14))) - .endObject() - ) - .get(); + index("test", "4", jsonBuilder().startObject().field("query", rangeQuery("field2").from(10).to(12)).endObject()); + index("test", "5", jsonBuilder().startObject().field("query", rangeQuery("field2").from(20).to(22)).endObject()); + index( + "test", + "6", + jsonBuilder().startObject() + .field("query", boolQuery().must(rangeQuery("field2").from(10).to(12)).must(rangeQuery("field2").from(12).to(14))) + .endObject() + ); indicesAdmin().prepareRefresh().get(); - prepareIndex("test").setId("7") - .setSource(jsonBuilder().startObject().field("query", rangeQuery("field3").from("192.168.1.0").to("192.168.1.5")).endObject()) - .get(); - prepareIndex("test").setId("8") - .setSource(jsonBuilder().startObject().field("query", rangeQuery("field3").from("192.168.1.20").to("192.168.1.30")).endObject()) - .get(); - prepareIndex("test").setId("9") - .setSource( - jsonBuilder().startObject() - .field( - "query", - boolQuery().must(rangeQuery("field3").from("192.168.1.0").to("192.168.1.5")) - .must(rangeQuery("field3").from("192.168.1.5").to("192.168.1.10")) - ) - .endObject() - ) - .get(); - prepareIndex("test").setId("10") - .setSource( - jsonBuilder().startObject() - .field( - "query", - boolQuery().must(rangeQuery("field4").from("2010-01-01").to("2018-01-01")) - .must(rangeQuery("field4").from("2010-01-01").to("now")) - ) - .endObject() - ) - .get(); + index( + "test", + "7", + jsonBuilder().startObject().field("query", rangeQuery("field3").from("192.168.1.0").to("192.168.1.5")).endObject() + ); + index( + "test", + "8", + jsonBuilder().startObject().field("query", rangeQuery("field3").from("192.168.1.20").to("192.168.1.30")).endObject() + ); + index( + "test", + "9", + jsonBuilder().startObject() + .field( + "query", + boolQuery().must(rangeQuery("field3").from("192.168.1.0").to("192.168.1.5")) + .must(rangeQuery("field3").from("192.168.1.5").to("192.168.1.10")) + ) + .endObject() + ); + index( + "test", + "10", + jsonBuilder().startObject() + .field( + "query", + boolQuery().must(rangeQuery("field4").from("2010-01-01").to("2018-01-01")) + .must(rangeQuery("field4").from("2010-01-01").to("now")) + ) + .endObject() + ); indicesAdmin().prepareRefresh().get(); // Test long range: @@ -290,38 +283,38 @@ public void testPercolatorGeoQueries() throws Exception { indicesAdmin().prepareCreate("test").setMapping("id", "type=keyword", "field1", "type=geo_point", "query", "type=percolator") ); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("query", geoDistanceQuery("field1").point(52.18, 4.38).distance(50, DistanceUnit.KILOMETERS)) - .field("id", "1") - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("query", geoDistanceQuery("field1").point(52.18, 4.38).distance(50, DistanceUnit.KILOMETERS)) + .field("id", "1") + .endObject() + ); - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .field("query", geoBoundingBoxQuery("field1").setCorners(52.3, 4.4, 52.1, 4.6)) - .field("id", "2") - .endObject() - ) - .get(); + index( + "test", + "2", + jsonBuilder().startObject() + .field("query", geoBoundingBoxQuery("field1").setCorners(52.3, 4.4, 52.1, 4.6)) + .field("id", "2") + .endObject() + ); - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .field( - "query", - geoShapeQuery( - "field1", - new Polygon(new LinearRing(new double[] { 4.4, 4.5, 4.6, 4.4 }, new double[] { 52.1, 52.3, 52.1, 52.1 })) - ) + index( + "test", + "3", + jsonBuilder().startObject() + .field( + "query", + geoShapeQuery( + "field1", + new Polygon(new LinearRing(new double[] { 4.4, 4.5, 4.6, 4.4 }, new double[] { 52.1, 52.3, 52.1, 52.1 })) ) - .field("id", "3") - .endObject() - ) - .get(); + ) + .field("id", "3") + .endObject() + ); refresh(); BytesReference source = BytesReference.bytes( @@ -344,24 +337,20 @@ public void testPercolatorQueryExistingDocument() throws Exception { .setMapping("id", "type=keyword", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") ); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("id", "1").field("query", matchAllQuery()).endObject()) - .get(); - prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("id", "2").field("query", matchQuery("field1", "value")).endObject()) - .get(); - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .field("id", "3") - .field("query", boolQuery().must(matchQuery("field1", "value")).must(matchQuery("field2", "value"))) - .endObject() - ) - .get(); + index("test", "1", jsonBuilder().startObject().field("id", "1").field("query", matchAllQuery()).endObject()); + index("test", "2", jsonBuilder().startObject().field("id", "2").field("query", matchQuery("field1", "value")).endObject()); + index( + "test", + "3", + jsonBuilder().startObject() + .field("id", "3") + .field("query", boolQuery().must(matchQuery("field1", "value")).must(matchQuery("field2", "value"))) + .endObject() + ); - prepareIndex("test").setId("4").setSource("{\"id\": \"4\"}", XContentType.JSON).get(); - prepareIndex("test").setId("5").setSource(XContentType.JSON, "id", "5", "field1", "value").get(); - prepareIndex("test").setId("6").setSource(XContentType.JSON, "id", "6", "field1", "value", "field2", "value").get(); + index("test", "4", "{\"id\": \"4\"}", XContentType.JSON); + indexDoc("test", "5", XContentType.JSON, "id", "5", "field1", "value"); + indexDoc("test", "6", XContentType.JSON, "id", "6", "field1", "value", "field2", "value"); indicesAdmin().prepareRefresh().get(); logger.info("percolating empty doc"); @@ -398,9 +387,9 @@ public void testPercolatorQueryExistingDocumentSourceDisabled() throws Exception .setMapping("_source", "enabled=false", "field1", "type=keyword", "query", "type=percolator") ); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("query", matchAllQuery()).endObject()); - prepareIndex("test").setId("2").setSource("{}", XContentType.JSON).get(); + index("test", "2", "{}", XContentType.JSON); indicesAdmin().prepareRefresh().get(); logger.info("percolating empty doc with source disabled"); @@ -416,67 +405,67 @@ public void testPercolatorSpecificQueries() throws Exception { .setMapping("id", "type=keyword", "field1", "type=text", "field2", "type=text", "query", "type=percolator") ); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("id", "1") - .field("query", multiMatchQuery("quick brown fox", "field1", "field2").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)) - .endObject() - ) - .get(); - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .field("id", "2") - .field( - "query", + index( + "test", + "1", + jsonBuilder().startObject() + .field("id", "1") + .field("query", multiMatchQuery("quick brown fox", "field1", "field2").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)) + .endObject() + ); + index( + "test", + "2", + jsonBuilder().startObject() + .field("id", "2") + .field( + "query", + spanNearQuery(spanTermQuery("field1", "quick"), 0).addClause(spanTermQuery("field1", "brown")) + .addClause(spanTermQuery("field1", "fox")) + .inOrder(true) + ) + .endObject() + ); + indicesAdmin().prepareRefresh().get(); + + index( + "test", + "3", + jsonBuilder().startObject() + .field("id", "3") + .field( + "query", + spanNotQuery( spanNearQuery(spanTermQuery("field1", "quick"), 0).addClause(spanTermQuery("field1", "brown")) .addClause(spanTermQuery("field1", "fox")) + .inOrder(true), + spanNearQuery(spanTermQuery("field1", "the"), 0).addClause(spanTermQuery("field1", "lazy")) + .addClause(spanTermQuery("field1", "dog")) .inOrder(true) - ) - .endObject() - ) - .get(); - indicesAdmin().prepareRefresh().get(); - - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .field("id", "3") - .field( - "query", - spanNotQuery( - spanNearQuery(spanTermQuery("field1", "quick"), 0).addClause(spanTermQuery("field1", "brown")) - .addClause(spanTermQuery("field1", "fox")) - .inOrder(true), - spanNearQuery(spanTermQuery("field1", "the"), 0).addClause(spanTermQuery("field1", "lazy")) - .addClause(spanTermQuery("field1", "dog")) - .inOrder(true) - ).dist(2) - ) - .endObject() - ) - .get(); + ).dist(2) + ) + .endObject() + ); // doesn't match - prepareIndex("test").setId("4") - .setSource( - jsonBuilder().startObject() - .field("id", "4") - .field( - "query", - spanNotQuery( - spanNearQuery(spanTermQuery("field1", "quick"), 0).addClause(spanTermQuery("field1", "brown")) - .addClause(spanTermQuery("field1", "fox")) - .inOrder(true), - spanNearQuery(spanTermQuery("field1", "the"), 0).addClause(spanTermQuery("field1", "lazy")) - .addClause(spanTermQuery("field1", "dog")) - .inOrder(true) - ).dist(3) - ) - .endObject() - ) - .get(); + index( + "test", + "4", + jsonBuilder().startObject() + .field("id", "4") + .field( + "query", + spanNotQuery( + spanNearQuery(spanTermQuery("field1", "quick"), 0).addClause(spanTermQuery("field1", "brown")) + .addClause(spanTermQuery("field1", "fox")) + .inOrder(true), + spanNearQuery(spanTermQuery("field1", "the"), 0).addClause(spanTermQuery("field1", "lazy")) + .addClause(spanTermQuery("field1", "dog")) + .inOrder(true) + ).dist(3) + ) + .endObject() + ); indicesAdmin().prepareRefresh().get(); BytesReference source = BytesReference.bytes( @@ -510,21 +499,11 @@ public void testPercolatorQueryWithHighlighting() throws Exception { indicesAdmin().prepareCreate("test") .setMapping("id", "type=keyword", "field1", fieldMapping.toString(), "query", "type=percolator") ); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("id", "1").field("query", matchQuery("field1", "brown fox")).endObject()) - .get(); - prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("id", "2").field("query", matchQuery("field1", "lazy dog")).endObject()) - .get(); - prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("id", "3").field("query", termQuery("field1", "jumps")).endObject()) - .get(); - prepareIndex("test").setId("4") - .setSource(jsonBuilder().startObject().field("id", "4").field("query", termQuery("field1", "dog")).endObject()) - .get(); - prepareIndex("test").setId("5") - .setSource(jsonBuilder().startObject().field("id", "5").field("query", termQuery("field1", "fox")).endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().field("id", "1").field("query", matchQuery("field1", "brown fox")).endObject()); + index("test", "2", jsonBuilder().startObject().field("id", "2").field("query", matchQuery("field1", "lazy dog")).endObject()); + index("test", "3", jsonBuilder().startObject().field("id", "3").field("query", termQuery("field1", "jumps")).endObject()); + index("test", "4", jsonBuilder().startObject().field("id", "4").field("query", termQuery("field1", "dog")).endObject()); + index("test", "5", jsonBuilder().startObject().field("id", "5").field("query", termQuery("field1", "fox")).endObject()); indicesAdmin().prepareRefresh().get(); BytesReference document = BytesReference.bytes( @@ -756,12 +735,16 @@ public void testTakePositionOffsetGapIntoAccount() throws Exception { assertAcked( indicesAdmin().prepareCreate("test").setMapping("field", "type=text,position_increment_gap=5", "query", "type=percolator") ); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", new MatchPhraseQueryBuilder("field", "brown fox").slop(4)).endObject()) - .get(); - prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", new MatchPhraseQueryBuilder("field", "brown fox").slop(5)).endObject()) - .get(); + index( + "test", + "1", + jsonBuilder().startObject().field("query", new MatchPhraseQueryBuilder("field", "brown fox").slop(4)).endObject() + ); + index( + "test", + "2", + jsonBuilder().startObject().field("query", new MatchPhraseQueryBuilder("field", "brown fox").slop(5)).endObject() + ); indicesAdmin().prepareRefresh().get(); assertResponse( @@ -833,18 +816,16 @@ public void testWithMultiplePercolatorFields() throws Exception { ); // Acceptable: - prepareIndex("test1").setId("1") - .setSource(jsonBuilder().startObject().field(queryFieldName, matchQuery("field", "value")).endObject()) - .get(); - prepareIndex("test2").setId("1") - .setSource( - jsonBuilder().startObject() - .startObject("object_field") - .field(queryFieldName, matchQuery("field", "value")) - .endObject() - .endObject() - ) - .get(); + index("test1", "1", jsonBuilder().startObject().field(queryFieldName, matchQuery("field", "value")).endObject()); + index( + "test2", + "1", + jsonBuilder().startObject() + .startObject("object_field") + .field(queryFieldName, matchQuery("field", "value")) + .endObject() + .endObject() + ); indicesAdmin().prepareRefresh().get(); BytesReference source = BytesReference.bytes(jsonBuilder().startObject().field("field", "value").endObject()); @@ -869,20 +850,20 @@ public void testWithMultiplePercolatorFields() throws Exception { // Unacceptable: DocumentParsingException e = expectThrows(DocumentParsingException.class, () -> { - prepareIndex("test2").setId("1") - .setSource( - jsonBuilder().startObject() - .startArray("object_field") - .startObject() - .field(queryFieldName, matchQuery("field", "value")) - .endObject() - .startObject() - .field(queryFieldName, matchQuery("field", "value")) - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test2", + "1", + jsonBuilder().startObject() + .startArray("object_field") + .startObject() + .field(queryFieldName, matchQuery("field", "value")) + .endObject() + .startObject() + .field(queryFieldName, matchQuery("field", "value")) + .endObject() + .endArray() + .endObject() + ); }); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); assertThat(e.getCause().getMessage(), equalTo("a document can only contain one percolator query")); @@ -912,35 +893,30 @@ public void testPercolateQueryWithNestedDocuments() throws Exception { .endObject() .endObject(); assertAcked(indicesAdmin().prepareCreate("test").setMapping(mapping)); - prepareIndex("test").setId("q1") - .setSource( - jsonBuilder().startObject() - .field("id", "q1") - .field( - "query", - QueryBuilders.nestedQuery( - "employee", - QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), - ScoreMode.Avg - ) + index( + "test", + "q1", + jsonBuilder().startObject() + .field("id", "q1") + .field( + "query", + QueryBuilders.nestedQuery( + "employee", + QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), + ScoreMode.Avg ) - .endObject() - ) - .get(); + ) + .endObject() + ); // this query should never match as it doesn't use nested query: - prepareIndex("test").setId("q2") - .setSource( - jsonBuilder().startObject() - .field("id", "q2") - .field("query", QueryBuilders.matchQuery("employee.name", "virginia")) - .endObject() - ) - .get(); + index( + "test", + "q2", + jsonBuilder().startObject().field("id", "q2").field("query", QueryBuilders.matchQuery("employee.name", "virginia")).endObject() + ); indicesAdmin().prepareRefresh().get(); - prepareIndex("test").setId("q3") - .setSource(jsonBuilder().startObject().field("id", "q3").field("query", QueryBuilders.matchAllQuery()).endObject()) - .get(); + index("test", "q3", jsonBuilder().startObject().field("id", "q3").field("query", QueryBuilders.matchAllQuery()).endObject()); indicesAdmin().prepareRefresh().get(); assertResponse( @@ -1079,19 +1055,17 @@ public void testPercolateQueryWithNestedDocuments() throws Exception { public void testPercolatorQueryViaMultiSearch() throws Exception { assertAcked(indicesAdmin().prepareCreate("test").setMapping("field1", "type=text", "query", "type=percolator")); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject()) - .get(); - prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject()).get(); - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .field("query", boolQuery().must(matchQuery("field1", "b")).must(matchQuery("field1", "c"))) - .endObject() - ) - .get(); - prepareIndex("test").setId("4").setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()).get(); - prepareIndex("test").setId("5").setSource(jsonBuilder().startObject().field("field1", "c").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject()); + index("test", "2", jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject()); + index( + "test", + "3", + jsonBuilder().startObject() + .field("query", boolQuery().must(matchQuery("field1", "b")).must(matchQuery("field1", "c"))) + .endObject() + ); + index("test", "4", jsonBuilder().startObject().field("query", matchAllQuery()).endObject()); + index("test", "5", jsonBuilder().startObject().field("field1", "c").endObject()); indicesAdmin().prepareRefresh().get(); assertResponse( @@ -1177,9 +1151,7 @@ public void testDisallowExpensiveQueries() throws IOException { indicesAdmin().prepareCreate("test").setMapping("id", "type=keyword", "field1", "type=keyword", "query", "type=percolator") ); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("id", "1").field("query", matchQuery("field1", "value")).endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().field("id", "1").field("query", matchQuery("field1", "value")).endObject()); refresh(); // Execute with search.allow_expensive_queries = null => default value = false => success @@ -1219,13 +1191,9 @@ public void testWrappedWithConstantScore() throws Exception { assertAcked(indicesAdmin().prepareCreate("test").setMapping("d", "type=date", "q", "type=percolator")); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("q", boolQuery().must(rangeQuery("d").gt("now"))).endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().field("q", boolQuery().must(rangeQuery("d").gt("now"))).endObject()); - prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("q", boolQuery().must(rangeQuery("d").lt("now"))).endObject()) - .get(); + index("test", "2", jsonBuilder().startObject().field("q", boolQuery().must(rangeQuery("d").lt("now"))).endObject()); indicesAdmin().prepareRefresh().get(); @@ -1282,7 +1250,7 @@ public void testWithWildcardFieldNames() throws Exception { ) ); - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource( jsonBuilder().startObject() .field("q_simple", simpleQueryStringQuery("yada").fields(Map.of("text*", 1f))) @@ -1291,8 +1259,9 @@ public void testWithWildcardFieldNames() throws Exception { .field("q_combo", combinedFieldsQuery("yada", "text*")) .endObject() ) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCount( prepareSearch("test").setQuery( @@ -1363,8 +1332,30 @@ public void testKnnQueryNotSupportedInPercolator() throws IOException { IndexRequestBuilder indexRequestBuilder = prepareIndex("index1").setId("knn_query1") .setSource(jsonBuilder().startObject().field("my_query", knnVectorQueryBuilder).endObject()); - DocumentParsingException exception = expectThrows(DocumentParsingException.class, () -> indexRequestBuilder.get()); - assertThat(exception.getMessage(), containsString("the [knn] query is unsupported inside a percolator")); + try { + DocumentParsingException exception = expectThrows(DocumentParsingException.class, () -> indexRequestBuilder.get()); + assertThat(exception.getMessage(), containsString("the [knn] query is unsupported inside a percolator")); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private DocWriteResponse index(String index, String id, String source, XContentType xContentType) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source, xContentType).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private DocWriteResponse indexDoc(String index, String id, XContentType xContentType, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(xContentType, source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 46b9e365fd0ea..1b1ac967b93be 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -31,6 +31,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -620,7 +621,9 @@ public void testStoringQueries() throws Exception { public void testQueryWithRewrite() throws Exception { addQueryFieldMappings(); - prepareIndex("remote").setId("1").setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("remote").setId("1").setSource("field", "value"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); QueryBuilder queryBuilder = termsLookupQuery("field", new TermsLookup("remote", "1", "field")); ParsedDocument doc = mapperService.documentMapper() .parse( diff --git a/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 244c3f2188237..a5848747b6c48 100644 --- a/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -49,15 +49,15 @@ public void setup() { createIndex(TEST_INDEX); ensureGreen(); - prepareIndex(TEST_INDEX).setId("1").setSource("id", 1, "text", "berlin", "title", "Berlin, Germany", "population", 3670622).get(); - prepareIndex(TEST_INDEX).setId("2").setSource("id", 2, "text", "amsterdam", "population", 851573).get(); - prepareIndex(TEST_INDEX).setId("3").setSource("id", 3, "text", "amsterdam", "population", 851573).get(); - prepareIndex(TEST_INDEX).setId("4").setSource("id", 4, "text", "amsterdam", "population", 851573).get(); - prepareIndex(TEST_INDEX).setId("5").setSource("id", 5, "text", "amsterdam", "population", 851573).get(); - prepareIndex(TEST_INDEX).setId("6").setSource("id", 6, "text", "amsterdam", "population", 851573).get(); + indexDoc(TEST_INDEX, "1", "id", 1, "text", "berlin", "title", "Berlin, Germany", "population", 3670622); + indexDoc(TEST_INDEX, "2", "id", 2, "text", "amsterdam", "population", 851573); + indexDoc(TEST_INDEX, "3", "id", 3, "text", "amsterdam", "population", 851573); + indexDoc(TEST_INDEX, "4", "id", 4, "text", "amsterdam", "population", 851573); + indexDoc(TEST_INDEX, "5", "id", 5, "text", "amsterdam", "population", 851573); + indexDoc(TEST_INDEX, "6", "id", 6, "text", "amsterdam", "population", 851573); // add another index for testing closed indices etc... - prepareIndex("test2").setId("7").setSource("id", 7, "text", "amsterdam", "population", 851573).get(); + indexDoc("test2", "7", "id", 7, "text", "amsterdam", "population", 851573); refresh(); // set up an alias that can also be used in tests diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java index 46271f8c61e9c..43e3c7e45f70c 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.engine.Engine; @@ -41,6 +42,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -69,12 +71,17 @@ public void setup() { public void testReindex() { Client client = client(); // tag::reindex1 - BulkByScrollResponse response = - new ReindexRequestBuilder(client) - .source("source_index") - .destination("target_index") - .filter(QueryBuilders.matchQuery("category", "xzy")) // <1> - .get(); + ReindexRequestBuilder requestBuilder = new ReindexRequestBuilder(client); + try { + BulkByScrollResponse response = + requestBuilder + .source("source_index") + .destination("target_index") + .filter(QueryBuilders.matchQuery("category", "xzy")) // <1> + .get(); + } finally { + requestBuilder.request().decRef(); + } // end::reindex1 } @@ -224,6 +231,7 @@ public void testTasks() throws Exception { // unblocking the blocked update ALLOWED_OPERATIONS.release(builder.request().getSlices()); + builder.request().decRef(); } @SuppressWarnings("unused") @@ -266,14 +274,13 @@ private ReindexRequestBuilder reindexAndPartiallyBlock() throws Exception { final int numDocs = randomIntBetween(10, 100); ALLOWED_OPERATIONS.release(numDocs); - indexRandom( - true, - false, - true, - IntStream.range(0, numDocs) - .mapToObj(i -> prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("n", Integer.toString(i))) - .collect(Collectors.toList()) - ); + List indexRequestBuilders = IntStream.range(0, numDocs) + .mapToObj(i -> prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("n", Integer.toString(i))) + .collect(Collectors.toList()); + indexRandom(true, false, true, indexRequestBuilders); + for (IndexRequestBuilder builder : indexRequestBuilders) { + builder.request().decRef(); + } // Checks that the all documents have been indexed and correctly counted assertHitCount(prepareSearch(INDEX_NAME).setSize(0), numDocs); diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java index 5830c72956108..aa510f50e0ae9 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java @@ -135,6 +135,7 @@ public void testReindex() throws Exception { assertThat(bulkByScrollResponse.getCreated(), is((long) reindexDocCount)); } ); + reindexRequestBuilder.request().decRef(); } public void testDeleteByQuery() throws Exception { @@ -184,6 +185,9 @@ public void testDeleteByQuery() throws Exception { indexRequests.add(prepareIndex(sourceIndex).setId(Integer.toString(i)).setSource(source)); } indexRandom(true, indexRequests); + for (IndexRequestBuilder builder : indexRequests) { + builder.request().decRef(); + } final ThreadPool threadPool = internalCluster().getDataNodeInstance(ThreadPool.class); @@ -218,7 +222,12 @@ public void testDeleteByQuery() throws Exception { if (scriptEnabled && searchHit.getSourceAsMap().containsKey(RETURN_NOOP_FIELD)) { conflictingOps--; } - conflictingUpdatesBulkRequest.add(createUpdatedIndexRequest(searchHit, targetIndex, useOptimisticConcurrency)); + IndexRequest indexRequest = createUpdatedIndexRequest(searchHit, targetIndex, useOptimisticConcurrency); + try { + conflictingUpdatesBulkRequest.add(indexRequest); + } finally { + indexRequest.decRef(); + } } // The bulk request is enqueued before the update by query diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java index a4f939fbe3af8..cb030f6b622d6 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.reindex; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -46,7 +47,12 @@ protected Collection> nodePlugins(String clusterAlias) { private int indexDocs(Client client, String index) { int numDocs = between(1, 100); for (int i = 0; i < numDocs; i++) { - client.prepareIndex(index).setSource("f", "v").get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index).setSource("f", "v"); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } } client.admin().indices().prepareRefresh(index).get(); return numDocs; @@ -57,7 +63,7 @@ public void testReindexFromRemoteGivenIndexExists() throws Exception { final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "source-index-001"); final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "source-index-001"; - new ReindexRequestBuilder(client(LOCAL_CLUSTER)).source(sourceIndexInRemote).destination("desc-index-001").get(); + reindex(sourceIndexInRemote, "desc-index-001"); assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { final TotalHits totalHits = SearchResponseUtils.getTotalHits( @@ -72,7 +78,7 @@ public void testReindexFromRemoteGivenSameIndexNames() throws Exception { final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "test-index-001"); final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "test-index-001"; - new ReindexRequestBuilder(client(LOCAL_CLUSTER)).source(sourceIndexInRemote).destination("test-index-001").get(); + reindex(sourceIndexInRemote, "test-index-001"); assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { final TotalHits totalHits = SearchResponseUtils.getTotalHits( @@ -91,9 +97,7 @@ public void testReindexManyTimesFromRemoteGivenSameIndexNames() throws Exception int N = randomIntBetween(2, 10); for (int attempt = 0; attempt < N; attempt++) { - BulkByScrollResponse response = new ReindexRequestBuilder(client(LOCAL_CLUSTER)).source(sourceIndexInRemote) - .destination("test-index-001") - .get(); + BulkByScrollResponse response = reindex(sourceIndexInRemote, "test-index-001"); if (attempt == 0) { assertThat(response.getCreated(), equalTo(docsNumber)); @@ -115,10 +119,7 @@ public void testReindexManyTimesFromRemoteGivenSameIndexNames() throws Exception public void testReindexFromRemoteThrowOnUnavailableIndex() throws Exception { final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "no-such-source-index-001"; - expectThrows( - IndexNotFoundException.class, - () -> new ReindexRequestBuilder(client(LOCAL_CLUSTER)).source(sourceIndexInRemote).destination("desc-index-001").get() - ); + expectThrows(IndexNotFoundException.class, () -> reindex(sourceIndexInRemote, "desc-index-001")); // assert that local index was not created either final IndexNotFoundException e = expectThrows( @@ -133,7 +134,7 @@ public void testReindexFromRemoteGivenSimpleDateMathIndexName() throws Interrupt final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "datemath-2001-01-02"); final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + ""; - new ReindexRequestBuilder(client(LOCAL_CLUSTER)).source(sourceIndexInRemote).destination("desc-index-001").get(); + reindex(sourceIndexInRemote, "desc-index-001"); assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { final TotalHits totalHits = SearchResponseUtils.getTotalHits( @@ -149,7 +150,7 @@ public void testReindexFromRemoteGivenComplexDateMathIndexName() throws Interrup // Remote name contains `:` symbol twice final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + ""; - new ReindexRequestBuilder(client(LOCAL_CLUSTER)).source(sourceIndexInRemote).destination("desc-index-001").get(); + reindex(sourceIndexInRemote, "desc-index-001"); assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { final TotalHits totalHits = SearchResponseUtils.getTotalHits( @@ -159,4 +160,12 @@ public void testReindexFromRemoteGivenComplexDateMathIndexName() throws Interrup })); } + private BulkByScrollResponse reindex(String source, String destination) { + ReindexRequestBuilder builder = new ReindexRequestBuilder(client(LOCAL_CLUSTER)); + try { + return builder.source(source).destination(destination).get(); + } finally { + builder.request().decRef(); + } + } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java index 49cbc30258915..14a707ff1a7d9 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest; @@ -272,9 +273,24 @@ protected BulkRequest buildBulk(Iterable docs try { for (ScrollableHitSource.Hit doc : docs) { if (accept(doc)) { - RequestWrapper request = scriptApplier.apply(copyMetadata(buildRequest(doc), doc), doc); - if (request != null) { - bulkRequest.add(request.self()); + RequestWrapper originalRequest = buildRequest(doc); + try { + RequestWrapper request = scriptApplier.apply(copyMetadata(originalRequest, doc), doc); + if (request != null) { + try { + bulkRequest.add(request.self()); + } finally { + if (originalRequest.self() != request.self()) { + if (request.self() instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } + } + } + } finally { + if (originalRequest.self() instanceof RefCounted refCounted) { + refCounted.decRef(); + } } } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java index 48c50450656f3..0f27792067ca3 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java @@ -8,6 +8,7 @@ package org.elasticsearch.reindex; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActiveShardCount; @@ -70,7 +71,7 @@ protected RestChannelConsumer doPrepareRequest( throw validationException; } final var responseListener = new SubscribableListener(); - final var task = client.executeLocally(action, internal, responseListener); + final var task = client.executeLocally(action, internal, ActionListener.runAfter(responseListener, internal::decRef)); responseListener.addListener(new LoggingTaskListener<>(task)); return sendTask(client.getLocalNodeId(), task); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java index e4f734fa5721c..75abb984fe97b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java @@ -167,7 +167,7 @@ private static > void sendS r -> worker.onSliceResponse(listener, slice.source().slice().getId(), r), e -> worker.onSliceFailure(listener, slice.source().slice().getId(), e) ); - client.execute(action, requestForSlice, sliceListener); + client.execute(action, requestForSlice, ActionListener.runAfter(sliceListener, requestForSlice::decRef)); } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index dbe1968bb076a..ec37b08d873e8 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -296,62 +296,66 @@ public BiFunction, ScrollableHitSource.Hit, RequestWrapper> @Override protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) { IndexRequest index = new IndexRequest(); + try { + // Copy the index from the request so we always write where it asked to write + index.index(mainRequest.getDestination().index()); - // Copy the index from the request so we always write where it asked to write - index.index(mainRequest.getDestination().index()); + /* + * Internal versioning can just use what we copied from the destination request. Otherwise we assume we're using external + * versioning and use the doc's version. + */ + index.versionType(mainRequest.getDestination().versionType()); + if (index.versionType() == INTERNAL) { + assert doc.getVersion() == -1 : "fetched version when we didn't have to"; + index.version(mainRequest.getDestination().version()); + } else { + index.version(doc.getVersion()); + } - /* - * Internal versioning can just use what we copied from the destination request. Otherwise we assume we're using external - * versioning and use the doc's version. - */ - index.versionType(mainRequest.getDestination().versionType()); - if (index.versionType() == INTERNAL) { - assert doc.getVersion() == -1 : "fetched version when we didn't have to"; - index.version(mainRequest.getDestination().version()); - } else { - index.version(doc.getVersion()); - } + // id and source always come from the found doc. Scripts can change them but they operate on the index request. + index.id(destinationIndexIdMapper.reindexId(doc.getId())); - // id and source always come from the found doc. Scripts can change them but they operate on the index request. - index.id(destinationIndexIdMapper.reindexId(doc.getId())); + // the source xcontent type and destination could be different + final XContentType sourceXContentType = doc.getXContentType(); + final XContentType mainRequestXContentType = mainRequest.getDestination().getContentType(); + if (mainRequestXContentType != null && doc.getXContentType() != mainRequestXContentType) { + // we need to convert + try ( + XContentParser parser = XContentHelper.createParserNotCompressed( + XContentParserConfiguration.EMPTY, + doc.getSource(), + sourceXContentType + ); + XContentBuilder builder = XContentBuilder.builder(mainRequestXContentType.xContent()) + ) { + parser.nextToken(); + builder.copyCurrentStructure(parser); + index.source(BytesReference.bytes(builder), builder.contentType()); + } catch (IOException e) { + throw new UncheckedIOException( + "failed to convert hit from " + sourceXContentType + " to " + mainRequestXContentType, + e + ); + } + } else { + index.source(doc.getSource(), doc.getXContentType()); + } - // the source xcontent type and destination could be different - final XContentType sourceXContentType = doc.getXContentType(); - final XContentType mainRequestXContentType = mainRequest.getDestination().getContentType(); - if (mainRequestXContentType != null && doc.getXContentType() != mainRequestXContentType) { - // we need to convert - try ( - XContentParser parser = XContentHelper.createParserNotCompressed( - XContentParserConfiguration.EMPTY, - doc.getSource(), - sourceXContentType - ); - XContentBuilder builder = XContentBuilder.builder(mainRequestXContentType.xContent()) - ) { - parser.nextToken(); - builder.copyCurrentStructure(parser); - index.source(BytesReference.bytes(builder), builder.contentType()); - } catch (IOException e) { - throw new UncheckedIOException( - "failed to convert hit from " + sourceXContentType + " to " + mainRequestXContentType, - e - ); + /* + * The rest of the index request just has to be copied from the template. It may be changed later from scripts or the + * superclass here on out operates on the index request rather than the template. + */ + index.routing(mainRequest.getDestination().routing()); + index.setPipeline(mainRequest.getDestination().getPipeline()); + if (mainRequest.getDestination().opType() == DocWriteRequest.OpType.CREATE) { + index.opType(mainRequest.getDestination().opType()); } - } else { - index.source(doc.getSource(), doc.getXContentType()); - } - /* - * The rest of the index request just has to be copied from the template. It may be changed later from scripts or the superclass - * here on out operates on the index request rather than the template. - */ - index.routing(mainRequest.getDestination().routing()); - index.setPipeline(mainRequest.getDestination().getPipeline()); - if (mainRequest.getDestination().opType() == DocWriteRequest.OpType.CREATE) { - index.opType(mainRequest.getDestination().opType()); + return wrap(index); + } catch (Exception e) { + index.decRef(); + throw e; } - - return wrap(index); } /** diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java index de84d74d05ee8..b6d36e11b5686 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java @@ -126,13 +126,18 @@ public BiFunction, ScrollableHitSource.Hit, RequestWrapper> @Override protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) { IndexRequest index = new IndexRequest(); - index.index(doc.getIndex()); - index.id(doc.getId()); - index.source(doc.getSource(), doc.getXContentType()); - index.setIfSeqNo(doc.getSeqNo()); - index.setIfPrimaryTerm(doc.getPrimaryTerm()); - index.setPipeline(mainRequest.getPipeline()); - return wrap(index); + try { + index.index(doc.getIndex()); + index.id(doc.getId()); + index.source(doc.getSource(), doc.getXContentType()); + index.setIfSeqNo(doc.getSeqNo()); + index.setIfPrimaryTerm(doc.getPrimaryTerm()); + index.setPipeline(mainRequest.getPipeline()); + return wrap(index); + } catch (Exception e) { + index.decRef(); + throw e; + } } static class UpdateByQueryScriptApplier extends ScriptApplier { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java index aa338b3c87eca..3cf1b63600ed2 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java @@ -72,9 +72,23 @@ public void execute() { } } ); - AbstractAsyncBulkByScrollAction action = action(scriptService, request().setScript(mockScript(""))); - RequestWrapper result = action.buildScriptApplier().apply(AbstractAsyncBulkByScrollAction.wrap(index), doc); - return (result != null) ? (T) result.self() : null; + Request request = request(); + try { + AbstractAsyncBulkByScrollAction action = action(scriptService, request.setScript(mockScript(""))); + RequestWrapper result = action.buildScriptApplier().apply(AbstractAsyncBulkByScrollAction.wrap(index), doc); + if (result.self() == null) { + return null; + } + if (result.self() != index) { + index.decRef(); + } + return (T) result.self(); + } catch (Exception e) { + index.decRef(); + throw e; + } finally { + request.decRef(); + } } public void testScriptAddingJunkToCtxIsError() { @@ -93,6 +107,7 @@ public void testChangeSource() { source.put("bar", "cat"); }); assertEquals("cat", index.sourceAsMap().get("bar")); + index.decRef(); } public void testSetOpTypeDelete() throws Exception { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java index dfdc580ad8d7d..f1b45d1b25ec4 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java @@ -632,7 +632,9 @@ private void bulkRetryTestCase(boolean failWithRejection) throws Exception { action.setScroll(scrollId()); BulkRequest request = new BulkRequest(); for (int i = 0; i < size + 1; i++) { - request.add(new IndexRequest("index").id("id" + i)); + IndexRequest indexRequest = new IndexRequest("index").id("id" + i); + request.add(indexRequest); + indexRequest.decRef(); } if (failWithRejection) { action.sendBulkRequest(request, Assert::fail, request::close); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/CancelTests.java index b211f7d92f51f..1eb6db1818a86 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/CancelTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/CancelTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -26,6 +27,7 @@ import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexRequestBuilder; import org.elasticsearch.index.reindex.UpdateByQueryAction; import org.elasticsearch.index.shard.IndexingOperationListener; import org.elasticsearch.index.shard.ShardId; @@ -39,6 +41,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -92,14 +95,13 @@ private void testCancel( ALLOWED_OPERATIONS.release(numDocs); logger.debug("setting up [{}] docs", numDocs); - indexRandom( - true, - false, - true, - IntStream.range(0, numDocs) - .mapToObj(i -> prepareIndex(INDEX).setId(String.valueOf(i)).setSource("n", i)) - .collect(Collectors.toList()) - ); + List builders = IntStream.range(0, numDocs) + .mapToObj(i -> prepareIndex(INDEX).setId(String.valueOf(i)).setSource("n", i)) + .collect(Collectors.toList()); + indexRandom(true, false, true, builders); + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } // Checks that the all documents have been indexed and correctly counted assertHitCount(prepareSearch(INDEX).setSize(0), numDocs); @@ -221,12 +223,14 @@ public static TaskInfo findTaskToCancel(String actionName, int workerCount) { } public void testReindexCancel() throws Exception { - testCancel(ReindexAction.NAME, reindex().source(INDEX).destination("dest"), (response, total, modified) -> { + ReindexRequestBuilder builder = reindex(); + testCancel(ReindexAction.NAME, builder.source(INDEX).destination("dest"), (response, total, modified) -> { assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request"))); refresh("dest"); assertHitCount(prepareSearch("dest").setSize(0), modified); }, equalTo("reindex from [" + INDEX + "] to [dest]")); + builder.request().decRef(); } public void testUpdateByQueryCancel() throws Exception { @@ -260,9 +264,10 @@ public void testDeleteByQueryCancel() throws Exception { } public void testReindexCancelWithWorkers() throws Exception { + ReindexRequestBuilder builder = reindex(); testCancel( ReindexAction.NAME, - reindex().source(INDEX).filter(QueryBuilders.matchAllQuery()).destination("dest").setSlices(5), + builder.source(INDEX).filter(QueryBuilders.matchAllQuery()).destination("dest").setSlices(5), (response, total, modified) -> { assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); refresh("dest"); @@ -270,6 +275,7 @@ public void testReindexCancelWithWorkers() throws Exception { }, equalTo("reindex from [" + INDEX + "] to [dest]") ); + builder.request().decRef(); } public void testUpdateByQueryCancelWithWorkers() throws Exception { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryBasicTests.java index 17dd1503e6c89..6c699279bb505 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryBasicTests.java @@ -51,7 +51,7 @@ protected Collection> nodePlugins() { } public void testBasics() throws Exception { - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("foo", "a"), prepareIndex("test").setId("2").setSource("foo", "a"), @@ -90,7 +90,7 @@ public void testDeleteByQueryWithOneIndex() throws Exception { for (int i = 0; i < docs; i++) { builders.add(prepareIndex("test").setId(String.valueOf(i)).setSource("fields1", 1)); } - indexRandom(true, true, true, builders); + indexRandomAndDecRefRequests(true, true, true, builders); assertThat(deleteByQuery().source("t*").filter(QueryBuilders.matchAllQuery()).refresh(true).get(), matcher().deleted(docs)); assertHitCount(prepareSearch("test").setSize(0), 0); @@ -116,7 +116,7 @@ public void testDeleteByQueryWithMultipleIndices() throws Exception { builders.add(prepareIndex("test-" + i).setId(String.valueOf(j)).setSource("candidate", candidate)); } } - indexRandom(true, true, true, builders); + indexRandomAndDecRefRequests(true, true, true, builders); // Deletes all the documents with candidate=true assertThat(deleteByQuery().source("test-*").filter(termQuery("candidate", true)).refresh(true).get(), matcher().deleted(deletions)); @@ -130,7 +130,7 @@ public void testDeleteByQueryWithMultipleIndices() throws Exception { } public void testDeleteByQueryWithMissingIndex() throws Exception { - indexRandom(true, prepareIndex("test").setId("1").setSource("foo", "a")); + indexRandomAndDecRefRequests(true, prepareIndex("test").setId("1").setSource("foo", "a")); assertHitCount(prepareSearch().setSize(0), 1); try { @@ -152,7 +152,7 @@ public void testDeleteByQueryWithRouting() throws Exception { for (int i = 0; i < docs; i++) { builders.add(prepareIndex("test").setId(String.valueOf(i)).setRouting(String.valueOf(i)).setSource("field1", 1)); } - indexRandom(true, true, true, builders); + indexRandomAndDecRefRequests(true, true, true, builders); logger.info("--> counting documents with no routing, should be equal to [{}]", docs); assertHitCount(prepareSearch().setSize(0), docs); @@ -181,7 +181,7 @@ public void testDeleteByMatchQuery() throws Exception { prepareIndex("test").setId(Integer.toString(i)).setRouting(randomAlphaOfLengthBetween(1, 5)).setSource("foo", "bar") ); } - indexRandom(true, true, true, builders); + indexRandomAndDecRefRequests(true, true, true, builders); int n = between(0, docs - 1); assertHitCount(prepareSearch("test").setSize(0).setQuery(matchQuery("_id", Integer.toString(n))), 1); @@ -194,7 +194,7 @@ public void testDeleteByMatchQuery() throws Exception { } public void testDeleteByQueryWithDateMath() throws Exception { - indexRandom(true, prepareIndex("test").setId("1").setSource("d", "2013-01-01")); + indexRandomAndDecRefRequests(true, prepareIndex("test").setId("1").setSource("d", "2013-01-01")); DeleteByQueryRequestBuilder delete = deleteByQuery().source("test").filter(rangeQuery("d").to("now-1h")); assertThat(delete.refresh(true).get(), matcher().deleted(1L)); @@ -210,7 +210,7 @@ public void testDeleteByQueryOnReadOnlyIndex() throws Exception { for (int i = 0; i < docs; i++) { builders.add(prepareIndex("test").setId(Integer.toString(i)).setSource("field", 1)); } - indexRandom(true, true, true, builders); + indexRandomAndDecRefRequests(true, true, true, builders); try { enableIndexBlock("test", SETTING_READ_ONLY); @@ -233,7 +233,7 @@ public void testDeleteByQueryOnReadOnlyAllowDeleteIndex() throws Exception { for (int i = 0; i < docs; i++) { builders.add(prepareIndex("test").setId(Integer.toString(i)).setSource("field", 1)); } - indexRandom(true, true, true, builders); + indexRandomAndDecRefRequests(true, true, true, builders); // Because the index level read_only_allow_delete block can be automatically released by disk allocation decider, // so we should test both case of disk allocation decider is enabled and disabled @@ -285,7 +285,7 @@ public void testDeleteByQueryOnReadOnlyAllowDeleteIndex() throws Exception { } public void testSlices() throws Exception { - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("foo", "a"), prepareIndex("test").setId("2").setSource("foo", "a"), @@ -329,7 +329,7 @@ public void testMultipleSources() throws Exception { } List allDocs = docs.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); - indexRandom(true, allDocs); + indexRandomAndDecRefRequests(true, true, true, allDocs); for (Map.Entry> entry : docs.entrySet()) { assertHitCount(prepareSearch(entry.getKey()).setSize(0), entry.getValue().size()); } @@ -365,4 +365,29 @@ private void setDiskAllocationDeciderEnabled(boolean value) { : Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), false); updateClusterSettings(settings); } + + private void indexRandomAndDecRefRequests( + boolean forceRefresh, + boolean dummyDocuments, + boolean autoFlush, + List builders + ) throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, autoFlush, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryConcurrentTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryConcurrentTests.java index 323b829fe93ff..f0c62cc83b791 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryConcurrentTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryConcurrentTests.java @@ -36,6 +36,9 @@ public void testConcurrentDeleteByQueriesOnDifferentDocs() throws Throwable { } } indexRandom(true, true, true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } final CountDownLatch start = new CountDownLatch(1); for (int t = 0; t < threads.length; t++) { @@ -76,6 +79,9 @@ public void testConcurrentDeleteByQueriesOnSameDocs() throws Throwable { builders.add(prepareIndex("test").setId(String.valueOf(i)).setSource("foo", "bar")); } indexRandom(true, true, true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } final Thread[] threads = new Thread[scaledRandomIntBetween(2, 9)]; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java index 21f6427dcb632..d25d3af8a82b0 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java @@ -28,7 +28,7 @@ public class ReindexBasicTests extends ReindexTestCase { public void testFiltering() throws Exception { - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("source").setId("1").setSource("foo", "a"), prepareIndex("source").setId("2").setSource("foo", "a"), @@ -40,22 +40,26 @@ public void testFiltering() throws Exception { // Copy all the docs ReindexRequestBuilder copy = reindex().source("source").destination("dest").refresh(true); assertThat(copy.get(), matcher().created(4)); + copy.request().decRef(); assertHitCount(prepareSearch("dest").setSize(0), 4); // Now none of them createIndex("none"); copy = reindex().source("source").destination("none").filter(termQuery("foo", "no_match")).refresh(true); assertThat(copy.get(), matcher().created(0)); + copy.request().decRef(); assertHitCount(prepareSearch("none").setSize(0), 0); // Now half of them copy = reindex().source("source").destination("dest_half").filter(termQuery("foo", "a")).refresh(true); assertThat(copy.get(), matcher().created(2)); + copy.request().decRef(); assertHitCount(prepareSearch("dest_half").setSize(0), 2); // Limit with maxDocs copy = reindex().source("source").destination("dest_size_one").maxDocs(1).refresh(true); assertThat(copy.get(), matcher().created(1)); + copy.request().decRef(); assertHitCount(prepareSearch("dest_size_one").setSize(0), 1); } @@ -66,7 +70,7 @@ public void testCopyMany() throws Exception { docs.add(prepareIndex("source").setId(Integer.toString(i)).setSource("foo", "a")); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); assertHitCount(prepareSearch("source").setSize(0), max); // Copy all the docs @@ -74,6 +78,7 @@ public void testCopyMany() throws Exception { // Use a small batch size so we have to use more than one batch copy.source().setSize(5); assertThat(copy.get(), matcher().created(max).batches(max, 5)); + copy.request().decRef(); assertHitCount(prepareSearch("dest").setSize(0), max); // Copy some of the docs @@ -83,6 +88,7 @@ public void testCopyMany() throws Exception { copy.source().setSize(5); copy.maxDocs(half); assertThat(copy.get(), matcher().created(half).batches(half, 5)); + copy.request().decRef(); assertHitCount(prepareSearch("dest_half").setSize(0), half); } @@ -93,7 +99,7 @@ public void testCopyManyWithSlices() throws Exception { docs.add(prepareIndex("source").setId(Integer.toString(i)).setSource("foo", "a")); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); assertHitCount(prepareSearch("source").setSize(0), max); int slices = randomSlices(); @@ -104,6 +110,7 @@ public void testCopyManyWithSlices() throws Exception { // Use a small batch size so we have to use more than one batch copy.source().setSize(5); assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(expectedSlices))); + copy.request().decRef(); assertHitCount(prepareSearch("dest").setSize(0), max); // Copy some of the docs @@ -113,6 +120,7 @@ public void testCopyManyWithSlices() throws Exception { copy.source().setSize(5); copy.maxDocs(half); BulkByScrollResponse response = copy.get(); + copy.request().decRef(); assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(expectedSlices))); assertHitCount(prepareSearch("dest_half").setSize(0), response.getCreated()); } @@ -132,7 +140,7 @@ public void testMultipleSources() throws Exception { } List allDocs = docs.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); - indexRandom(true, allDocs); + indexRandomAndDecRefRequests(true, allDocs); for (Map.Entry> entry : docs.entrySet()) { assertHitCount(prepareSearch(entry.getKey()).setSize(0), entry.getValue().size()); } @@ -144,6 +152,7 @@ public void testMultipleSources() throws Exception { ReindexRequestBuilder request = reindex().source(sourceIndexNames).destination("dest").refresh(true).setSlices(slices); BulkByScrollResponse response = request.get(); + request.request().decRef(); assertThat(response, matcher().created(allDocs.size()).slices(hasSize(expectedSlices))); assertHitCount(prepareSearch("dest").setSize(0), allDocs.size()); } @@ -159,7 +168,7 @@ public void testMissingSources() { public void testReindexFromComplexDateMathIndexName() throws Exception { String sourceIndexName = "datemath-2001-01-01-14"; String destIndexName = ""; - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex(sourceIndexName).setId("1").setSource("foo", "a"), prepareIndex(sourceIndexName).setId("2").setSource("foo", "a"), @@ -171,7 +180,27 @@ public void testReindexFromComplexDateMathIndexName() throws Exception { // Copy all the docs ReindexRequestBuilder copy = reindex().source(sourceIndexName).destination(destIndexName).refresh(true); assertThat(copy.get(), matcher().created(4)); + copy.request().decRef(); assertHitCount(prepareSearch(destIndexName).setSize(0), 4); } + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFailureTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFailureTests.java index 1da998831ecc2..ac004680700f3 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFailureTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFailureTests.java @@ -37,7 +37,9 @@ public void testFailuresCauseAbortDefault() throws Exception { * Create the destination index such that the copy will cause a mapping * conflict on every request. */ - indexRandom(true, prepareIndex("dest").setId("test").setSource("test", 10) /* Its a string in the source! */); + IndexRequestBuilder builder = prepareIndex("dest").setId("test").setSource("test", 10); /* Its a string in the source! */ + indexRandom(true, builder); + builder.request().decRef(); indexDocs(100); @@ -50,6 +52,7 @@ public void testFailuresCauseAbortDefault() throws Exception { copy.source().setSize(1); BulkByScrollResponse response = copy.get(); + copy.request().decRef(); assertThat(response, matcher().batches(1).failures(both(greaterThan(0)).and(lessThanOrEqualTo(maximumNumberOfShards())))); for (Failure failure : response.getBulkFailures()) { assertThat(failure.getCause().getCause(), instanceOf(IllegalArgumentException.class)); @@ -59,7 +62,9 @@ public void testFailuresCauseAbortDefault() throws Exception { public void testAbortOnVersionConflict() throws Exception { // Just put something in the way of the copy. - indexRandom(true, prepareIndex("dest").setId("1").setSource("test", "test")); + IndexRequestBuilder builder = prepareIndex("dest").setId("1").setSource("test", "test"); + indexRandom(true, builder); + builder.request().decRef(); indexDocs(100); @@ -68,6 +73,7 @@ public void testAbortOnVersionConflict() throws Exception { copy.destination().setOpType(CREATE); BulkByScrollResponse response = copy.get(); + copy.request().decRef(); assertThat(response, matcher().batches(1).versionConflicts(1).failures(1).created(99)); for (Failure failure : response.getBulkFailures()) { assertThat(failure.getMessage(), containsString("VersionConflictEngineException: [")); @@ -113,6 +119,8 @@ public void testResponseOnSearchFailure() throws Exception { .or(containsString("Partial shards failure")) ); return; + } finally { + copy.request().decRef(); } } assumeFalse("Wasn't able to trigger a reindex failure in " + attempt + " attempts.", true); @@ -121,20 +129,28 @@ public void testResponseOnSearchFailure() throws Exception { public void testDateMathResolvesSameIndexName() throws Exception { String sourceIndexName = "datemath-2001-01-01-14"; String destIndexName = ""; - indexRandom( - true, + List builders = List.of( prepareIndex(sourceIndexName).setId("1").setSource("foo", "a"), prepareIndex(sourceIndexName).setId("2").setSource("foo", "a"), prepareIndex(sourceIndexName).setId("3").setSource("foo", "b"), prepareIndex(sourceIndexName).setId("4").setSource("foo", "c") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertHitCount(prepareSearch(sourceIndexName).setSize(0), 4); - ActionRequestValidationException e = expectThrows( - ActionRequestValidationException.class, - () -> reindex().source(sourceIndexName).destination(destIndexName).get() - ); - assertThat(e.getMessage(), containsString("reindex cannot write into an index its reading from [datemath-2001-01-01-14]")); + ReindexRequestBuilder requestBuilder = reindex(); + try { + ActionRequestValidationException e = expectThrows( + ActionRequestValidationException.class, + () -> requestBuilder.source(sourceIndexName).destination(destIndexName).get() + ); + assertThat(e.getMessage(), containsString("reindex cannot write into an index its reading from [datemath-2001-01-01-14]")); + } finally { + requestBuilder.request().decRef(); + } } private void indexDocs(int count) throws Exception { @@ -143,5 +159,8 @@ private void indexDocs(int count) throws Exception { docs.add(prepareIndex("source").setId(Integer.toString(i)).setSource("test", "words words")); } indexRandom(true, docs); + for (IndexRequestBuilder builder : docs) { + builder.request().decRef(); + } } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java index c641e04f84134..6bbd8a9d643b0 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilterChain; @@ -80,7 +81,12 @@ protected Settings nodeSettings() { @Before public void setupSourceIndex() { - prepareIndex("source").setSource("test", "test").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + IndexRequestBuilder builder = prepareIndex("source"); + try { + builder.setSource("test", "test").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + } finally { + builder.request().decRef(); + } } @Before @@ -112,33 +118,46 @@ public void testReindexFromRemoteWithAuthentication() throws Exception { .destination("dest") .setRemoteInfo(newRemoteInfo("Aladdin", "open sesame", emptyMap())); assertThat(request.get(), matcher().created(1)); + request.request().decRef(); } public void testReindexSendsHeaders() throws Exception { ReindexRequestBuilder request = new ReindexRequestBuilder(client()).source("source") .destination("dest") .setRemoteInfo(newRemoteInfo(null, null, singletonMap(TestFilter.EXAMPLE_HEADER, "doesn't matter"))); - ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> request.get()); - assertEquals(RestStatus.BAD_REQUEST, e.status()); - assertThat(e.getMessage(), containsString("Hurray! Sent the header!")); + try { + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> request.get()); + assertEquals(RestStatus.BAD_REQUEST, e.status()); + assertThat(e.getMessage(), containsString("Hurray! Sent the header!")); + } finally { + request.request().decRef(); + } } public void testReindexWithoutAuthenticationWhenRequired() throws Exception { ReindexRequestBuilder request = new ReindexRequestBuilder(client()).source("source") .destination("dest") .setRemoteInfo(newRemoteInfo(null, null, emptyMap())); - ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> request.get()); - assertEquals(RestStatus.UNAUTHORIZED, e.status()); - assertThat(e.getMessage(), containsString("\"reason\":\"Authentication required\"")); - assertThat(e.getMessage(), containsString("\"WWW-Authenticate\":\"Basic realm=auth-realm\"")); + try { + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> request.get()); + assertEquals(RestStatus.UNAUTHORIZED, e.status()); + assertThat(e.getMessage(), containsString("\"reason\":\"Authentication required\"")); + assertThat(e.getMessage(), containsString("\"WWW-Authenticate\":\"Basic realm=auth-realm\"")); + } finally { + request.request().decRef(); + } } public void testReindexWithBadAuthentication() throws Exception { ReindexRequestBuilder request = new ReindexRequestBuilder(client()).source("source") .destination("dest") .setRemoteInfo(newRemoteInfo("junk", "auth", emptyMap())); - ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> request.get()); - assertThat(e.getMessage(), containsString("\"reason\":\"Bad Authorization\"")); + try { + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> request.get()); + assertThat(e.getMessage(), containsString("\"reason\":\"Bad Authorization\"")); + } finally { + request.request().decRef(); + } } /** diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java index 644787446547e..c698f991910e8 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.reindex; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; @@ -35,23 +36,23 @@ */ public class ReindexIdTests extends AbstractAsyncBulkByScrollActionTestCase { public void testEmptyStateCopiesId() throws Exception { - assertThat(action(ClusterState.EMPTY_STATE).buildRequest(doc()).getId(), equalTo(doc().getId())); + testReindexId(ClusterState.EMPTY_STATE, false); } public void testStandardIndexCopiesId() throws Exception { - assertThat(action(stateWithIndex(standardSettings())).buildRequest(doc()).getId(), equalTo(doc().getId())); + testReindexId(stateWithIndex(standardSettings()), false); } public void testTsdbIndexClearsId() throws Exception { - assertThat(action(stateWithIndex(tsdbSettings())).buildRequest(doc()).getId(), nullValue()); + testReindexId(stateWithIndex(tsdbSettings()), true); } public void testMissingIndexWithStandardTemplateCopiesId() throws Exception { - assertThat(action(stateWithTemplate(standardSettings())).buildRequest(doc()).getId(), equalTo(doc().getId())); + testReindexId(stateWithTemplate(standardSettings()), false); } public void testMissingIndexWithTsdbTemplateClearsId() throws Exception { - assertThat(action(stateWithTemplate(tsdbSettings())).buildRequest(doc()).getId(), nullValue()); + testReindexId(stateWithTemplate(tsdbSettings()), true); } private ClusterState stateWithTemplate(Settings.Builder settings) { @@ -70,10 +71,12 @@ private ClusterState stateWithTemplate(Settings.Builder settings) { } private ClusterState stateWithIndex(Settings.Builder settings) { - IndexMetadata.Builder meta = IndexMetadata.builder(request().getDestination().index()) + ReindexRequest reindexRequest = request(); + IndexMetadata.Builder meta = IndexMetadata.builder(reindexRequest.getDestination().index()) .settings(settings.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) .numberOfReplicas(0) .numberOfShards(1); + reindexRequest.decRef(); return ClusterState.builder(ClusterState.EMPTY_STATE).metadata(Metadata.builder(Metadata.EMPTY_METADATA).put(meta)).build(); } @@ -102,4 +105,23 @@ protected ReindexRequest request() { private Reindexer.AsyncIndexBySearchAction action(ClusterState state) { return new Reindexer.AsyncIndexBySearchAction(task, logger, null, null, threadPool, null, state, null, request(), listener()); } + + private void testReindexId(ClusterState state, boolean nullValueExpected) { + Reindexer.AsyncIndexBySearchAction action = action(state); + try { + AbstractAsyncBulkByScrollAction.RequestWrapper indexRequest = action.buildRequest(doc()); + try { + String id = indexRequest.getId(); + if (nullValueExpected) { + assertThat(id, nullValue()); + } else { + assertThat(id, equalTo(doc().getId())); + } + } finally { + indexRequest.self().decRef(); + } + } finally { + action.mainRequest.decRef(); + } + } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java index a24536a7502c0..1e3e294b7c05c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java @@ -20,8 +20,11 @@ public class ReindexMetadataTests extends AbstractAsyncBulkByScrollActionMetadataTestCase { public void testRoutingCopiedByDefault() throws Exception { IndexRequest index = new IndexRequest(); - action().copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo")); + TestAction action = action(); + action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo")); assertEquals("foo", index.routing()); + index.decRef(); + action.mainRequest().decRef(); } public void testRoutingCopiedIfRequested() throws Exception { @@ -30,6 +33,8 @@ public void testRoutingCopiedIfRequested() throws Exception { IndexRequest index = new IndexRequest(); action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo")); assertEquals("foo", index.routing()); + index.decRef(); + action.mainRequest().decRef(); } public void testRoutingDiscardedIfRequested() throws Exception { @@ -38,6 +43,8 @@ public void testRoutingDiscardedIfRequested() throws Exception { IndexRequest index = new IndexRequest(); action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo")); assertEquals(null, index.routing()); + index.decRef(); + action.mainRequest().decRef(); } public void testRoutingSetIfRequested() throws Exception { @@ -46,6 +53,8 @@ public void testRoutingSetIfRequested() throws Exception { IndexRequest index = new IndexRequest(); action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo")); assertEquals("cat", index.routing()); + index.decRef(); + action.mainRequest().decRef(); } public void testRoutingSetIfWithDegenerateValue() throws Exception { @@ -54,6 +63,8 @@ public void testRoutingSetIfWithDegenerateValue() throws Exception { IndexRequest index = new IndexRequest(); action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo")); assertEquals("=]", index.routing()); + index.decRef(); + action.mainRequest().decRef(); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java index 380510b814f06..068f74b1b52b6 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java @@ -29,6 +29,7 @@ public void testSetIndex() throws Exception { Object dest = randomFrom(new Object[] { 234, 234L, "pancake" }); IndexRequest index = applyScript((Map ctx) -> ctx.put("_index", dest)); assertEquals(dest.toString(), index.index()); + index.decRef(); } public void testSettingIndexToNullIsError() throws Exception { @@ -47,6 +48,7 @@ public void testSetId() throws Exception { } else { assertEquals(id.toString(), index.id()); } + index.decRef(); } public void testSetVersion() throws Exception { @@ -57,6 +59,7 @@ public void testSetVersion() throws Exception { } else { assertEquals(version.longValue(), index.version()); } + index.decRef(); } public void testSettingVersionToJunkIsAnError() throws Exception { @@ -77,6 +80,7 @@ public void testSetRouting() throws Exception { String routing = randomRealisticUnicodeOfLengthBetween(5, 20); IndexRequest index = applyScript((Map ctx) -> ctx.put("_routing", routing)); assertEquals(routing, index.routing()); + index.decRef(); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSingleNodeTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSingleNodeTests.java index 855cb1863f399..1fc4220c8a3f8 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSingleNodeTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSingleNodeTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.reindex; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.reindex.ReindexRequestBuilder; import org.elasticsearch.plugins.Plugin; @@ -29,7 +30,9 @@ protected Collection> getPlugins() { public void testDeprecatedSort() { int max = between(2, 20); for (int i = 0; i < max; i++) { - prepareIndex("source").setId(Integer.toString(i)).setSource("foo", i).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setId(Integer.toString(i)).setSource("foo", i); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } indicesAdmin().prepareRefresh("source").get(); @@ -41,6 +44,7 @@ public void testDeprecatedSort() { copy.maxDocs(subsetSize); copy.request().addSortField("foo", SortOrder.DESC); assertThat(copy.get(), matcher().created(subsetSize)); + copy.request().decRef(); assertHitCount(client().prepareSearch("dest").setSize(0), subsetSize); assertHitCount(client().prepareSearch("dest").setQuery(new RangeQueryBuilder("foo").gte(0).lt(max - subsetSize)), 0); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSourceTargetValidationTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSourceTargetValidationTests.java index 38f289d894bd6..a31084dbedf68 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSourceTargetValidationTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSourceTargetValidationTests.java @@ -164,14 +164,19 @@ private void succeeds(String target, String... sources) { } private void succeeds(RemoteInfo remoteInfo, String target, String... sources) { - ReindexValidator.validateAgainstAliases( - new SearchRequest(sources), - new IndexRequest(target), - remoteInfo, - INDEX_NAME_EXPRESSION_RESOLVER, - AUTO_CREATE_INDEX, - STATE - ); + IndexRequest indexRequest = new IndexRequest(target); + try { + ReindexValidator.validateAgainstAliases( + new SearchRequest(sources), + indexRequest, + remoteInfo, + INDEX_NAME_EXPRESSION_RESOLVER, + AUTO_CREATE_INDEX, + STATE + ); + } finally { + indexRequest.decRef(); + } } private static IndexMetadata index(String name, String... aliases) { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexVersioningTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexVersioningTests.java index 8e42b29468b5c..7d6fbac384f43 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexVersioningTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexVersioningTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.reindex; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.ReindexRequestBuilder; @@ -80,8 +81,13 @@ public void testCreateVersionConflictsOnNewer() throws Exception { */ private BulkByScrollResponse reindexExternal() { ReindexRequestBuilder reindex = reindex().source("source").destination("dest").abortOnVersionConflict(false); - reindex.destination().setVersionType(EXTERNAL); - return reindex.get(); + try { + reindex.destination().setVersionType(EXTERNAL); + return reindex.get(); + } finally { + reindex.request().decRef(); + + } } /** @@ -89,8 +95,12 @@ private BulkByScrollResponse reindexExternal() { */ private BulkByScrollResponse reindexInternal() { ReindexRequestBuilder reindex = reindex().source("source").destination("dest").abortOnVersionConflict(false); - reindex.destination().setVersionType(INTERNAL); - return reindex.get(); + try { + reindex.destination().setVersionType(INTERNAL); + return reindex.get(); + } finally { + reindex.request().decRef(); + } } /** @@ -98,22 +108,33 @@ private BulkByScrollResponse reindexInternal() { */ private BulkByScrollResponse reindexCreate() { ReindexRequestBuilder reindex = reindex().source("source").destination("dest").abortOnVersionConflict(false); - reindex.destination().setOpType(CREATE); - return reindex.get(); + try { + reindex.destination().setOpType(CREATE); + return reindex.get(); + } finally { + reindex.request().decRef(); + } } private void setupSourceAbsent() throws Exception { - indexRandom( - true, - prepareIndex("source").setId("test").setVersionType(EXTERNAL).setVersion(SOURCE_VERSION).setSource("foo", "source") - ); + IndexRequestBuilder builder = prepareIndex("source").setId("test") + .setVersionType(EXTERNAL) + .setVersion(SOURCE_VERSION) + .setSource("foo", "source"); + indexRandom(true, builder); + builder.request().decRef(); assertEquals(SOURCE_VERSION, client().prepareGet("source", "test").get().getVersion()); } private void setupDest(int version) throws Exception { setupSourceAbsent(); - indexRandom(true, prepareIndex("dest").setId("test").setVersionType(EXTERNAL).setVersion(version).setSource("foo", "dest")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("dest").setId("test") + .setVersionType(EXTERNAL) + .setVersion(version) + .setSource("foo", "dest"); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertEquals(version, client().prepareGet("dest", "test").get().getVersion()); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java index 3484b61ca2c9a..278cb605c712a 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java @@ -70,6 +70,7 @@ public void testSetScrollTimeout() throws IOException { requestBuilder.withContent(new BytesArray("{}"), XContentType.JSON); ReindexRequest request = action.buildRequest(requestBuilder.build(), new NamedWriteableRegistry(Collections.emptyList())); assertEquals(AbstractBulkByScrollRequest.DEFAULT_SCROLL_TIMEOUT, request.getScrollTime()); + request.decRef(); } { FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry()); @@ -77,6 +78,7 @@ public void testSetScrollTimeout() throws IOException { requestBuilder.withContent(new BytesArray("{}"), XContentType.JSON); ReindexRequest request = action.buildRequest(requestBuilder.build(), new NamedWriteableRegistry(Collections.emptyList())); assertEquals("10m", request.getScrollTime().toString()); + request.decRef(); } } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RethrottleTests.java index 5f1af05571585..d88816c13e1fa 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RethrottleTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RethrottleTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexRequestBuilder; import org.elasticsearch.index.reindex.UpdateByQueryAction; import org.elasticsearch.tasks.TaskId; @@ -47,7 +48,9 @@ public class RethrottleTests extends ReindexTestCase { public void testReindex() throws Exception { - testCase(reindex().source("test").destination("dest"), ReindexAction.NAME); + ReindexRequestBuilder requestBuilder = reindex(); + testCase(requestBuilder.source("test").destination("dest"), ReindexAction.NAME); + requestBuilder.request().decRef(); } public void testUpdateByQuery() throws Exception { @@ -59,7 +62,9 @@ public void testDeleteByQuery() throws Exception { } public void testReindexWithWorkers() throws Exception { - testCase(reindex().source("test").destination("dest").setSlices(randomSlices()), ReindexAction.NAME); + ReindexRequestBuilder builder = reindex(); + testCase(builder.source("test").destination("dest").setSlices(randomSlices()), ReindexAction.NAME); + builder.request().decRef(); } public void testUpdateByQueryWithWorkers() throws Exception { @@ -84,6 +89,9 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a docs.add(prepareIndex("test").setId(Integer.toString(i)).setSource("foo", "bar")); } indexRandom(true, docs); + for (IndexRequestBuilder doc : docs) { + doc.request().decRef(); + } // Start a request that will never finish unless we rethrottle it request.setRequestsPerSecond(.000001f); // Throttle "forever" diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RetryTests.java index a3d61fcaf1ac0..7ade74f375bd5 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RetryTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.Retry; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; @@ -178,7 +179,9 @@ private void testCase( // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools. try (BulkRequestBuilder bulk = client().prepareBulk()) { for (int i = 0; i < DOC_COUNT; i++) { - bulk.add(prepareIndex("source").setSource("foo", "bar " + i)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setSource("foo", "bar " + i); + bulk.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } Retry retry = new Retry(BackoffPolicy.exponentialBackoff(), client().threadPool()); @@ -214,6 +217,7 @@ private void testCase( } assertThat(response.getSearchFailures(), empty()); assertThat(response.getBulkFailures(), empty()); + builder.request().decRef(); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RoundTripTests.java index eeb27cbca4b8a..e6d03e2f62757 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RoundTripTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RoundTripTests.java @@ -82,6 +82,8 @@ public void testReindexRequest() throws IOException { reindex.setSlices(between(1, Integer.MAX_VALUE)); tripped = new ReindexRequest(toInputByteStream(reindex)); assertRequestEquals(reindex, tripped); + reindex.decRef(); + tripped.decRef(); } public void testUpdateByQueryRequest() throws IOException { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryBasicTests.java index 6b1f3a21a1aad..abf6a0d2346b7 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryBasicTests.java @@ -27,7 +27,7 @@ public class UpdateByQueryBasicTests extends ReindexTestCase { public void testBasics() throws Exception { - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("foo", "a"), prepareIndex("test").setId("2").setSource("foo", "a"), @@ -67,7 +67,7 @@ public void testBasics() throws Exception { } public void testSlices() throws Exception { - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("foo", "a"), prepareIndex("test").setId("2").setSource("foo", "a"), @@ -123,6 +123,9 @@ public void testMultipleSources() throws Exception { List allDocs = docs.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); indexRandom(true, allDocs); + for (IndexRequestBuilder doc : allDocs) { + doc.request().decRef(); + } for (Map.Entry> entry : docs.entrySet()) { assertHitCount(prepareSearch(entry.getKey()).setSize(0), entry.getValue().size()); } @@ -149,4 +152,14 @@ public void testMissingSources() { .get(); assertThat(response, matcher().updated(0).slices(hasSize(0))); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWhileModifyingTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWhileModifyingTests.java index 5c2e82f6d4256..40ba705c84650 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWhileModifyingTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWhileModifyingTests.java @@ -32,7 +32,9 @@ public class UpdateByQueryWhileModifyingTests extends ReindexTestCase { public void testUpdateWhileReindexing() throws Exception { AtomicReference value = new AtomicReference<>(randomSimpleString(random())); - indexRandom(true, prepareIndex("test").setId("test").setSource("test", value.get())); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("test").setSource("test", value.get()); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); AtomicReference failure = new AtomicReference<>(); AtomicBoolean keepUpdating = new AtomicBoolean(true); @@ -67,6 +69,7 @@ public void testUpdateWhileReindexing() throws Exception { attempts++; try { index.setIfSeqNo(get.getSeqNo()).setIfPrimaryTerm(get.getPrimaryTerm()).get(); + index.request().decRef(); break; } catch (VersionConflictEngineException e) { if (attempts >= MAX_ATTEMPTS) { diff --git a/plugins/analysis-icu/src/internalClusterTest/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java b/plugins/analysis-icu/src/internalClusterTest/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java index 955bcaf8f0352..a1253185f14cb 100644 --- a/plugins/analysis-icu/src/internalClusterTest/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java +++ b/plugins/analysis-icu/src/internalClusterTest/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java @@ -11,6 +11,7 @@ import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin; @@ -67,7 +68,7 @@ public void testBasicUsage() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); // both values should collate to same value - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) @@ -109,7 +110,7 @@ public void testMultipleValues() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); // everything should be indexed fine, no exceptions - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex(index).setId("1") .setSource("{\"id\":\"1\", \"collate\":[\"" + equivalent[0] + "\", \"" + equivalent[1] + "\"]}", XContentType.JSON), @@ -173,7 +174,7 @@ public void testNormalization() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) @@ -219,7 +220,7 @@ public void testSecondaryStrength() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) @@ -265,7 +266,7 @@ public void testIgnorePunctuation() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) @@ -311,7 +312,7 @@ public void testIgnoreWhitespace() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"foo bar\"}", XContentType.JSON), prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"foobar\"}", XContentType.JSON), @@ -353,7 +354,7 @@ public void testNumerics() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); - indexRandom(true, prepareIndex(index).setId("1").setSource(""" + indexRandomAndDecRefRequests(true, prepareIndex(index).setId("1").setSource(""" {"collate":"foobar-10"}""", XContentType.JSON), prepareIndex(index).setId("2").setSource(""" {"collate":"foobar-9"}""", XContentType.JSON)); @@ -391,7 +392,7 @@ public void testIgnoreAccentsButNotCase() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); - indexRandom(true, prepareIndex(index).setId("1").setSource(""" + indexRandomAndDecRefRequests(true, prepareIndex(index).setId("1").setSource(""" {"id":"1","collate":"résumé"}""", XContentType.JSON), prepareIndex(index).setId("2").setSource(""" {"id":"2","collate":"Resume"}""", XContentType.JSON), prepareIndex(index).setId("3").setSource(""" {"id":"3","collate":"resume"}""", XContentType.JSON), prepareIndex(index).setId("4").setSource(""" @@ -428,7 +429,7 @@ public void testUpperCaseFirst() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex(index).setId("1").setSource("{\"collate\":\"resume\"}", XContentType.JSON), prepareIndex(index).setId("2").setSource("{\"collate\":\"Resume\"}", XContentType.JSON) @@ -477,7 +478,7 @@ public void testCustomRules() throws Exception { assertAcked(indicesAdmin().prepareCreate(index).setMapping(builder)); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) @@ -497,4 +498,14 @@ public void testCustomRules() throws Exception { assertOrderedSearchHits(response, "2", "1"); }); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java index ee16153a98de1..afe6ad0fe97a0 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.plugin.mapper.MapperSizePlugin; import org.elasticsearch.plugins.Plugin; @@ -100,7 +101,7 @@ private void assertSizeMappingEnabled(String index, boolean enabled) throws IOEx public void testBasic() throws Exception { assertAcked(prepareCreate("test").setMapping("_size", "enabled=true")); final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100) + "\"}"; - indexRandom(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); GetResponse getResponse = client().prepareGet("test", "1").setStoredFields("_size").get(); assertNotNull(getResponse.getField("_size")); assertEquals(source.length(), (int) getResponse.getField("_size").getValue()); @@ -109,7 +110,7 @@ public void testBasic() throws Exception { public void testGetWithFields() throws Exception { assertAcked(prepareCreate("test").setMapping("_size", "enabled=true")); final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100) + "\"}"; - indexRandom(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); assertResponse( prepareSearch("test").addFetchField("_size"), response -> assertEquals( @@ -134,7 +135,7 @@ public void testGetWithFields() throws Exception { public void testWildCardWithFieldsWhenDisabled() throws Exception { assertAcked(prepareCreate("test").setMapping("_size", "enabled=false")); final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100) + "\"}"; - indexRandom(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); assertResponse( prepareSearch("test").addFetchField("_size"), response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")) @@ -154,7 +155,7 @@ public void testWildCardWithFieldsWhenDisabled() throws Exception { public void testWildCardWithFieldsWhenNotProvided() throws Exception { assertAcked(prepareCreate("test")); final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100) + "\"}"; - indexRandom(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); assertResponse( prepareSearch("test").addFetchField("_size"), response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")) @@ -170,4 +171,14 @@ public void testWildCardWithFieldsWhenNotProvided() throws Exception { response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")) ); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java index 313dcdd6623c4..0441a06d1df3c 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.Settings; @@ -62,9 +63,9 @@ public void testSimpleWorkflow() { logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { - prepareIndex("test-idx-1").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); - prepareIndex("test-idx-2").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); - prepareIndex("test-idx-3").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); + indexDoc("test-idx-1", Integer.toString(i), "foo", "bar" + i); + indexDoc("test-idx-2", Integer.toString(i), "foo", "bar" + i); + indexDoc("test-idx-3", Integer.toString(i), "foo", "bar" + i); } client().admin().indices().prepareRefresh().get(); assertThat(count(client, "test-idx-1"), equalTo(100L)); @@ -279,4 +280,13 @@ public void testReplicationFactorOverReplicationMax() { private long count(Client client, String index) { return SearchResponseUtils.getTotalHitsValue(client.prepareSearch(index).setSize(0)); } + + private void indexDoc(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java index 8fbe9abe7f1b3..e2bc919721414 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.TransportMultiSearchAction; @@ -186,7 +187,10 @@ private static void indexTestData() { // Make sure we have a few segments try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int j = 0; j < 20; j++) { - bulkRequestBuilder.add(prepareIndex("test").setId(Integer.toString(i * 5 + j)).setSource("field", "value")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i * 5 + j)) + .setSource("field", "value"); + bulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } assertNoFailures(bulkRequestBuilder.get()); } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java index 081135d6b1e17..e66557dc4ea32 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.http; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; @@ -237,7 +238,10 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli indexRequest.source(Map.of("some_field", "some_value")); return channel -> client.index( indexRequest, - new RestToXContentListener<>(channel, DocWriteResponse::status, r -> r.getLocation(indexRequest.routing())) + ActionListener.runAfter( + new RestToXContentListener<>(channel, DocWriteResponse::status, r -> r.getLocation(indexRequest.routing())), + indexRequest::decRef + ) ); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java index 587081321c95f..2c20a6d20f943 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java @@ -54,6 +54,7 @@ import org.elasticsearch.action.get.TransportGetAction; import org.elasticsearch.action.get.TransportMultiGetAction; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchTransportService; import org.elasticsearch.action.search.SearchType; @@ -214,6 +215,7 @@ public void testIndex() { clearInterceptedActions(); assertSameIndices(indexRequest, indexShardActions); + indexRequest.decRef(); } public void testDelete() { @@ -236,13 +238,16 @@ public void testUpdate() { interceptTransportActions(updateShardActions); String indexOrAlias = randomIndexOrAlias(); - prepareIndex(indexOrAlias).setId("id").setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexOrAlias).setId("id").setSource("field", "value"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet(); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); clearInterceptedActions(); assertSameIndices(updateRequest, updateShardActions); + updateRequest.decRef(); } public void testUpdateUpsert() { @@ -261,6 +266,7 @@ public void testUpdateUpsert() { clearInterceptedActions(); assertSameIndices(updateRequest, updateShardActions); + assertThat("Something still has references to the update request", updateRequest.decRef(), equalTo(true)); } public void testUpdateDelete() { @@ -272,7 +278,9 @@ public void testUpdateDelete() { interceptTransportActions(updateShardActions); String indexOrAlias = randomIndexOrAlias(); - prepareIndex(indexOrAlias).setId("id").setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexOrAlias).setId("id").setSource("field", "value"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "id").script( new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "ctx.op='delete'", Collections.emptyMap()) ); @@ -281,6 +289,7 @@ public void testUpdateDelete() { clearInterceptedActions(); assertSameIndices(updateRequest, updateShardActions); + updateRequest.decRef(); } public void testBulk() { @@ -292,7 +301,9 @@ public void testBulk() { int numIndexRequests = iterations(1, 10); for (int i = 0; i < numIndexRequests; i++) { String indexOrAlias = randomIndexOrAlias(); - bulkRequest.add(new IndexRequest(indexOrAlias).id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value")); + IndexRequest indexRequest = new IndexRequest(indexOrAlias).id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + bulkRequest.add(indexRequest); + indexRequest.decRef(); indicesOrAliases.add(indexOrAlias); } int numDeleteRequests = iterations(1, 10); @@ -304,7 +315,9 @@ public void testBulk() { int numUpdateRequests = iterations(1, 10); for (int i = 0; i < numUpdateRequests; i++) { String indexOrAlias = randomIndexOrAlias(); - bulkRequest.add(new UpdateRequest(indexOrAlias, "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1")); + UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); + bulkRequest.add(updateRequest); + updateRequest.decRef(); indicesOrAliases.add(indexOrAlias); } @@ -564,7 +577,9 @@ public void testSearchQueryThenFetch() throws Exception { String[] randomIndicesOrAliases = randomIndicesOrAliases(); for (int i = 0; i < randomIndicesOrAliases.length; i++) { - prepareIndex(randomIndicesOrAliases[i]).setId("id-" + i).setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(randomIndicesOrAliases[i]).setId("id-" + i).setSource("field", "value"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } refresh(); @@ -594,7 +609,9 @@ public void testSearchDfsQueryThenFetch() throws Exception { String[] randomIndicesOrAliases = randomIndicesOrAliases(); for (int i = 0; i < randomIndicesOrAliases.length; i++) { - prepareIndex(randomIndicesOrAliases[i]).setId("id-" + i).setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(randomIndicesOrAliases[i]).setId("id-" + i).setSource("field", "value"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/ListenerActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/ListenerActionIT.java index 535b9b73c9dc2..573bb550c588d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/ListenerActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/ListenerActionIT.java @@ -29,7 +29,7 @@ public void testThreadedListeners() throws Throwable { request.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); } - client.index(request, new ActionListener() { + client.index(request, ActionListener.runAfter(new ActionListener<>() { @Override public void onResponse(DocWriteResponse indexResponse) { threadName.set(Thread.currentThread().getName()); @@ -42,7 +42,7 @@ public void onFailure(Exception e) { failure.set(e); latch.countDown(); } - }); + }, request::decRef)); latch.await(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/RejectionActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/RejectionActionIT.java index a30d654900c20..4495e6ba0a716 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/RejectionActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/RejectionActionIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.action; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; @@ -43,7 +44,9 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { public void testSimulatedSearchRejectionLoad() throws Throwable { for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "1").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)).setSource("field", "1"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } int numberOfAsyncOps = randomIntBetween(200, 700); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java index 45865ddd35ced..b45cb2e2e06db 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.logging.ChunkedLoggingStreamTests; @@ -91,12 +92,13 @@ public void onFailure(Exception e) { } }); - indexRandom( - true, - prepareIndex("test").setId("1").setSource("field1", "value1"), - prepareIndex("test").setId("2").setSource("field1", "value2"), - prepareIndex("test").setId("3").setSource("field1", "value3") - ); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test").setId("1").setSource("field1", "value1"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test").setId("2").setSource("field1", "value2"); + IndexRequestBuilder indexRequestBuilder3 = prepareIndex("test").setId("3").setSource("field1", "value3"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2, indexRequestBuilder3); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); + indexRequestBuilder3.request().decRef(); ensureSearchable(); while (latch.getCount() > 0) { assertHitCount( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java index 5c49dcba3a28f..01838ba4643aa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java @@ -130,7 +130,9 @@ private static Set getShardNodeIds(IndexShardRoutingTable shardRoutingTa private static void indexData(String index) { try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { for (int i = 0; i < randomIntBetween(5, 32); i++) { - bulkRequestBuilder.add(new IndexRequest(index).id(String.valueOf(i)).source("field", "foo " + i)); + IndexRequest indexRequest = new IndexRequest(index).id(String.valueOf(i)).source("field", "foo " + i); + bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } var bulkResponse = bulkRequestBuilder.get(); assertFalse(bulkResponse.hasFailures()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 7a1de49fcb031..6b779134d7cfb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.TransportIndexAction; import org.elasticsearch.action.search.SearchTransportService; import org.elasticsearch.action.search.TransportSearchAction; @@ -305,7 +306,10 @@ public void testTransportBulkTasks() { // ensures the mapping is available on all nodes so we won't retry the request (in case replicas don't have the right mapping). indicesAdmin().preparePutMapping("test").setSource("foo", "type=keyword").get(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - bulkRequestBuilder.add(prepareIndex("test").setId("test_id").setSource("{\"foo\": \"bar\"}", XContentType.JSON)).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("test_id") + .setSource("{\"foo\": \"bar\"}", XContentType.JSON); + bulkRequestBuilder.add(indexRequestBuilder).get(); + indexRequestBuilder.request().decRef(); } // the bulk operation should produce one main task @@ -355,10 +359,11 @@ public void testSearchTaskDescriptions() { registerTaskManagerListeners(TransportSearchAction.TYPE.name() + "[*]"); // shard task createIndex("test"); ensureGreen("test"); // Make sure all shards are allocated to catch replication tasks - prepareIndex("test").setId("test_id") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("test_id") .setSource("{\"foo\": \"bar\"}", XContentType.JSON) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Map headers = new HashMap<>(); headers.put(Task.X_OPAQUE_ID_HTTP_HEADER, "my_id"); @@ -449,7 +454,9 @@ public void onTaskRegistered(Task task) { } // Need to run the task in a separate thread because node client's .execute() is blocked by our task listener index = new Thread(() -> { - DocWriteResponse indexResponse = prepareIndex("test").setSource("test", "test").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource("test", "test"); + DocWriteResponse indexResponse = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertArrayEquals(ReplicationResponse.NO_FAILURES, indexResponse.getShardInfo().getFailures()); }); index.start(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index b6b0b2e54e691..c467057bc2fc8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.RestStatus; @@ -46,11 +47,15 @@ protected void setUpRepository() throws Exception { int docs = between(10, 100); for (int i = 0; i < docs; i++) { - prepareIndex(INDEX_NAME).setSource("test", "init").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(INDEX_NAME).setSource("test", "init"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } docs = between(10, 100); for (int i = 0; i < docs; i++) { - prepareIndex(OTHER_INDEX_NAME).setSource("test", "init").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(OTHER_INDEX_NAME).setSource("test", "init"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } logger.info("--> register a repository"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java index dc00c36470de2..50f0b32a0c6fd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; @@ -115,8 +116,9 @@ public void testWriteToAliasPrimaryAutoCreatedFirst() throws Exception { CreateIndexRequest request = new CreateIndexRequest(INDEX_NAME + "-2"); client().execute(AutoCreateAction.INSTANCE, request).get(); } - - DocWriteResponse response = prepareIndex(INDEX_NAME).setSource("{\"foo\":\"bar\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(INDEX_NAME).setSource("{\"foo\":\"bar\"}", XContentType.JSON); + DocWriteResponse response = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED)); } @@ -135,7 +137,9 @@ public void testWriteToAliasSecondaryAutoCreatedFirst() throws Exception { client().execute(AutoCreateAction.INSTANCE, request).get(); } - DocWriteResponse response = prepareIndex(INDEX_NAME).setSource("{\"foo\":\"bar\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(INDEX_NAME).setSource("{\"foo\":\"bar\"}", XContentType.JSON); + DocWriteResponse response = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED)); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java index d006192579ead..70aafa69ec1a8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; @@ -39,7 +40,12 @@ public void testCreateCloneIndex() { ).get(); final int docs = randomIntBetween(0, 128); for (int i = 0; i < docs; i++) { - prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } internalCluster().ensureAtLeastNumDataNodes(2); // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node @@ -89,7 +95,12 @@ public void testCreateCloneIndex() { } for (int i = docs; i < 2 * docs; i++) { - prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("target").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } flushAndRefresh(); assertHitCount(prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")), 2 * docs); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 7574cd0271c46..c4cd400344de6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -212,7 +213,7 @@ public void testCreateAndDeleteIndexConcurrently() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(1); int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setSource("index_version", indexVersion.get()).get(); + indexDoc("test", null, "index_version", indexVersion.get()); } synchronized (indexVersionLock) { // not necessarily needed here but for completeness we lock here too indexVersion.incrementAndGet(); @@ -225,7 +226,7 @@ public void onResponse(AcknowledgedResponse deleteIndexResponse) { public void run() { try { // recreate that index - prepareIndex("test").setSource("index_version", indexVersion.get()).get(); + indexDoc("test", null, "index_version", indexVersion.get()); synchronized (indexVersionLock) { // we sync here since we have to ensure that all indexing operations below for a given ID are done before // we increment the index version otherwise a doc that is in-flight could make it into an index that it @@ -249,15 +250,19 @@ public void onFailure(Exception e) { }); numDocs = randomIntBetween(100, 200); for (int i = 0; i < numDocs; i++) { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); try { synchronized (indexVersionLock) { - prepareIndex("test").setSource("index_version", indexVersion.get()).setTimeout(TimeValue.timeValueSeconds(10)).get(); + indexRequestBuilder.setSource("index_version", indexVersion.get()).setTimeout(TimeValue.timeValueSeconds(10)); + indexRequestBuilder.get(); } } catch (IndexNotFoundException inf) { // fine } catch (UnavailableShardsException ex) { assertEquals(ex.getCause().getClass(), IndexNotFoundException.class); // fine we run into a delete index while retrying + } finally { + indexRequestBuilder.request().decRef(); } } latch.await(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index c82ec12ff6244..d2d13262b50b6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -311,8 +311,11 @@ public void onFailure(Exception e) { } }; for (int i = 0; i < count; i++) { + IndexRequest indexRequest = new IndexRequest(INDEX_NAME).source(Map.of("foo", "bar")); try (BulkRequest bulkRequest = new BulkRequest()) { - client.bulk(bulkRequest.add(new IndexRequest(INDEX_NAME).source(Map.of("foo", "bar"))), listener); + client.bulk(bulkRequest.add(indexRequest), listener); + } finally { + indexRequest.decRef(); } } assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index aa4fee3a3f94d..06af1f0867594 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -81,7 +82,10 @@ public void testCreateShrinkIndexToN() { internalCluster().ensureAtLeastNumDataNodes(2); prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0])).get(); for (int i = 0; i < 20; i++) { - prepareIndex("source").setId(Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setId(Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", " + "\"i\" : " + i + "}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } Map dataNodes = clusterAdmin().prepareState().get().getState().nodes().getDataNodes(); assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); @@ -106,9 +110,10 @@ public void testCreateShrinkIndexToN() { assertHitCount(prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20); for (int i = 0; i < 20; i++) { // now update - prepareIndex("first_shrink").setId(Integer.toString(i)) - .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("first_shrink").setId(Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } flushAndRefresh(); assertHitCount(prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20); @@ -138,9 +143,10 @@ public void testCreateShrinkIndexToN() { assertHitCount(prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20); for (int i = 0; i < 20; i++) { // now update - prepareIndex("second_shrink").setId(Integer.toString(i)) - .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("second_shrink").setId(Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } flushAndRefresh(); assertHitCount(prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20); @@ -186,6 +192,7 @@ public void testShrinkIndexPrimaryTerm() throws Exception { final IndexRequest request = new IndexRequest("source").id(s) .source("{ \"f\": \"" + s + "\"}", XContentType.JSON); client().index(request).get(); + request.decRef(); break; } else { id++; @@ -233,7 +240,12 @@ public void testCreateShrinkIndex() { ).get(); final int docs = randomIntBetween(0, 128); for (int i = 0; i < docs; i++) { - prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } Map dataNodes = clusterAdmin().prepareState().get().getState().nodes().getDataNodes(); assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); @@ -313,7 +325,12 @@ public void testCreateShrinkIndex() { } for (int i = docs; i < 2 * docs; i++) { - prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("target").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } flushAndRefresh(); assertHitCount(prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")), 2 * docs); @@ -339,7 +356,12 @@ public void testCreateShrinkIndexFails() throws Exception { Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7)).put("number_of_replicas", 0) ).get(); for (int i = 0; i < 20; i++) { - prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } Map dataNodes = clusterAdmin().prepareState().get().getState().nodes().getDataNodes(); assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); @@ -419,7 +441,10 @@ public void testCreateShrinkWithIndexSort() throws Exception { .put("number_of_replicas", 0) ).setMapping("id", "type=keyword,doc_values=true").get(); for (int i = 0; i < 20; i++) { - prepareIndex("source").setId(Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"id\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setId(Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", " + "\"id\" : " + i + "}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } Map dataNodes = clusterAdmin().prepareState().get().getState().nodes().getDataNodes(); assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); @@ -462,7 +487,12 @@ public void testCreateShrinkWithIndexSort() throws Exception { // ... and that the index sort is also applied to updates for (int i = 20; i < 40; i++) { - prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("target").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } flushAndRefresh(); assertSortedSegments("target", expectedIndexSort); @@ -473,7 +503,12 @@ public void testShrinkCommitsMergeOnIdle() throws Exception { Settings.builder().put(indexSettings()).put("index.number_of_replicas", 0).put("number_of_shards", 5) ).get(); for (int i = 0; i < 30; i++) { - prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } indicesAdmin().prepareFlush("source").get(); Map dataNodes = clusterAdmin().prepareState().get().getState().nodes().getDataNodes(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 27fd54c39cc95..b5e191909e3a8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -151,26 +151,34 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha }; for (int i = 0; i < numDocs; i++) { IndexRequestBuilder builder = indexFunc.apply("source", i); - if (useRouting) { - String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 10); - if (useMixedRouting && randomBoolean()) { - routingValue[i] = null; - } else { - routingValue[i] = routing; + try { + if (useRouting) { + String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 10); + if (useMixedRouting && randomBoolean()) { + routingValue[i] = null; + } else { + routingValue[i] = routing; + } + builder.setRouting(routingValue[i]); } - builder.setRouting(routingValue[i]); + builder.get(); + } finally { + builder.request().decRef(); } - builder.get(); } if (randomBoolean()) { for (int i = 0; i < numDocs; i++) { // let's introduce some updates / deletes on the index if (randomBoolean()) { IndexRequestBuilder builder = indexFunc.apply("source", i); - if (useRouting) { - builder.setRouting(routingValue[i]); + try { + if (useRouting) { + builder.setRouting(routingValue[i]); + } + builder.get(); + } finally { + builder.request().decRef(); } - builder.get(); } } } @@ -193,10 +201,14 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha for (int i = 0; i < numDocs; i++) { // now update IndexRequestBuilder builder = indexFunc.apply("first_split", i); - if (useRouting) { - builder.setRouting(routingValue[i]); + try { + if (useRouting) { + builder.setRouting(routingValue[i]); + } + builder.get(); + } finally { + builder.request().decRef(); } - builder.get(); } flushAndRefresh(); assertHitCount(prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), numDocs); @@ -225,10 +237,14 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha for (int i = 0; i < numDocs; i++) { // now update IndexRequestBuilder builder = indexFunc.apply("second_split", i); - if (useRouting) { - builder.setRouting(routingValue[i]); + try { + if (useRouting) { + builder.setRouting(routingValue[i]); + } + builder.get(); + } finally { + builder.request().decRef(); } - builder.get(); } flushAndRefresh(); for (int i = 0; i < numDocs; i++) { @@ -301,6 +317,7 @@ public void testSplitIndexPrimaryTerm() throws Exception { final IndexRequest request = new IndexRequest("source").id(s) .source("{ \"f\": \"" + s + "\"}", XContentType.JSON); client().index(request).get(); + request.decRef(); break; } else { id++; @@ -343,7 +360,12 @@ public void testCreateSplitIndex() throws Exception { ).get(); final int docs = randomIntBetween(0, 128); for (int i = 0; i < docs; i++) { - prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } internalCluster().ensureAtLeastNumDataNodes(2); // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node @@ -412,7 +434,12 @@ public void testCreateSplitIndex() throws Exception { } for (int i = docs; i < 2 * docs; i++) { - prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("target").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } flushAndRefresh(); assertHitCount(prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")), 2 * docs); @@ -445,7 +472,10 @@ public void testCreateSplitWithIndexSort() throws Exception { .put("number_of_replicas", 0) ).setMapping("id", "type=keyword,doc_values=true").get(); for (int i = 0; i < 20; i++) { - prepareIndex("source").setId(Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"id\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source").setId(Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"id\" : " + i + "}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due @@ -482,7 +512,12 @@ public void testCreateSplitWithIndexSort() throws Exception { // ... and that the index sort is also applied to updates for (int i = 20; i < 40; i++) { - prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("target").setSource( + "{\"foo\" : \"bar\", \"i\" : " + i + "}", + XContentType.JSON + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } flushAndRefresh(); assertSortedSegments("target", expectedIndexSort); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java index 3560b74189d1d..0dd91b98e1acb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.indices.delete; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -44,12 +45,16 @@ public void testDeleteIndexWithBlocks() { public void testDeleteIndexOnIndexReadOnlyAllowDeleteSetting() { createIndex("test"); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("foo", "bar").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("foo", "bar"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); try { updateIndexSettings(Settings.builder().put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, true), "test"); assertSearchHits(prepareSearch(), "1"); - assertBlocked(prepareIndex("test").setId("2").setSource("foo", "bar"), IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); + indexRequestBuilder = prepareIndex("test").setId("2").setSource("foo", "bar"); + assertBlocked(indexRequestBuilder, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); + indexRequestBuilder.request().decRef(); assertSearchHits(prepareSearch(), "1"); assertAcked(indicesAdmin().prepareDelete("test")); } finally { @@ -65,7 +70,9 @@ public void testClusterBlockMessageHasIndexName() { createIndex("test"); ensureGreen("test"); updateIndexSettings(Settings.builder().put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, true), "test"); - ClusterBlockException e = expectThrows(ClusterBlockException.class, prepareIndex("test").setId("1").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("foo", "bar"); + ClusterBlockException e = expectThrows(ClusterBlockException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertEquals( "index [test] blocked by: [TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark, " + "index has read-only-allow-delete block];", @@ -79,12 +86,16 @@ public void testClusterBlockMessageHasIndexName() { public void testDeleteIndexOnClusterReadOnlyAllowDeleteSetting() { createIndex("test"); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("foo", "bar").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("foo", "bar"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); try { updateClusterSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true)); assertSearchHits(prepareSearch(), "1"); - assertBlocked(prepareIndex("test").setId("2").setSource("foo", "bar"), Metadata.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK); + indexRequestBuilder = prepareIndex("test").setId("2").setSource("foo", "bar"); + assertBlocked(indexRequestBuilder, Metadata.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK); + indexRequestBuilder.request().decRef(); assertBlocked( indicesAdmin().prepareUpdateSettings("test").setSettings(Settings.builder().put("index.number_of_replicas", 2)), Metadata.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java index a0d437d8baa73..eef0341ee8f78 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java @@ -10,6 +10,7 @@ import org.apache.lucene.tests.util.English; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -115,7 +116,9 @@ public void testSimple() throws Exception { .field("english_text", English.intToEnglish(value)) .field("value", value) .endObject(); - prepareIndex(index).setId("id-" + i).setSource(doc).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId("id-" + i).setSource(doc); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } final boolean forceNorms = randomBoolean(); if (forceNorms) { @@ -123,7 +126,9 @@ public void testSimple() throws Exception { .startObject() .field("english_text", "A long sentence to make sure that norms is non-zero") .endObject(); - prepareIndex(index).setId("id").setSource(doc).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId("id").setSource(doc); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } // Force merge to ensure that there are more than one numeric value to justify doc value. client().admin().indices().prepareForceMerge(index).setMaxNumSegments(1).get(); @@ -167,7 +172,9 @@ public void testFailOnFlush() throws Exception { .field("english_text", English.intToEnglish(value)) .field("value", value) .endObject(); - prepareIndex(indexName).setId("id-" + i).setSource(doc).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId("id-" + i).setSource(doc); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } Index index = clusterService().state().metadata().index(indexName).getIndex(); List failedShards = randomSubsetOf( @@ -203,7 +210,9 @@ public void testManyShards() throws Exception { .field("english_text", English.intToEnglish(value)) .field("value", value) .endObject(); - prepareIndex(indexName).setId("id-" + i).setSource(doc).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId("id-" + i).setSource(doc); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } } @@ -237,7 +246,9 @@ public void testFailingTargetShards() throws Exception { .field("english_text", English.intToEnglish(value)) .field("value", value) .endObject(); - prepareIndex(indexName).setId("id-" + i).setSource(doc).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId("id-" + i).setSource(doc); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } final Index index = resolveIndex(indexName); final List failingShards = randomSubsetOf( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java index 4e2fade87196f..801fe5ce990ce 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.indices.flush; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -32,7 +33,9 @@ public void testFlushWithBlocks() { int docs = between(10, 100); for (int i = 0; i < docs; i++) { - prepareIndex("test").setId("" + i).setSource("test", "init").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("" + i).setSource("test", "init"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } // Request is not blocked diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java index b5d8ef0308b91..fe75e92364d6a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.indices.forcemerge; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; @@ -44,7 +45,9 @@ public void testForceMergeWithBlocks() { int docs = between(10, 100); for (int i = 0; i < docs; i++) { - prepareIndex("test").setId("" + i).setSource("test", "init").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("" + i).setSource("test", "init"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } // Request is not blocked diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java index 0705e1216af43..b619503693d19 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java @@ -29,7 +29,7 @@ public void testIndicesSegmentsWithBlocks() { int docs = between(10, 100); for (int i = 0; i < docs; i++) { - prepareIndex("test-blocks").setId("" + i).setSource("test", "init").get(); + indexDoc("test-blocks", "" + i, "test", "init"); } indicesAdmin().prepareFlush("test-blocks").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java index 1a070c8bd0de3..8c2c4a33cf769 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java @@ -211,6 +211,9 @@ private void indexRandomData(String index) throws InterruptedException { builders[i] = prepareIndex(index).setSource("field", "value"); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } indicesAdmin().prepareFlush().setForce(true).get(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 2741652c37a92..1ab14678e9b61 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -14,8 +14,10 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.ingest.IngestTestPlugin; @@ -88,6 +90,7 @@ public void testBulkWithWriteIndexAndRouting() { indexRequestWithAlias.source(Collections.singletonMap("foo", "baz")); try (BulkRequestBuilder bulkBuilder = client().prepareBulk()) { BulkResponse bulkResponse = bulkBuilder.add(indexRequestWithAlias).get(); + indexRequestWithAlias.decRef(); assertThat(bulkResponse.getItems()[0].getResponse().getIndex(), equalTo("index3")); assertThat(bulkResponse.getItems()[0].getResponse().getShardId().getId(), equalTo(0)); assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(1L)); @@ -96,7 +99,9 @@ public void testBulkWithWriteIndexAndRouting() { } try (BulkRequestBuilder bulkBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkBuilder.add(client().prepareUpdate("alias1", "id").setDoc("foo", "updated")).get(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("alias1", "id").setDoc("foo", "updated"); + BulkResponse bulkResponse = bulkBuilder.add(updateRequestBuilder).get(); + updateRequestBuilder.request().decRef(); assertFalse(bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); assertThat(client().prepareGet("index3", "id").setRouting("1").get().getSource().get("foo"), equalTo("updated")); } @@ -117,6 +122,8 @@ public void testExternallySetAutoGeneratedTimestamp() { assertThat(expectThrows(IllegalArgumentException.class, () -> { try (BulkRequestBuilder bulkBuilder = client().prepareBulk()) { bulkBuilder.add(indexRequest).get(); + } finally { + indexRequest.decRef(); } }).getMessage(), containsString("autoGeneratedTimestamp should not be set externally")); } @@ -163,14 +170,17 @@ public void testDeleteIndexWhileIndexing() throws Exception { threads[i] = new Thread(() -> { while (stopped.get() == false && docID.get() < 5000) { String id = Integer.toString(docID.incrementAndGet()); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); try { - DocWriteResponse response = prepareIndex(index).setId(id) + DocWriteResponse response = indexRequestBuilder.setId(id) .setSource(Map.of("f" + randomIntBetween(1, 10), randomNonNegativeLong()), XContentType.JSON) .get(); assertThat(response.getResult(), is(oneOf(CREATED, UPDATED))); logger.info("--> index id={} seq_no={}", response.getId(), response.getSeqNo()); } catch (ElasticsearchException ignore) { logger.info("--> fail to index id={}", id); + } finally { + indexRequestBuilder.request().decRef(); } } }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java index f5fdb7b50c59c..93c05808db12b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java @@ -175,16 +175,17 @@ public void testBulkProcessor2ConcurrentRequestsReadOnlyIndex() throws Exception for (int i = 1; i <= numDocs; i++) { if (randomBoolean()) { testDocs++; - processor.add( - new IndexRequest("test").id(Integer.toString(testDocs)).source(Requests.INDEX_CONTENT_TYPE, "field", "value") - ); + IndexRequest indexRequest = new IndexRequest("test").id(Integer.toString(testDocs)) + .source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + processor.add(indexRequest); + indexRequest.decRef(); multiGetRequestBuilder.add("test", Integer.toString(testDocs)); } else { testReadOnlyDocs++; - processor.add( - new IndexRequest("test-ro").id(Integer.toString(testReadOnlyDocs)) - .source(Requests.INDEX_CONTENT_TYPE, "field", "value") - ); + IndexRequest indexRequest = new IndexRequest("test-ro").id(Integer.toString(testReadOnlyDocs)) + .source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + processor.add(indexRequest); + indexRequest.decRef(); } } } finally { @@ -223,10 +224,10 @@ public void testBulkProcessor2ConcurrentRequestsReadOnlyIndex() throws Exception private static MultiGetRequestBuilder indexDocs(Client client, BulkProcessor2 processor, int numDocs) throws Exception { MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet(); for (int i = 1; i <= numDocs; i++) { - processor.add( - new IndexRequest("test").id(Integer.toString(i)) - .source(Requests.INDEX_CONTENT_TYPE, "field", randomRealisticUnicodeOfLengthBetween(1, 30)) - ); + IndexRequest indexRequest = new IndexRequest("test").id(Integer.toString(i)) + .source(Requests.INDEX_CONTENT_TYPE, "field", randomRealisticUnicodeOfLengthBetween(1, 30)); + processor.add(indexRequest); + indexRequest.decRef(); multiGetRequestBuilder.add("test", Integer.toString(i)); } return multiGetRequestBuilder; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java index 6a2ab41fae5d6..bf1e603d87f76 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryBuilders; @@ -151,11 +152,10 @@ public void afterBulk(long executionId, BulkRequest request, Exception failure) private static void indexDocs(BulkProcessor2 processor, int numDocs) { for (int i = 1; i <= numDocs; i++) { - processor.add( - prepareIndex(INDEX_NAME).setId(Integer.toString(i)) - .setSource("field", randomRealisticUnicodeOfLengthBetween(1, 30)) - .request() - ); + IndexRequestBuilder indexRequestBuilder = prepareIndex(INDEX_NAME).setId(Integer.toString(i)) + .setSource("field", randomRealisticUnicodeOfLengthBetween(1, 30)); + processor.add(indexRequestBuilder.request()); + indexRequestBuilder.request().decRef(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java index 7809b7701808a..c7605a89d8478 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.bulk; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; @@ -31,10 +32,16 @@ public void testBulkProcessorAutoCreateRestrictions() { clusterAdmin().prepareHealth("willwork").setWaitForGreenStatus().get(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - bulkRequestBuilder.add(prepareIndex("willwork").setId("1").setSource("{\"foo\":1}", XContentType.JSON)); - bulkRequestBuilder.add(prepareIndex("wontwork").setId("2").setSource("{\"foo\":2}", XContentType.JSON)); - bulkRequestBuilder.add(prepareIndex("willwork").setId("3").setSource("{\"foo\":3}", XContentType.JSON)); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("willwork").setId("1").setSource("{\"foo\":1}", XContentType.JSON); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("wontwork").setId("2").setSource("{\"foo\":2}", XContentType.JSON); + IndexRequestBuilder indexRequestBuilder3 = prepareIndex("willwork").setId("3").setSource("{\"foo\":3}", XContentType.JSON); + bulkRequestBuilder.add(indexRequestBuilder1); + bulkRequestBuilder.add(indexRequestBuilder2); + bulkRequestBuilder.add(indexRequestBuilder3); BulkResponse br = bulkRequestBuilder.get(); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); + indexRequestBuilder3.request().decRef(); BulkItemResponse[] responses = br.getItems(); assertEquals(3, responses.length); assertFalse("Operation on existing index should succeed", responses[0].isFailed()); @@ -53,9 +60,9 @@ public void testBulkProcessorAutoCreateRestrictions() { public void testIndexWithDisabledAutoCreateIndex() { updateClusterSettings(Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), randomFrom("-*", "+.*"))); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - final BulkItemResponse itemResponse = bulkRequestBuilder.add(prepareIndex("test-index").setSource("foo", "bar")) - .get() - .getItems()[0]; + IndexRequestBuilder indexRequestBuilder = prepareIndex("test-index").setSource("foo", "bar"); + final BulkItemResponse itemResponse = bulkRequestBuilder.add(indexRequestBuilder).get().getItems()[0]; + indexRequestBuilder.request().decRef(); assertThat(itemResponse.getFailure().getCause(), instanceOf(IndexNotFoundException.class)); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorIT.java index dc0ccb803eb52..818e5ad4af61a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorIT.java @@ -256,16 +256,17 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception for (int i = 1; i <= numDocs; i++) { if (randomBoolean()) { testDocs++; - processor.add( - new IndexRequest("test").id(Integer.toString(testDocs)).source(Requests.INDEX_CONTENT_TYPE, "field", "value") - ); + IndexRequest indexRequest = new IndexRequest("test").id(Integer.toString(testDocs)) + .source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + processor.add(indexRequest); + indexRequest.decRef(); multiGetRequestBuilder.add("test", Integer.toString(testDocs)); } else { testReadOnlyDocs++; - processor.add( - new IndexRequest("test-ro").id(Integer.toString(testReadOnlyDocs)) - .source(Requests.INDEX_CONTENT_TYPE, "field", "value") - ); + IndexRequest indexRequest = new IndexRequest("test-ro").id(Integer.toString(testReadOnlyDocs)) + .source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + processor.add(indexRequest); + indexRequest.decRef(); } } } @@ -302,10 +303,10 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception private static MultiGetRequestBuilder indexDocs(Client client, BulkProcessor processor, int numDocs) throws Exception { MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet(); for (int i = 1; i <= numDocs; i++) { - processor.add( - new IndexRequest("test").id(Integer.toString(i)) - .source(Requests.INDEX_CONTENT_TYPE, "field", randomRealisticUnicodeOfLengthBetween(1, 30)) - ); + IndexRequest indexRequest = new IndexRequest("test").id(Integer.toString(i)) + .source(Requests.INDEX_CONTENT_TYPE, "field", randomRealisticUnicodeOfLengthBetween(1, 30)); + processor.add(indexRequest); + indexRequest.decRef(); multiGetRequestBuilder.add("test", Integer.toString(i)); } return multiGetRequestBuilder; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java index 8bd3a6cf02868..f6b157259aa59 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; @@ -156,11 +157,10 @@ private void assertRetriedCorrectly(CorrelatingBackoffPolicy internalPolicy, Obj private static void indexDocs(BulkProcessor processor, int numDocs) { for (int i = 1; i <= numDocs; i++) { - processor.add( - prepareIndex(INDEX_NAME).setId(Integer.toString(i)) - .setSource("field", randomRealisticUnicodeOfLengthBetween(1, 30)) - .request() - ); + IndexRequestBuilder indexRequestBuilder = prepareIndex(INDEX_NAME).setId(Integer.toString(i)) + .setSource("field", randomRealisticUnicodeOfLengthBetween(1, 30)); + processor.add(indexRequestBuilder.request()); + indexRequestBuilder.request().decRef(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkRejectionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkRejectionIT.java index 79033da64d806..0659ecef8b279 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkRejectionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkRejectionIT.java @@ -64,15 +64,17 @@ public void testBulkRejectionAfterDynamicMappingUpdate() throws Exception { ensureGreen(); try (BulkRequest request1 = new BulkRequest()) { for (int i = 0; i < 500; ++i) { - request1.add(new IndexRequest(index).source(Collections.singletonMap("key" + i, "value" + i))) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexRequest indexRequest = new IndexRequest(index).source(Collections.singletonMap("key" + i, "value" + i)); + request1.add(indexRequest).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequest.decRef(); } // Huge request to keep the write pool busy so that requests waiting on a mapping update in the other bulk request get rejected // by the write pool try (BulkRequest request2 = new BulkRequest()) { for (int i = 0; i < 10_000; ++i) { - request2.add(new IndexRequest(index).source(Collections.singletonMap("key", "valuea" + i))) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexRequest indexRequest = new IndexRequest(index).source(Collections.singletonMap("key", "valuea" + i)); + request2.add(indexRequest).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequest.decRef(); } final ActionFuture bulkFuture1 = client().bulk(request1); final ActionFuture bulkFuture2 = client().bulk(request2); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java index d68326122ce0f..50493daef9ea1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java @@ -8,12 +8,14 @@ package org.elasticsearch.action.bulk; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteRequest.OpType; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; @@ -22,6 +24,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.VersionType; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.plugins.Plugin; @@ -109,6 +112,11 @@ public void testBulkUpdateSimple() throws Exception { .add(prepareIndex(indexOrAlias()).setId("4").setSource("field", 4)) .add(prepareIndex(indexOrAlias()).setId("5").setSource("field", 5)) .get(); + for (DocWriteRequest docWriteRequest : bulkRequestBuilder.request().requests()) { + if (docWriteRequest instanceof IndexRequest indexRequest) { + indexRequest.decRef(); + } + } assertThat(bulkResponse.hasFailures(), equalTo(false)); assertThat(bulkResponse.getItems().length, equalTo(5)); @@ -131,6 +139,11 @@ public void testBulkUpdateSimple() throws Exception { .setDoc(jsonBuilder().startObject().field("field1", "test").endObject()) ) .get(); + for (DocWriteRequest request : bulkRequestBuilder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } assertThat(bulkResponse.hasFailures(), equalTo(false)); assertThat(bulkResponse.getItems().length, equalTo(3)); @@ -159,18 +172,21 @@ public void testBulkUpdateSimple() throws Exception { assertThat(getResponse.getVersion(), equalTo(2L)); assertThat(getResponse.getSource().get("field1").toString(), equalTo("test")); } - try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add( - client().prepareUpdate() - .setIndex(indexOrAlias()) - .setId("6") - .setScript(script) - .setUpsert(jsonBuilder().startObject().field("field", 0).endObject()) - ) - .add(client().prepareUpdate().setIndex(indexOrAlias()).setId("7").setScript(script)) - .add(client().prepareUpdate().setIndex(indexOrAlias()).setId("2").setScript(script)) + UpdateRequestBuilder updateRequestBuilder1 = client().prepareUpdate() + .setIndex(indexOrAlias()) + .setId("6") + .setScript(script) + .setUpsert(jsonBuilder().startObject().field("field", 0).endObject()); + UpdateRequestBuilder updateRequestBuilder2 = client().prepareUpdate().setIndex(indexOrAlias()).setId("7").setScript(script); + UpdateRequestBuilder updateRequestBuilder3 = client().prepareUpdate().setIndex(indexOrAlias()).setId("2").setScript(script); + BulkResponse bulkResponse = bulkRequestBuilder.add(updateRequestBuilder1) + .add(updateRequestBuilder2) + .add(updateRequestBuilder3) .get(); + updateRequestBuilder1.request().decRef(); + updateRequestBuilder2.request().decRef(); + updateRequestBuilder3.request().decRef(); assertThat(bulkResponse.hasFailures(), equalTo(true)); assertThat(bulkResponse.getItems().length, equalTo(3)); @@ -218,6 +234,11 @@ public void testBulkUpdateWithScriptedUpsertAndDynamicMappingUpdate() throws Exc .setUpsert("field", 1) ) .get(); + for (DocWriteRequest request : bulkRequestBuilder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } logger.info(bulkResponse.buildFailureMessage()); @@ -251,6 +272,11 @@ public void testBulkWithCAS() throws Exception { .add(prepareIndex("test").setId("2").setCreate(true).setSource("field", "1")) .add(prepareIndex("test").setId("1").setSource("field", "2")) .get(); + for (DocWriteRequest docWriteRequest : bulkRequestBuilder.request().requests()) { + if (docWriteRequest instanceof IndexRequest indexRequest) { + indexRequest.decRef(); + } + } assertEquals(DocWriteResponse.Result.CREATED, bulkResponse.getItems()[0].getResponse().getResult()); assertThat(bulkResponse.getItems()[0].getResponse().getSeqNo(), equalTo(0L)); @@ -269,6 +295,11 @@ public void testBulkWithCAS() throws Exception { client().prepareUpdate("test", "1").setIfSeqNo(2L).setIfPrimaryTerm(1).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "3") ) .get(); + for (DocWriteRequest request : bulkRequestBuilder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict")); assertThat(bulkResponse.getItems()[1].getResponse().getSeqNo(), equalTo(3L)); @@ -282,6 +313,11 @@ public void testBulkWithCAS() throws Exception { .add(prepareIndex("test").setId("e2").setSource("field", "1").setVersion(10).setVersionType(VersionType.EXTERNAL)) .add(prepareIndex("test").setId("e1").setSource("field", "2").setVersion(12).setVersionType(VersionType.EXTERNAL)) .get(); + for (DocWriteRequest docWriteRequest : bulkRequestBuilder.request().requests()) { + if (docWriteRequest instanceof IndexRequest indexRequest) { + indexRequest.decRef(); + } + } assertEquals(DocWriteResponse.Result.CREATED, bulkResponse.getItems()[0].getResponse().getResult()); assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(10L)); @@ -302,6 +338,11 @@ public void testBulkWithCAS() throws Exception { .setIfPrimaryTerm(1) ) .get(); + for (DocWriteRequest request : bulkRequestBuilder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict")); assertThat(bulkResponse.getItems()[1].getFailureMessage(), containsString("version conflict")); @@ -317,6 +358,11 @@ public void testBulkUpdateMalformedScripts() throws Exception { .add(prepareIndex("test").setId("2").setSource("field", 1)) .add(prepareIndex("test").setId("3").setSource("field", 1)) .get(); + for (DocWriteRequest docWriteRequest : bulkRequestBuilder.request().requests()) { + if (docWriteRequest instanceof IndexRequest indexRequest) { + indexRequest.decRef(); + } + } assertThat(bulkResponse.hasFailures(), equalTo(false)); assertThat(bulkResponse.getItems().length, equalTo(3)); @@ -359,6 +405,11 @@ public void testBulkUpdateMalformedScripts() throws Exception { ) ) .get(); + for (DocWriteRequest request : bulkRequestBuilder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } assertThat(bulkResponse.hasFailures(), equalTo(true)); assertThat(bulkResponse.getItems().length, equalTo(3)); @@ -399,6 +450,11 @@ public void testBulkUpdateLargerVolume() throws Exception { .setUpsert(jsonBuilder().startObject().field("counter", 1).endObject()) ); } + for (DocWriteRequest request : builder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } BulkResponse response = builder.get(); assertThat(response.hasFailures(), equalTo(false)); @@ -440,6 +496,11 @@ public void testBulkUpdateLargerVolume() throws Exception { } BulkResponse response = builder.get(); + for (DocWriteRequest request : builder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } assertThat(response.hasFailures(), equalTo(false)); assertThat(response.getItems().length, equalTo(numDocs)); for (int i = 0; i < numDocs; i++) { @@ -459,6 +520,11 @@ public void testBulkUpdateLargerVolume() throws Exception { builder.add(client().prepareUpdate().setIndex("test").setId(Integer.toString(i)).setScript(script)); } BulkResponse response = builder.get(); + for (DocWriteRequest request : builder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } assertThat(response.hasFailures(), equalTo(true)); assertThat(response.getItems().length, equalTo(numDocs)); for (int i = 0; i < numDocs; i++) { @@ -485,6 +551,11 @@ public void testBulkUpdateLargerVolume() throws Exception { ); } BulkResponse response = builder.get(); + for (DocWriteRequest request : builder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } assertThat(response.buildFailureMessage(), response.hasFailures(), equalTo(false)); assertThat(response.getItems().length, equalTo(numDocs)); for (int i = 0; i < numDocs; i++) { @@ -505,6 +576,11 @@ public void testBulkUpdateLargerVolume() throws Exception { ); } BulkResponse response = builder.get(); + for (DocWriteRequest request : builder.request().requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } assertThat("expected no failures but got: " + response.buildFailureMessage(), response.hasFailures(), equalTo(false)); assertThat(response.getItems().length, equalTo(numDocs)); for (int i = 0; i < numDocs; i++) { @@ -536,7 +612,9 @@ public void testBulkIndexingWhileInitializing() throws Exception { for (int i = 0; i < numDocs;) { try (BulkRequestBuilder builder = client().prepareBulk()) { for (int j = 0; j < bulk && i < numDocs; j++, i++) { - builder.add(prepareIndex("test").setId(Integer.toString(i)).setSource("val", i)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)).setSource("val", i); + builder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } logger.info("bulk indexing {}-{}", i - bulk, i - 1); BulkResponse response = builder.get(); @@ -567,13 +645,13 @@ public void testFailingVersionedUpdatedOnBulk() throws Exception { return; } try (BulkRequestBuilder requestBuilder = client().prepareBulk()) { - requestBuilder.add( - client().prepareUpdate("test", "1") - .setIfSeqNo(0L) - .setIfPrimaryTerm(1) - .setDoc(Requests.INDEX_CONTENT_TYPE, "field", threadID) - ); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", "1") + .setIfSeqNo(0L) + .setIfPrimaryTerm(1) + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", threadID); + requestBuilder.add(updateRequestBuilder); responses[threadID] = requestBuilder.get(); + assert updateRequestBuilder.request().decRef() == false : "decref failed"; } }); @@ -613,7 +691,9 @@ public void testThatInvalidIndexNamesShouldNotBreakCompleteBulkRequest() { } else { name = "test"; } - builder.add(prepareIndex(name).setId("1").setSource("field", 1)); + IndexRequestBuilder indexRequestBuilder = prepareIndex(name).setId("1").setSource("field", 1); + builder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } BulkResponse bulkResponse = builder.get(); assertThat(bulkResponse.hasFailures(), is(expectFailure)); @@ -635,6 +715,11 @@ public void testThatFailedUpdateRequestReturnsCorrectType() throws Exception { .add(new IndexRequest("test").id("6").source("{ \"title\" : \"Great Title of doc 6\" }", XContentType.JSON)) .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); + for (DocWriteRequest docWriteRequest : bulkRequestBuilder.request().requests()) { + if (docWriteRequest instanceof IndexRequest indexRequest) { + indexRequest.decRef(); + } + } assertNoFailures(indexBulkItemResponse); } @@ -648,6 +733,11 @@ public void testThatFailedUpdateRequestReturnsCorrectType() throws Exception { .add(new DeleteRequest("test", "5")) .add(new DeleteRequest("test", "6")) .get(); + for (DocWriteRequest docWriteRequest : bulkRequestBuilder.request().requests()) { + if (docWriteRequest instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } assertThat("Expected a failure due to month 13: ", bulkItemResponse.hasFailures(), CoreMatchers.is(true)); assertThat(bulkItemResponse.getItems().length, is(6)); @@ -674,6 +764,11 @@ public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception { .add(new UpdateRequest("bulkindex2", "2").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar")) .add(new DeleteRequest("bulkindex2", "3")) .setRefreshPolicy(RefreshPolicy.IMMEDIATE); + for (DocWriteRequest docWriteRequest : bulkRequest.requests()) { + if (docWriteRequest instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } client().bulk(bulkRequest).get(); assertHitCount(prepareSearch("bulkindex*"), 3); @@ -688,6 +783,11 @@ public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception { .add(new UpdateRequest("bulkindex2", "2").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar")) .add(new DeleteRequest("bulkindex2", "3")) .setRefreshPolicy(RefreshPolicy.IMMEDIATE); + for (DocWriteRequest docWriteRequest : bulkRequest2.requests()) { + if (docWriteRequest instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } BulkResponse bulkResponse = client().bulk(bulkRequest2).get(); assertThat(bulkResponse.hasFailures(), is(true)); @@ -699,15 +799,19 @@ public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception { public void testFailedRequestsOnClosedIndex() throws Exception { createIndex("bulkindex1"); - prepareIndex("bulkindex1").setId("1").setSource("text", "test").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("bulkindex1").setId("1").setSource("text", "test"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertBusy(() -> assertAcked(indicesAdmin().prepareClose("bulkindex1"))); try (BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE)) { - bulkRequest.add(new IndexRequest("bulkindex1").id("1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1")) - .add(new UpdateRequest("bulkindex1", "1").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar")) - .add(new DeleteRequest("bulkindex1", "1")); + IndexRequest indexRequest = new IndexRequest("bulkindex1").id("1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1"); + UpdateRequest updateRequest = new UpdateRequest("bulkindex1", "1").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); + bulkRequest.add(indexRequest).add(updateRequest).add(new DeleteRequest("bulkindex1", "1")); BulkResponse bulkResponse = client().bulk(bulkRequest).get(); + indexRequest.decRef(); + updateRequest.decRef(); assertThat(bulkResponse.hasFailures(), is(true)); BulkItemResponse[] responseItems = bulkResponse.getItems(); assertThat(responseItems.length, is(3)); @@ -723,12 +827,18 @@ public void testFailedRequestsOnClosedIndex() throws Exception { // issue 9821 public void testInvalidIndexNamesCorrectOpType() { try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add( - prepareIndex("INVALID.NAME").setId("1").setSource(Requests.INDEX_CONTENT_TYPE, "field", 1) - ) - .add(client().prepareUpdate().setIndex("INVALID.NAME").setId("1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", randomInt())) + IndexRequestBuilder indexRequestBuilder = prepareIndex("INVALID.NAME").setId("1") + .setSource(Requests.INDEX_CONTENT_TYPE, "field", 1); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate() + .setIndex("INVALID.NAME") + .setId("1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", randomInt()); + BulkResponse bulkResponse = bulkRequestBuilder.add(indexRequestBuilder) + .add(updateRequestBuilder) .add(client().prepareDelete().setIndex("INVALID.NAME").setId("1")) .get(); + indexRequestBuilder.request().decRef(); + updateRequestBuilder.request().decRef(); assertThat(bulkResponse.getItems().length, is(3)); assertThat(bulkResponse.getItems()[0].getOpType(), is(OpType.INDEX)); assertThat(bulkResponse.getItems()[1].getOpType(), is(OpType.UPDATE)); @@ -744,12 +854,14 @@ public void testNoopUpdate() { DocWriteResponse doc = index(indexName, "1", Map.of("user", "xyz")); assertThat(doc.getShardInfo().getSuccessful(), equalTo(2)); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - final BulkResponse bulkResponse = bulkRequestBuilder.add( - new UpdateRequest().index(indexName).id("1").detectNoop(true).doc("user", "xyz") - ) // noop update - .add(new UpdateRequest().index(indexName).id("2").docAsUpsert(false).doc("f", "v")) // not_found update + UpdateRequest updateRequest1 = new UpdateRequest().index(indexName).id("1").detectNoop(true).doc("user", "xyz"); + UpdateRequest updateRequest2 = new UpdateRequest().index(indexName).id("2").docAsUpsert(false).doc("f", "v"); + final BulkResponse bulkResponse = bulkRequestBuilder.add(updateRequest1) // noop update + .add(updateRequest2) // not_found update .add(new DeleteRequest().index(indexName).id("2")) // not_found delete .get(); + updateRequest1.decRef(); + updateRequest2.decRef(); assertThat(bulkResponse.getItems(), arrayWithSize(3)); final BulkItemResponse noopUpdate = bulkResponse.getItems()[0]; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java index 29a5e491dd3fd..4e3f56b482156 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java @@ -38,6 +38,9 @@ public void testIndexWithWriteDelayEnabled() throws Exception { builders.add(prepareIndex("test").setSource("field", "value_" + j)); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } logger.info("verifying indexed content"); int numOfChecks = randomIntBetween(8, 12); for (int j = 0; j < numOfChecks; j++) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java index 4bd97f772e4c3..6b177fe59d8e3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.search; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; @@ -67,7 +68,7 @@ public List> getQueries() { void indexDocs(Client client, String index, int numDocs) { for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); - client.prepareIndex(index).setId(id).setSource("value", i).get(); + indexDoc(client, index, id, "value", i); } client.admin().indices().prepareRefresh(index).get(); } @@ -95,11 +96,11 @@ public void testBasic() { String pitId = openPointInTime(indices.toArray(new String[0]), TimeValue.timeValueMinutes(2)); try { if (randomBoolean()) { - localClient.prepareIndex("local_test").setId("local_new").setSource().get(); + indexDocEmptySource(localClient, "local_test", "local_new"); localClient.admin().indices().prepareRefresh().get(); } if (randomBoolean()) { - remoteClient.prepareIndex("remote_test").setId("remote_new").setSource().get(); + indexDocEmptySource(remoteClient, "remote_test", "remote_new"); remoteClient.admin().indices().prepareRefresh().get(); } assertNoFailuresAndResponse( @@ -143,15 +144,15 @@ public void testOpenPITWithIndexFilter() { assertAcked( localClient.admin().indices().prepareCreate("local_test").setSettings(Settings.builder().put("index.number_of_shards", 3)) ); - localClient.prepareIndex("local_test").setId("1").setSource("value", "1", "@timestamp", "2024-03-01").get(); - localClient.prepareIndex("local_test").setId("2").setSource("value", "2", "@timestamp", "2023-12-01").get(); + indexDoc(localClient, "local_test", "1", "value", "1", "@timestamp", "2024-03-01"); + indexDoc(localClient, "local_test", "2", "value", "2", "@timestamp", "2023-12-01"); localClient.admin().indices().prepareRefresh("local_test").get(); assertAcked( remoteClient.admin().indices().prepareCreate("remote_test").setSettings(Settings.builder().put("index.number_of_shards", 3)) ); - remoteClient.prepareIndex("remote_test").setId("1").setSource("value", "1", "@timestamp", "2024-01-01").get(); - remoteClient.prepareIndex("remote_test").setId("2").setSource("value", "2", "@timestamp", "2023-12-01").get(); + indexDoc(remoteClient, "remote_test", "1", "value", "1", "@timestamp", "2024-01-01"); + indexDoc(remoteClient, "remote_test", "2", "value", "2", "@timestamp", "2023-12-01"); remoteClient.admin().indices().prepareRefresh("remote_test").get(); List indices = new ArrayList<>(); @@ -165,11 +166,11 @@ public void testOpenPITWithIndexFilter() { String pitId = response.getPointInTimeId(); if (randomBoolean()) { - localClient.prepareIndex("local_test").setId("local_new").setSource().get(); + indexDocEmptySource(localClient, "local_test", "local_new"); localClient.admin().indices().prepareRefresh().get(); } if (randomBoolean()) { - remoteClient.prepareIndex("remote_test").setId("remote_new").setSource().get(); + indexDocEmptySource(remoteClient, "remote_test", "remote_new"); remoteClient.admin().indices().prepareRefresh().get(); } @@ -255,11 +256,11 @@ public void testFailuresOnOneShardsWithPointInTime() throws ExecutionException, String pitId = openPointInTime(indices.toArray(new String[0]), TimeValue.timeValueMinutes(2)); try { if (randomBoolean()) { - localClient.prepareIndex("local_test").setId("local_new").setSource().get(); + indexDocEmptySource(localClient, "local_test", "local_new"); localClient.admin().indices().prepareRefresh().get(); } if (randomBoolean()) { - remoteClient.prepareIndex("remote_test").setId("remote_new").setSource().get(); + indexDocEmptySource(remoteClient, "remote_test", "remote_new"); remoteClient.admin().indices().prepareRefresh().get(); } // shardId 0 means to throw the Exception only on shard 0; all others should work @@ -317,4 +318,22 @@ private String openPointInTime(String[] indices, TimeValue keepAlive) { private void closePointInTime(String readerId) { client().execute(TransportClosePointInTimeAction.TYPE, new ClosePointInTimeRequest(readerId)).actionGet(); } + + private void indexDoc(Client client, String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private void indexDocEmptySource(Client client, String index, String id) { + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource().get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/LookupRuntimeFieldIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/LookupRuntimeFieldIT.java index bac52a0d4ad9b..a1bb52345c0a6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/LookupRuntimeFieldIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/LookupRuntimeFieldIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -43,7 +44,10 @@ public void populateIndex() throws Exception { Map.of("author", "jack", "first_name", "Jack", "last_name", "Austin", "joined", "1999-11-03") ); for (Map author : authors) { - prepareIndex("authors").setSource(author).setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("authors").setSource(author) + .setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } indicesAdmin().prepareRefresh("authors").get(); @@ -51,10 +55,12 @@ public void populateIndex() throws Exception { .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5))) .get(); try (BulkRequestBuilder bulkBuilder = client().prepareBulk("publishers")) { - bulkBuilder.add(new IndexRequest().id("p1").source("name", "The first publisher", "city", List.of("Montreal", "Vancouver"))) - .add(new IndexRequest().id("p2").source("name", "The second publisher", "city", "Toronto")) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + IndexRequest indexRequest1 = new IndexRequest().id("p1") + .source("name", "The first publisher", "city", List.of("Montreal", "Vancouver")); + IndexRequest indexRequest2 = new IndexRequest().id("p2").source("name", "The second publisher", "city", "Toronto"); + bulkBuilder.add(indexRequest1).add(indexRequest2).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + indexRequest1.decRef(); + indexRequest2.decRef(); } indicesAdmin().prepareCreate("books").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)).setMapping(""" { @@ -130,7 +136,10 @@ public void populateIndex() throws Exception { Map.of("title", "the fifth book", "genre", "science", "author_id", "mike", "publisher_id", "p2", "published_date", "2021-06-30") ); for (Map book : books) { - prepareIndex("books").setSource(book).setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("books").setSource(book) + .setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } indicesAdmin().prepareRefresh("books").get(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java index 0c1930c0cf925..e573bae146e09 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -79,7 +80,7 @@ public void testBasic() { int numDocs = randomIntBetween(10, 50); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); - prepareIndex("test").setId(id).setSource("value", i).get(); + indexDocs("test", id, "value", i); } refresh("test"); String pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2)); @@ -125,7 +126,7 @@ public void testMultipleIndices() { for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); String index = "index-" + randomIntBetween(1, numIndices); - prepareIndex(index).setId(id).setSource("value", i).get(); + indexDocs(index, id, "value", i); } refresh(); String pitId = openPointInTime(new String[] { "*" }, TimeValue.timeValueMinutes(2)); @@ -138,7 +139,7 @@ public void testMultipleIndices() { for (int i = 0; i < moreDocs; i++) { String id = "more-" + i; String index = "index-" + randomIntBetween(1, numIndices); - prepareIndex(index).setId(id).setSource("value", i).get(); + indexDocs(index, id, "value", i); } refresh(); }); @@ -160,7 +161,7 @@ public void testIndexFilter() { createIndex(index); for (int j = 1; j <= numDocs; j++) { String id = Integer.toString(j); - client().prepareIndex(index).setId(id).setSource("@timestamp", "2023-0" + i + "-0" + j).get(); + indexDocs(index, id, "@timestamp", "2023-0" + i + "-0" + j); } } refresh(); @@ -207,7 +208,7 @@ public void testRelocation() throws Exception { ensureGreen("test"); int numDocs = randomIntBetween(10, 50); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("value", i).get(); + indexDocs("test", Integer.toString(i), "value", i); } refresh(); String pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2)); @@ -228,7 +229,7 @@ public void testRelocation() throws Exception { if (randomBoolean()) { int moreDocs = randomIntBetween(10, 50); for (int i = 0; i < moreDocs; i++) { - prepareIndex("test").setId("more-" + i).setSource("value", i).get(); + indexDocs("test", "more-" + i, "value", i); } refresh(); } @@ -259,7 +260,7 @@ public void testPointInTimeNotFound() throws Exception { int index1 = randomIntBetween(10, 50); for (int i = 0; i < index1; i++) { String id = Integer.toString(i); - prepareIndex("index").setId(id).setSource("value", i).get(); + indexDocs("index", id, "value", i); } refresh(); String pit = openPointInTime(new String[] { "index" }, TimeValue.timeValueSeconds(5)); @@ -294,13 +295,13 @@ public void testIndexNotFound() { int index1 = randomIntBetween(10, 50); for (int i = 0; i < index1; i++) { String id = Integer.toString(i); - prepareIndex("index-1").setId(id).setSource("value", i).get(); + indexDocs("index-1", id, "value", i); } int index2 = randomIntBetween(10, 50); for (int i = 0; i < index2; i++) { String id = Integer.toString(i); - prepareIndex("index-2").setId(id).setSource("value", i).get(); + indexDocs("index-2", id, "value", i); } refresh(); String pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueMinutes(2)); @@ -355,7 +356,7 @@ public void testCanMatch() throws Exception { } } } - prepareIndex("test").setId("1").setSource("created_date", "2020-01-01").get(); + indexDocs("test", "1", "created_date", "2020-01-01"); assertResponse( prepareSearch().setQuery(new RangeQueryBuilder("created_date").gte("2020-01-02").lte("2020-01-03")) .setSearchType(SearchType.QUERY_THEN_FETCH) @@ -406,11 +407,11 @@ public void testPartialResults() throws Exception { int numDocs1 = randomIntBetween(10, 50); for (int i = 0; i < numDocs1; i++) { - prepareIndex(randomFrom("test-1")).setId(Integer.toString(i)).setSource("value", i).get(); + indexDocs(randomFrom("test-1"), Integer.toString(i), "value", i); } int numDocs2 = randomIntBetween(10, 50); for (int i = 0; i < numDocs2; i++) { - prepareIndex(randomFrom("test-2")).setId(Integer.toString(i)).setSource("value", i).get(); + indexDocs(randomFrom("test-2"), Integer.toString(i), "value", i); } refresh(); String pitId = openPointInTime(new String[] { "test-*" }, TimeValue.timeValueMinutes(2)); @@ -440,7 +441,7 @@ public void testPITTiebreak() throws Exception { createIndex(index, Settings.builder().put("index.number_of_shards", 1).build()); int numDocs = randomIntBetween(3, 20); for (int j = 0; j < numDocs; j++) { - prepareIndex(index).setSource("value", randomIntBetween(0, 2)).get(); + indexDocs(index, null, "value", randomIntBetween(0, 2)); expectedNumDocs++; } } @@ -594,4 +595,17 @@ private String openPointInTime(String[] indices, TimeValue keepAlive) { private void closePointInTime(String readerId) { client().execute(TransportClosePointInTimeAction.TYPE, new ClosePointInTimeRequest(readerId)).actionGet(); } + + private void indexDocs(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + if (id != null) { + indexRequestBuilder.setId(id); + } + indexRequestBuilder.setSource(source); + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java index 227a3b8612331..f7ebf4261bab9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java @@ -10,6 +10,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.core.Strings; @@ -189,7 +190,9 @@ private static List createRandomIndices(Client client) { for (int i = 0; i < numIndices; i++) { String indexName = Strings.format("index-%03d", i); assertAcked(client.admin().indices().prepareCreate(indexName).get()); - client.prepareIndex(indexName).setSource("number", i, "foo", "bar").get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName).setSource("number", i, "foo", "bar"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } client.admin().indices().prepareRefresh("index-*").get(); ClusterSearchShardsResponse resp = client.admin().cluster().prepareSearchShards("index-*").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchShardsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchShardsIT.java index 7da015052fe82..cb34fcd289a42 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchShardsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchShardsIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.search; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -49,7 +50,9 @@ public void testBasic() { ); int numDocs = randomIntBetween(1, 10); for (int j = 0; j < numDocs; j++) { - prepareIndex(index).setSource("value", i).setId(Integer.toString(i)).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setSource("value", i).setId(Integer.toString(i)); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } indicesAdmin().prepareRefresh(index).get(); } @@ -117,7 +120,9 @@ public void testRandom() throws ExecutionException, InterruptedException { ); int numDocs = randomIntBetween(10, 1000); for (int j = 0; j < numDocs; j++) { - prepareIndex(index).setSource("value", i).setId(Integer.toString(i)).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setSource("value", i).setId(Integer.toString(i)); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } indicesAdmin().prepareRefresh(index).get(); } @@ -172,7 +177,9 @@ public void testNoCanMatchWithoutQuery() { totalShards += numShards; int numDocs = randomIntBetween(10, 100); for (int j = 0; j < numDocs; j++) { - prepareIndex(index).setSource("value", i).setId(Integer.toString(i)).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setSource("value", i).setId(Integer.toString(i)); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } indicesAdmin().prepareRefresh(index).get(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java index dd71b82c106a8..cc222de8a0d38 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; @@ -129,6 +130,7 @@ public void testLocalClusterAlias() throws ExecutionException, InterruptedExcept indexRequest.source("field", "value"); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); TaskId parentTaskId = new TaskId("node", randomNonNegativeLong()); @@ -180,6 +182,7 @@ public void testAbsoluteStartMillis() throws ExecutionException, InterruptedExce indexRequest.source("date", "1970-01-01"); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); } { @@ -188,6 +191,7 @@ public void testAbsoluteStartMillis() throws ExecutionException, InterruptedExce indexRequest.source("date", "1982-01-01"); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); } { @@ -254,6 +258,7 @@ public void testFinalReduce() throws ExecutionException, InterruptedException { indexRequest.id("1"); indexRequest.source("price", 10); DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); } { @@ -261,6 +266,7 @@ public void testFinalReduce() throws ExecutionException, InterruptedException { indexRequest.id("2"); indexRequest.source("price", 100); DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); } indicesAdmin().prepareRefresh("test").get(); @@ -424,9 +430,15 @@ public void testSearchIdle() throws Exception { } } }); - prepareIndex("test").setId("1").setSource("created_date", "2020-01-01").get(); - prepareIndex("test").setId("2").setSource("created_date", "2020-01-02").get(); - prepareIndex("test").setId("3").setSource("created_date", "2020-01-03").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("created_date", "2020-01-01"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex("test").setId("2").setSource("created_date", "2020-01-02"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex("test").setId("3").setSource("created_date", "2020-01-03"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertBusy( () -> assertResponse( prepareSearch("test").setQuery(new RangeQueryBuilder("created_date").gte("2020-01-02").lte("2020-01-03")) @@ -552,7 +564,9 @@ private void indexSomeDocs(String indexName, int numberOfShards, int numberOfDoc createIndex(indexName, Settings.builder().put("index.number_of_shards", numberOfShards).build()); for (int i = 0; i < numberOfDocs; i++) { - DocWriteResponse indexResponse = prepareIndex(indexName).setSource("number", randomInt()).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setSource("number", randomInt()); + DocWriteResponse indexResponse = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); } indicesAdmin().prepareRefresh(indexName).get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/AutoCreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/AutoCreateIndexIT.java index e4ef0fa7f2d4f..bb7ba2eba65de 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/AutoCreateIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/AutoCreateIndexIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.test.ESIntegTestCase; @@ -39,7 +40,8 @@ public void testBatchingWithDeprecationWarnings() throws Exception { final var countDownLatch = new CountDownLatch(2); final var client = client(); - client.prepareIndex("no-dot").setSource("{}", XContentType.JSON).execute(new ActionListener<>() { + IndexRequestBuilder indexRequestBuilder1 = client.prepareIndex("no-dot").setSource("{}", XContentType.JSON); + indexRequestBuilder1.execute(new ActionListener<>() { @Override public void onResponse(DocWriteResponse indexResponse) { try { @@ -66,7 +68,8 @@ public void onFailure(Exception e) { } }); - client.prepareIndex(".has-dot").setSource("{}", XContentType.JSON).execute(new ActionListener<>() { + IndexRequestBuilder indexRequestBuilder2 = client.prepareIndex(".has-dot").setSource("{}", XContentType.JSON); + indexRequestBuilder2.execute(new ActionListener<>() { @Override public void onResponse(DocWriteResponse indexResponse) { try { @@ -101,5 +104,7 @@ public void onFailure(Exception e) { barrier.await(10, TimeUnit.SECONDS); assertTrue(countDownLatch.await(10, TimeUnit.SECONDS)); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java index 6737d02434c0f..5a87bc24f5065 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.elasticsearch.core.Strings; @@ -34,13 +35,15 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { assertAcked(createIndexResponse); // indexing, by default, will work (waiting for one shard copy only) - prepareIndex("test").setId("1").setSource(source("1", "test"), XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource(source("1", "test"), XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex("test").setId("1") + .setSource(source("1", "test"), XContentType.JSON) + .setWaitForActiveShards(2) // wait for 2 active shard copies + .setTimeout(timeValueMillis(100)); try { - prepareIndex("test").setId("1") - .setSource(source("1", "test"), XContentType.JSON) - .setWaitForActiveShards(2) // wait for 2 active shard copies - .setTimeout(timeValueMillis(100)) - .get(); + indexRequestBuilder.get(); fail("can't index, does not enough active shard copies"); } catch (UnavailableShardsException e) { assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); @@ -49,6 +52,8 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { startsWith("[test][0] Not enough active copies to meet shard count of [2] (have 1, needed 2). Timeout: [100ms], request:") ); // but really, all is well + } finally { + indexRequestBuilder.request().decRef(); } allowNodes("test", 2); @@ -63,18 +68,19 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); // this should work, since we now have two - prepareIndex("test").setId("1") + indexRequestBuilder = prepareIndex("test").setId("1") .setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(2) - .setTimeout(timeValueSeconds(1)) - .get(); + .setTimeout(timeValueSeconds(1)); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); try { - prepareIndex("test").setId("1") + indexRequestBuilder = prepareIndex("test").setId("1") .setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(ActiveShardCount.ALL) - .setTimeout(timeValueMillis(100)) - .get(); + .setTimeout(timeValueMillis(100)); + indexRequestBuilder.get(); fail("can't index, not enough active shard copies"); } catch (UnavailableShardsException e) { assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); @@ -87,6 +93,8 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { ) ); // but really, all is well + } finally { + indexRequestBuilder.request().decRef(); } allowNodes("test", 3); @@ -100,11 +108,12 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN)); // this should work, since we now have all shards started - prepareIndex("test").setId("1") + indexRequestBuilder = prepareIndex("test").setId("1") .setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(ActiveShardCount.ALL) - .setTimeout(timeValueSeconds(1)) - .get(); + .setTimeout(timeValueSeconds(1)); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } private String source(String id, String nameValue) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java index 1887e37cbbf47..05be71665fb42 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.support.master; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -69,8 +70,13 @@ public void run() { } for (int i = 0; i < 10; i++) { // index data with mapping changes - DocWriteResponse response = client(dataNode).prepareIndex("myindex").setSource("field_" + i, "val").get(); - assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); + IndexRequestBuilder indexRequestBuilder = client(dataNode).prepareIndex("myindex").setSource("field_" + i, "val"); + try { + DocWriteResponse response = indexRequestBuilder.get(); + assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); + } finally { + indexRequestBuilder.request().decRef(); + } } } }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java index 9661f4ebb966d..8b442c0d6dfcf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java @@ -69,7 +69,7 @@ public void testNoSuchDoc() throws Exception { .endObject(); assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setMapping(mapping)); - prepareIndex("test").setId("666").setSource("field", "foo bar").get(); + indexDoc("test", "666", "field", "foo bar"); refresh(); for (int i = 0; i < 20; i++) { ActionFuture termVector = client().termVectors(new TermVectorsRequest(indexOrAlias(), "" + i)); @@ -96,7 +96,7 @@ public void testExistingFieldWithNoTermVectorsNoNPE() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setMapping(mapping)); // when indexing a field that simply has a question mark, the term vectors will be null - prepareIndex("test").setId("0").setSource("existingfield", "?").get(); + indexDoc("test", "0", "existingfield", "?"); refresh(); ActionFuture termVector = client().termVectors( new TermVectorsRequest(indexOrAlias(), "0").selectedFields(new String[] { "existingfield" }) @@ -124,7 +124,7 @@ public void testExistingFieldButNotInDocNPE() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setMapping(mapping)); // when indexing a field that simply has a question mark, the term vectors will be null - prepareIndex("test").setId("0").setSource("anotherexistingfield", 1).get(); + indexDoc("test", "0", "anotherexistingfield", 1); refresh(); ActionFuture termVectors = client().termVectors( new TermVectorsRequest(indexOrAlias(), "0").selectedFields(randomBoolean() ? new String[] { "existingfield" } : null) @@ -165,6 +165,9 @@ public void testNotIndexedField() throws Exception { indexBuilders.add(prepareIndex("test").setId(String.valueOf(i)).setSource("field" + i, i)); } indexRandom(true, indexBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexBuilders) { + indexRequestBuilder.request().decRef(); + } for (int i = 0; i < 4; i++) { TermVectorsResponse resp = client().prepareTermVectors(indexOrAlias(), String.valueOf(i)).setSelectedFields("field" + i).get(); @@ -204,15 +207,15 @@ public void testSimpleTermVectors() throws IOException { ) ); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)) - .setSource( - jsonBuilder().startObject() - .field("field", "the quick brown fox jumps over the lazy dog") - // 0the3 4quick9 10brown15 16fox19 20jumps25 26over30 - // 31the34 35lazy39 40dog43 - .endObject() - ) - .get(); + index( + "test", + Integer.toString(i), + jsonBuilder().startObject() + .field("field", "the quick brown fox jumps over the lazy dog") + // 0the3 4quick9 10brown15 16fox19 20jumps25 26over30 + // 31the34 35lazy39 40dog43 + .endObject() + ); refresh(); } for (int i = 0; i < 10; i++) { @@ -305,15 +308,15 @@ public void testRandomSingleTermVectors() throws IOException { ) ); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)) - .setSource( - jsonBuilder().startObject() - .field("field", "the quick brown fox jumps over the lazy dog") - // 0the3 4quick9 10brown15 16fox19 20jumps25 26over30 - // 31the34 35lazy39 40dog43 - .endObject() - ) - .get(); + index( + "test", + Integer.toString(i), + jsonBuilder().startObject() + .field("field", "the quick brown fox jumps over the lazy dog") + // 0the3 4quick9 10brown15 16fox19 20jumps25 26over30 + // 31the34 35lazy39 40dog43 + .endObject() + ); refresh(); } String[] values = { "brown", "dog", "fox", "jumps", "lazy", "over", "quick", "the" }; @@ -465,7 +468,7 @@ public void testSimpleTermVectorsWithGenerate() throws IOException { ensureGreen(); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource(source).get(); + index("test", Integer.toString(i), source); refresh(); } @@ -549,6 +552,9 @@ public void testDuelWithAndWithoutTermVectors() throws IOException, ExecutionExc } } indexRandom(true, indexBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexBuilders) { + indexRequestBuilder.request().decRef(); + } // request tvs and compare from each index for (int id = 0; id < content.length; id++) { @@ -622,7 +628,7 @@ public void testSimpleWildCards() throws IOException { assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setMapping(mapping)); ensureGreen(); - prepareIndex("test").setId("0").setSource(source).get(); + index("test", "0", source); refresh(); TermVectorsResponse response = client().prepareTermVectors(indexOrAlias(), "0").setSelectedFields("field*").get(); @@ -649,6 +655,9 @@ public void testArtificialVsExisting() throws ExecutionException, InterruptedExc indexBuilders.add(prepareIndex("test").setId(String.valueOf(i)).setSource("field1", content[i])); } indexRandom(true, indexBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexBuilders) { + indexRequestBuilder.request().decRef(); + } for (int i = 0; i < content.length; i++) { // request tvs from existing document @@ -734,7 +743,7 @@ public void testPerFieldAnalyzer() throws IOException { ensureGreen(); // index a single document with prepared source - prepareIndex("test").setId("0").setSource(source).get(); + index("test", "0", source); refresh(); // create random per_field_analyzer and selected fields @@ -808,7 +817,7 @@ public void testTermVectorsWithVersion() { assertThat(response.isExists(), equalTo(false)); logger.info("--> index doc 1"); - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").get(); + indexDoc("test", "1", "field1", "value1", "field2", "value2"); // From translog: @@ -854,7 +863,7 @@ public void testTermVectorsWithVersion() { } logger.info("--> index doc 1 again, so increasing the version"); - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").get(); + indexDoc("test", "1", "field1", "value1", "field2", "value2"); // From translog: @@ -917,7 +926,9 @@ public void testFilterLength() throws ExecutionException, InterruptedException, } tags.add(tag); } - indexRandom(true, prepareIndex("test").setId("1").setSource("tags", tags)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("tags", tags); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); logger.info("Checking best tags by longest to shortest size ..."); TermVectorsRequest.FilterSettings filterSettings = new TermVectorsRequest.FilterSettings(); @@ -953,7 +964,9 @@ public void testFilterTermFreq() throws ExecutionException, InterruptedException } uniqueTags.add(tag); } - indexRandom(true, prepareIndex("test").setId("1").setSource("tags", tags)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("tags", tags); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); logger.info("Checking best tags by highest to lowest term freq ..."); TermVectorsRequest.FilterSettings filterSettings = new TermVectorsRequest.FilterSettings(); @@ -987,6 +1000,9 @@ public void testFilterDocFreq() throws ExecutionException, InterruptedException, builders.add(prepareIndex("test").setId(i + "").setSource("tags", tags)); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } logger.info("Checking best terms by highest to lowest idf ..."); TermVectorsRequest.FilterSettings filterSettings = new TermVectorsRequest.FilterSettings(); @@ -1010,7 +1026,9 @@ public void testArtificialDocWithPreference() throws InterruptedException, IOExc ensureGreen(); // index document - indexRandom(true, prepareIndex("test").setId("1").setSource("field1", "random permutation")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field1", "random permutation"); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); // Get search shards ClusterSearchShardsResponse searchShardsResponse = clusterAdmin().prepareSearchShards("test").get(); @@ -1046,7 +1064,9 @@ public void testTermVectorsWithIgnoredField() throws IOException, InterruptedExc ensureGreen(); // add a doc with a bad long field - indexRandom(true, prepareIndex("index").setId("1").setSource("{\"field\":\"foo\"}", XContentType.JSON)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("{\"field\":\"foo\"}", XContentType.JSON); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); // do a tv request for all fields, _ignored should be returned TermVectorsResponse resp = client().prepareTermVectors("index", "1").setSelectedFields("*").get(); @@ -1088,6 +1108,9 @@ public void testWithKeywordAndNormalizer() throws IOException, ExecutionExceptio } } indexRandom(true, indexBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexBuilders) { + indexRequestBuilder.request().decRef(); + } // request tvs and compare from each index for (int id = 0; id < content.length; id++) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java index 9beffa7f964d9..a3d25bce7a635 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java @@ -84,7 +84,7 @@ public void testMultiTermVectorsWithVersion() throws Exception { assertThat(response.getResponses()[0].getResponse().isExists(), equalTo(false)); for (int i = 0; i < 3; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } // Version from translog @@ -133,7 +133,7 @@ public void testMultiTermVectorsWithVersion() throws Exception { assertThat(response.getResponses()[2].getFailure().getCause().getCause(), instanceOf(VersionConflictEngineException.class)); for (int i = 0; i < 3; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } // Version from translog diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java index 2f10711db7371..a258743f50478 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.cluster.ClusterState; @@ -86,30 +87,34 @@ public void testAliases() throws Exception { assertAcked(indicesAdmin().prepareAliases().addAlias("test", "alias1", false)); }); - logger.info("--> indexing against [alias1], should fail now"); - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - client().index(new IndexRequest("alias1").id("1").source(source("2", "test"), XContentType.JSON)) - ); - assertThat( - exception.getMessage(), - equalTo( - "no write index is defined for alias [alias1]." - + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" - + " indices without one being designated as a write index" - ) - ); + { + logger.info("--> indexing against [alias1], should fail now"); + IndexRequest indexRequest = new IndexRequest("alias1").id("1").source(source("2", "test"), XContentType.JSON); + try { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, client().index(indexRequest)); + assertThat( + exception.getMessage(), + equalTo( + "no write index is defined for alias [alias1]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index" + ) + ); + } finally { + indexRequest.decRef(); + } + } assertAliasesVersionIncreases("test", () -> { logger.info("--> aliasing index [test] with [alias1]"); assertAcked(indicesAdmin().prepareAliases().addAlias("test", "alias1")); }); - logger.info("--> indexing against [alias1], should work now"); - DocWriteResponse indexResponse = client().index(new IndexRequest("alias1").id("1").source(source("1", "test"), XContentType.JSON)) - .actionGet(); - assertThat(indexResponse.getIndex(), equalTo("test")); - + { + logger.info("--> indexing against [alias1], should work now"); + DocWriteResponse indexResponse = indexDoc("alias1", "1", source("1", "test"), XContentType.JSON); + assertThat(indexResponse.getIndex(), equalTo("test")); + } logger.info("--> creating index [test_x]"); createIndex("test_x"); @@ -120,48 +125,57 @@ public void testAliases() throws Exception { assertAcked(indicesAdmin().prepareAliases().addAlias("test_x", "alias1")); }); - logger.info("--> indexing against [alias1], should fail now"); - exception = expectThrows( - IllegalArgumentException.class, - client().index(new IndexRequest("alias1").id("1").source(source("2", "test"), XContentType.JSON)) - ); - assertThat( - exception.getMessage(), - equalTo( - "no write index is defined for alias [alias1]." - + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" - + " indices without one being designated as a write index" - ) - ); + { + logger.info("--> indexing against [alias1], should fail now"); + IndexRequest indexRequest = new IndexRequest("alias1").id("1").source(source("2", "test"), XContentType.JSON); + try { + Exception exception = expectThrows(IllegalArgumentException.class, client().index(indexRequest)); + assertThat( + exception.getMessage(), + equalTo( + "no write index is defined for alias [alias1]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index" + ) + ); + } finally { + indexRequest.decRef(); + } + } - logger.info("--> deleting against [alias1], should fail now"); - exception = expectThrows(IllegalArgumentException.class, client().delete(new DeleteRequest("alias1").id("1"))); - assertThat( - exception.getMessage(), - equalTo( - "no write index is defined for alias [alias1]." - + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" - + " indices without one being designated as a write index" - ) - ); + { + logger.info("--> deleting against [alias1], should fail now"); + Exception exception = expectThrows(IllegalArgumentException.class, client().delete(new DeleteRequest("alias1").id("1"))); + assertThat( + exception.getMessage(), + equalTo( + "no write index is defined for alias [alias1]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index" + ) + ); + } assertAliasesVersionIncreases("test_x", () -> { logger.info("--> remove aliasing index [test_x] with [alias1]"); assertAcked(indicesAdmin().prepareAliases().removeAlias("test_x", "alias1")); }); - logger.info("--> indexing against [alias1], should work now"); - indexResponse = client().index(new IndexRequest("alias1").id("1").source(source("1", "test"), XContentType.JSON)).actionGet(); - assertThat(indexResponse.getIndex(), equalTo("test")); - + { + logger.info("--> indexing against [alias1], should work now"); + DocWriteResponse indexResponse = indexDoc("alias1", "1", source("1", "test"), XContentType.JSON); + assertThat(indexResponse.getIndex(), equalTo("test")); + } assertAliasesVersionIncreases("test_x", () -> { logger.info("--> add index [test_x] with [alias1] as write-index"); assertAcked(indicesAdmin().prepareAliases().addAlias("test_x", "alias1", true)); }); - logger.info("--> indexing against [alias1], should work now"); - indexResponse = client().index(new IndexRequest("alias1").id("1").source(source("1", "test"), XContentType.JSON)).actionGet(); - assertThat(indexResponse.getIndex(), equalTo("test_x")); + { + logger.info("--> indexing against [alias1], should work now"); + DocWriteResponse indexResponse = indexDoc("alias1", "1", source("1", "test"), XContentType.JSON); + assertThat(indexResponse.getIndex(), equalTo("test_x")); + } logger.info("--> deleting against [alias1], should fail now"); DeleteResponse deleteResponse = client().delete(new DeleteRequest("alias1").id("1")).actionGet(); @@ -172,9 +186,11 @@ public void testAliases() throws Exception { assertAcked(indicesAdmin().prepareAliases().removeAlias("test", "alias1").addAlias("test_x", "alias1")); }); - logger.info("--> indexing against [alias1], should work against [test_x]"); - indexResponse = client().index(new IndexRequest("alias1").id("1").source(source("1", "test"), XContentType.JSON)).actionGet(); - assertThat(indexResponse.getIndex(), equalTo("test_x")); + { + logger.info("--> indexing against [alias1], should work against [test_x]"); + DocWriteResponse indexResponse = indexDoc("alias1", "1", source("1", "test"), XContentType.JSON); + assertThat(indexResponse.getIndex(), equalTo("test_x")); + } } public void testFailedFilter() throws Exception { @@ -246,20 +262,10 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception { ); logger.info("--> indexing against [test]"); - client().index( - new IndexRequest("test").id("1").source(source("1", "foo test"), XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE) - ).actionGet(); - client().index( - new IndexRequest("test").id("2").source(source("2", "bar test"), XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE) - ).actionGet(); - client().index( - new IndexRequest("test").id("3").source(source("3", "baz test"), XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE) - ).actionGet(); - client().index( - new IndexRequest("test").id("4") - .source(source("4", "something else"), XContentType.JSON) - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - ).actionGet(); + indexDoc("test", "1", source("1", "foo test"), XContentType.JSON); + indexDoc("test", "2", source("2", "bar test"), XContentType.JSON); + indexDoc("test", "3", source("3", "baz test"), XContentType.JSON); + indexDoc("test", "4", source("4", "something else"), XContentType.JSON); logger.info("--> checking single filtering alias search"); assertResponse( @@ -375,16 +381,16 @@ public void testSearchingFilteringAliasesTwoIndices() throws Exception { ); logger.info("--> indexing against [test1]"); - client().index(new IndexRequest("test1").id("1").source(source("1", "foo test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test1").id("2").source(source("2", "bar test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test1").id("3").source(source("3", "baz test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test1").id("4").source(source("4", "something else"), XContentType.JSON)).get(); + indexDoc("test1", "1", source("1", "foo test"), XContentType.JSON); + indexDoc("test1", "2", source("2", "bar test"), XContentType.JSON); + indexDoc("test1", "3", source("3", "baz test"), XContentType.JSON); + indexDoc("test1", "4", source("4", "something else"), XContentType.JSON); logger.info("--> indexing against [test2]"); - client().index(new IndexRequest("test2").id("5").source(source("5", "foo test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test2").id("6").source(source("6", "bar test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test2").id("7").source(source("7", "baz test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test2").id("8").source(source("8", "something else"), XContentType.JSON)).get(); + indexDoc("test2", "5", source("5", "foo test"), XContentType.JSON); + indexDoc("test2", "6", source("6", "bar test"), XContentType.JSON); + indexDoc("test2", "7", source("7", "baz test"), XContentType.JSON); + indexDoc("test2", "8", source("8", "something else"), XContentType.JSON); refresh(); @@ -486,17 +492,18 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception { ); logger.info("--> indexing against [test1]"); - client().index(new IndexRequest("test1").id("11").source(source("11", "foo test1"), XContentType.JSON)).get(); - client().index(new IndexRequest("test1").id("12").source(source("12", "bar test1"), XContentType.JSON)).get(); - client().index(new IndexRequest("test1").id("13").source(source("13", "baz test1"), XContentType.JSON)).get(); - client().index(new IndexRequest("test2").id("21").source(source("21", "foo test2"), XContentType.JSON)).get(); - client().index(new IndexRequest("test2").id("22").source(source("22", "bar test2"), XContentType.JSON)).get(); - client().index(new IndexRequest("test2").id("23").source(source("23", "baz test2"), XContentType.JSON)).get(); + indexDoc("test1", "11", source("11", "foo test1"), XContentType.JSON); + indexDoc("test1", "12", source("12", "bar test1"), XContentType.JSON); + indexDoc("test1", "13", source("13", "baz test1"), XContentType.JSON); - client().index(new IndexRequest("test3").id("31").source(source("31", "foo test3"), XContentType.JSON)).get(); - client().index(new IndexRequest("test3").id("32").source(source("32", "bar test3"), XContentType.JSON)).get(); - client().index(new IndexRequest("test3").id("33").source(source("33", "baz test3"), XContentType.JSON)).get(); + indexDoc("test2", "21", source("21", "foo test2"), XContentType.JSON); + indexDoc("test2", "22", source("22", "bar test2"), XContentType.JSON); + indexDoc("test2", "23", source("23", "baz test2"), XContentType.JSON); + + indexDoc("test3", "31", source("31", "foo test3"), XContentType.JSON); + indexDoc("test3", "32", source("32", "bar test3"), XContentType.JSON); + indexDoc("test3", "33", source("33", "baz test3"), XContentType.JSON); refresh(); @@ -591,16 +598,16 @@ public void testDeletingByQueryFilteringAliases() throws Exception { ); logger.info("--> indexing against [test1]"); - client().index(new IndexRequest("test1").id("1").source(source("1", "foo test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test1").id("2").source(source("2", "bar test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test1").id("3").source(source("3", "baz test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test1").id("4").source(source("4", "something else"), XContentType.JSON)).get(); + indexDoc("test1", "1", source("1", "foo test"), XContentType.JSON); + indexDoc("test1", "2", source("2", "bar test"), XContentType.JSON); + indexDoc("test1", "3", source("3", "baz test"), XContentType.JSON); + indexDoc("test1", "4", source("4", "something else"), XContentType.JSON); logger.info("--> indexing against [test2]"); - client().index(new IndexRequest("test2").id("5").source(source("5", "foo test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test2").id("6").source(source("6", "bar test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test2").id("7").source(source("7", "baz test"), XContentType.JSON)).get(); - client().index(new IndexRequest("test2").id("8").source(source("8", "something else"), XContentType.JSON)).get(); + indexDoc("test2", "5", source("5", "foo test"), XContentType.JSON); + indexDoc("test2", "6", source("6", "bar test"), XContentType.JSON); + indexDoc("test2", "7", source("7", "baz test"), XContentType.JSON); + indexDoc("test2", "8", source("8", "something else"), XContentType.JSON); refresh(); @@ -687,7 +694,7 @@ public void testWaitForAliasCreationMultipleShards() throws Exception { for (int i = 0; i < 10; i++) { final String aliasName = "alias" + i; assertAliasesVersionIncreases("test", () -> assertAcked(indicesAdmin().prepareAliases().addAlias("test", aliasName))); - client().index(new IndexRequest(aliasName).id("1").source(source("1", "test"), XContentType.JSON)).get(); + indexDoc(aliasName, "1", source("1", "test"), XContentType.JSON); } } @@ -700,7 +707,7 @@ public void testWaitForAliasCreationSingleShard() throws Exception { for (int i = 0; i < 10; i++) { final String aliasName = "alias" + i; assertAliasesVersionIncreases("test", () -> assertAcked(indicesAdmin().prepareAliases().addAlias("test", aliasName))); - client().index(new IndexRequest(aliasName).id("1").source(source("1", "test"), XContentType.JSON)).get(); + indexDoc(aliasName, "1", source("1", "test"), XContentType.JSON); } } @@ -719,7 +726,7 @@ public void testWaitForAliasSimultaneousUpdate() throws Exception { @Override public void run() { assertAliasesVersionIncreases("test", () -> assertAcked(indicesAdmin().prepareAliases().addAlias("test", aliasName))); - client().index(new IndexRequest(aliasName).id("1").source(source("1", "test"), XContentType.JSON)).actionGet(); + indexDoc(aliasName, "1", source("1", "test"), XContentType.JSON); } }); } @@ -1130,7 +1137,7 @@ public void testAliasFilterWithNowInRangeFilterAndQuery() { final int numDocs = scaledRandomIntBetween(5, 52); for (int i = 1; i <= numDocs; i++) { - prepareIndex("my-index").setSource("timestamp", "2016-12-12").get(); + indexDoc("my-index", null, "timestamp", "2016-12-12"); if (i % 2 == 0) { refresh(); assertHitCount(prepareSearch("filter1"), i); @@ -1223,7 +1230,9 @@ public void testAliasActionRemoveIndex() throws InterruptedException, ExecutionE public void testRemoveIndexAndReplaceWithAlias() throws InterruptedException, ExecutionException { assertAcked(indicesAdmin().prepareCreate("test")); - indexRandom(true, prepareIndex("test_2").setId("test").setSource("test", "test")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test_2").setId("test").setSource("test", "test"); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertAliasesVersionIncreases( "test_2", () -> assertAcked(indicesAdmin().prepareAliases().addAlias("test_2", "test").removeIndex("test")) @@ -1307,13 +1316,12 @@ public void testIndexingAndQueryingHiddenAliases() throws Exception { ensureGreen(); // Put a couple docs in each index directly - DocWriteResponse res = client().index(new IndexRequest(nonWriteIndex).id("1").source(source("1", "nonwrite"), XContentType.JSON)) - .get(); + DocWriteResponse res = indexDoc(nonWriteIndex, "1", source("1", "nonwrite"), XContentType.JSON); assertThat(res.status().getStatus(), equalTo(201)); - res = client().index(new IndexRequest(writeIndex).id("2").source(source("2", "writeindex"), XContentType.JSON)).get(); + res = indexDoc(writeIndex, "2", source("2", "writeindex"), XContentType.JSON); assertThat(res.status().getStatus(), equalTo(201)); // And through the alias - res = client().index(new IndexRequest(alias).id("3").source(source("3", "through alias"), XContentType.JSON)).get(); + res = indexDoc(alias, "3", source("3", "through alias"), XContentType.JSON); assertThat(res.status().getStatus(), equalTo(201)); refresh(writeIndex, nonWriteIndex); @@ -1434,4 +1442,17 @@ private void assertAliasesVersionUnchanged(final String index, final Runnable ru assertThat(afterAliasesVersion, equalTo(beforeAliasesVersion)); } + private DocWriteResponse indexDoc(String index, String id, String source, XContentType contentType) { + IndexRequest indexRequest = new IndexRequest(index); + try { + if (id != null) { + indexRequest.id(id); + } + indexRequest.source(source, contentType); + indexRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE); + return client().index(indexRequest).actionGet(); + } finally { + indexRequest.decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java index 2e2340294d058..d5c7cff3f5512 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java @@ -44,6 +44,7 @@ public void testGetAliasWithNetNewSystemIndices() throws Exception { final IndexRequest request = new IndexRequest(SYSTEM_INDEX_NAME); request.source("some_field", "some_value"); DocWriteResponse resp = client().index(request).get(); + request.decRef(); assertThat(resp.status().getStatus(), is(201)); } ensureGreen(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java index 136db24767d22..5d4f0b3671ca5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java @@ -36,7 +36,6 @@ import java.util.stream.IntStream; import static java.util.Collections.emptySet; -import static java.util.stream.Collectors.toList; import static org.elasticsearch.action.support.IndicesOptions.lenientExpandOpen; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_WRITE; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY; @@ -127,6 +126,7 @@ private void canIndexDocument(String index) { IndexRequestBuilder builder = prepareIndex(index); builder.setSource("foo", "bar"); DocWriteResponse r = builder.get(); + builder.request().decRef(); assertThat(r, notNullValue()); } catch (ClusterBlockException e) { fail(); @@ -134,13 +134,15 @@ private void canIndexDocument(String index) { } private void canNotIndexDocument(String index) { + IndexRequestBuilder builder = prepareIndex(index); try { - IndexRequestBuilder builder = prepareIndex(index); builder.setSource("foo", "bar"); builder.get(); fail(); } catch (ClusterBlockException e) { // all is well + } finally { + builder.request().decRef(); } } @@ -244,12 +246,13 @@ public void testAddIndexBlock() throws Exception { ensureGreen(indexName); final int nbDocs = randomIntBetween(0, 50); - indexRandom( - randomBoolean(), - false, - randomBoolean(), - IntStream.range(0, nbDocs).mapToObj(i -> prepareIndex(indexName).setId(String.valueOf(i)).setSource("num", i)).collect(toList()) - ); + List builders = IntStream.range(0, nbDocs) + .mapToObj(i -> prepareIndex(indexName).setId(String.valueOf(i)).setSource("num", i)) + .toList(); + indexRandom(randomBoolean(), false, randomBoolean(), builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } final APIBlock block = randomAddableBlock(); try { @@ -269,14 +272,13 @@ public void testSameBlockTwice() throws Exception { createIndex(indexName); if (randomBoolean()) { - indexRandom( - randomBoolean(), - false, - randomBoolean(), - IntStream.range(0, randomIntBetween(1, 10)) - .mapToObj(i -> prepareIndex(indexName).setId(String.valueOf(i)).setSource("num", i)) - .collect(toList()) - ); + List builders = IntStream.range(0, randomIntBetween(1, 10)) + .mapToObj(i -> prepareIndex(indexName).setId(String.valueOf(i)).setSource("num", i)) + .toList(); + indexRandom(randomBoolean(), false, randomBoolean(), builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } } final APIBlock block = randomAddableBlock(); try { @@ -315,12 +317,14 @@ public void testConcurrentAddBlock() throws InterruptedException { createIndex(indexName); final int nbDocs = randomIntBetween(10, 50); - indexRandom( - randomBoolean(), - false, - randomBoolean(), - IntStream.range(0, nbDocs).mapToObj(i -> prepareIndex(indexName).setId(String.valueOf(i)).setSource("num", i)).collect(toList()) - ); + List builders = IntStream.range(0, nbDocs) + .mapToObj(i -> prepareIndex(indexName).setId(String.valueOf(i)).setSource("num", i)) + .toList(); + indexRandom(randomBoolean(), false, randomBoolean(), builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + ensureYellowAndNoInitializingShards(indexName); final CountDownLatch startClosing = new CountDownLatch(1); @@ -393,14 +397,13 @@ public void testAddBlockWhileDeletingIndices() throws Exception { final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); createIndex(indexName); if (randomBoolean()) { - indexRandom( - randomBoolean(), - false, - randomBoolean(), - IntStream.range(0, 10) - .mapToObj(n -> prepareIndex(indexName).setId(String.valueOf(n)).setSource("num", n)) - .collect(toList()) - ); + List builders = IntStream.range(0, 10) + .mapToObj(n -> prepareIndex(indexName).setId(String.valueOf(n)).setSource("num", n)) + .toList(); + indexRandom(randomBoolean(), false, randomBoolean(), builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } } indices[i] = indexName; } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java index c45f980553431..57de6ea3094b2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java @@ -8,7 +8,6 @@ package org.elasticsearch.broadcast; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -33,9 +32,9 @@ public void testBroadcastOperations() throws IOException { NumShards numShards = getNumShards("test"); logger.info("Running Cluster Health"); - client().index(new IndexRequest("test").id("1").source(source("1", "test"))).actionGet(); + index("test", "1", source("1", "test")); flush(); - client().index(new IndexRequest("test").id("2").source(source("2", "test"))).actionGet(); + index("test", "2", source("2", "test")); refresh(); logger.info("Count"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index d3cbab2760747..7dcf6645e9082 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -93,7 +93,7 @@ public void testTwoNodesNoMasterBlock() throws Exception { NumShards numShards = getNumShards("test"); logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get(); + indexDoc("test", Integer.toString(i), "field", "value"); } // make sure that all shards recovered before trying to flush assertThat( @@ -234,7 +234,7 @@ public void testThreeNodesNoMasterBlock() throws Exception { NumShards numShards = getNumShards("test"); logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get(); + indexDoc("test", Integer.toString(i), "field", "value"); } ensureGreen(); // make sure that all shards recovered before trying to flush diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java index bce1b1dae4b44..24aff4f92edda 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java @@ -14,7 +14,9 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.AutoCreateIndex; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.Requests; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; @@ -136,60 +138,64 @@ public void testNoMasterActions() throws Exception { RestStatus.SERVICE_UNAVAILABLE ); - checkUpdateAction( - false, - timeout, - clientToMasterlessNode.prepareUpdate("test", "1") - .setScript(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())) - .setTimeout(timeout) - ); + UpdateRequestBuilder updateRequestBuilder = clientToMasterlessNode.prepareUpdate("test", "1") + .setScript(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())) + .setTimeout(timeout); + checkUpdateAction(false, timeout, updateRequestBuilder); + updateRequestBuilder.request().decRef(); - checkUpdateAction( - true, - timeout, - clientToMasterlessNode.prepareUpdate("no_index", "1") - .setScript(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())) - .setTimeout(timeout) - ); + updateRequestBuilder = clientToMasterlessNode.prepareUpdate("no_index", "1") + .setScript(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())) + .setTimeout(timeout); + checkUpdateAction(true, timeout, updateRequestBuilder); + updateRequestBuilder.request().decRef(); - checkWriteAction( - clientToMasterlessNode.prepareIndex("test") + { + IndexRequestBuilder indexRequestBuilder = clientToMasterlessNode.prepareIndex("test") .setId("1") .setSource(XContentFactory.jsonBuilder().startObject().endObject()) - .setTimeout(timeout) - ); + .setTimeout(timeout); + checkWriteAction(indexRequestBuilder); + indexRequestBuilder.request().decRef(); + } - checkWriteAction( - clientToMasterlessNode.prepareIndex("no_index") + { + IndexRequestBuilder indexRequestBuilder = clientToMasterlessNode.prepareIndex("no_index") .setId("1") .setSource(XContentFactory.jsonBuilder().startObject().endObject()) - .setTimeout(timeout) - ); + .setTimeout(timeout); + checkWriteAction(indexRequestBuilder); + indexRequestBuilder.request().decRef(); + } try (BulkRequestBuilder bulkRequestBuilder = clientToMasterlessNode.prepareBulk()) { - bulkRequestBuilder.add( - clientToMasterlessNode.prepareIndex("test").setId("1").setSource(XContentFactory.jsonBuilder().startObject().endObject()) - ); - bulkRequestBuilder.add( - clientToMasterlessNode.prepareIndex("test").setId("2").setSource(XContentFactory.jsonBuilder().startObject().endObject()) - ); + IndexRequestBuilder indexRequestBuilder1 = clientToMasterlessNode.prepareIndex("test") + .setId("1") + .setSource(XContentFactory.jsonBuilder().startObject().endObject()); + IndexRequestBuilder indexRequestBuilder2 = clientToMasterlessNode.prepareIndex("test") + .setId("2") + .setSource(XContentFactory.jsonBuilder().startObject().endObject()); + bulkRequestBuilder.add(indexRequestBuilder1); + bulkRequestBuilder.add(indexRequestBuilder2); bulkRequestBuilder.setTimeout(timeout); checkWriteAction(bulkRequestBuilder); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); } try (BulkRequestBuilder bulkRequestBuilder = clientToMasterlessNode.prepareBulk()) { - bulkRequestBuilder.add( - clientToMasterlessNode.prepareIndex("no_index") - .setId("1") - .setSource(XContentFactory.jsonBuilder().startObject().endObject()) - ); - bulkRequestBuilder.add( - clientToMasterlessNode.prepareIndex("no_index") - .setId("2") - .setSource(XContentFactory.jsonBuilder().startObject().endObject()) - ); + IndexRequestBuilder indexRequestBuilder1 = clientToMasterlessNode.prepareIndex("no_index") + .setId("1") + .setSource(XContentFactory.jsonBuilder().startObject().endObject()); + IndexRequestBuilder indexRequestBuilder2 = clientToMasterlessNode.prepareIndex("no_index") + .setId("2") + .setSource(XContentFactory.jsonBuilder().startObject().endObject()); + bulkRequestBuilder.add(indexRequestBuilder1); + bulkRequestBuilder.add(indexRequestBuilder2); bulkRequestBuilder.setTimeout(timeout); checkWriteAction(bulkRequestBuilder); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); } internalCluster().clearDisruptionScheme(true); @@ -230,8 +236,8 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception { prepareCreate("test1").setSettings(indexSettings(1, 2)).get(); prepareCreate("test2").setSettings(indexSettings(3, 0)).get(); clusterAdmin().prepareHealth("_all").setWaitForGreenStatus().get(); - prepareIndex("test1").setId("1").setSource("field", "value1").get(); - prepareIndex("test2").setId("1").setSource("field", "value1").get(); + indexDoc("test1", "1", "field", "value1"); + indexDoc("test2", "1", "field", "value1"); refresh(); ensureSearchable("test1", "test2"); @@ -268,11 +274,11 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception { TimeValue timeout = TimeValue.timeValueMillis(200); long now = System.currentTimeMillis(); + UpdateRequestBuilder updateRequestBuilder = clientToMasterlessNode.prepareUpdate("test1", "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2") + .setTimeout(timeout); try { - clientToMasterlessNode.prepareUpdate("test1", "1") - .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2") - .setTimeout(timeout) - .get(); + updateRequestBuilder.get(); fail("Expected ClusterBlockException"); } catch (ClusterBlockException e) { assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50)); @@ -280,17 +286,21 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception { } catch (Exception e) { logger.info("unexpected", e); throw e; + } finally { + updateRequestBuilder.request().decRef(); } + IndexRequestBuilder indexRequestBuilder = clientToMasterlessNode.prepareIndex("test1") + .setId("1") + .setSource(XContentFactory.jsonBuilder().startObject().endObject()) + .setTimeout(timeout); try { - clientToMasterlessNode.prepareIndex("test1") - .setId("1") - .setSource(XContentFactory.jsonBuilder().startObject().endObject()) - .setTimeout(timeout) - .get(); + indexRequestBuilder.get(); fail("Expected ClusterBlockException"); } catch (ClusterBlockException e) { assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); + } finally { + indexRequestBuilder.request().decRef(); } internalCluster().clearDisruptionScheme(true); @@ -306,7 +316,7 @@ public void testNoMasterActionsMetadataWriteMasterBlock() throws Exception { prepareCreate("test1").setSettings(indexSettings(1, 1)).get(); clusterAdmin().prepareHealth("_all").setWaitForGreenStatus().get(); - prepareIndex("test1").setId("1").setSource("field", "value1").get(); + indexDoc("test1", "1", "field", "value1"); refresh(); ensureGreen("test1"); @@ -357,48 +367,69 @@ public void testNoMasterActionsMetadataWriteMasterBlock() throws Exception { expectThrows(Exception.class, client(partitionedNode).prepareSearch("test1").setAllowPartialSearchResults(true).setSize(0)); TimeValue timeout = TimeValue.timeValueMillis(200); - client(randomFrom(nodesWithShards)).prepareUpdate("test1", "1") - .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2") - .setTimeout(timeout) - .get(); + { + UpdateRequestBuilder updateRequestBuilder = client(randomFrom(nodesWithShards)).prepareUpdate("test1", "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2") + .setTimeout(timeout); + updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + } + { + UpdateRequestBuilder updateRequestBuilder = client(partitionedNode).prepareUpdate("test1", "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2") + .setTimeout(timeout); + expectThrows(Exception.class, updateRequestBuilder); + updateRequestBuilder.request().decRef(); + } - expectThrows( - Exception.class, - client(partitionedNode).prepareUpdate("test1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").setTimeout(timeout) - ); + expectThrows(Exception.class, () -> { + UpdateRequestBuilder updateRequestBuilder = client(partitionedNode).prepareUpdate("test1", "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2") + .setTimeout(timeout); + try { + updateRequestBuilder.get(); + } finally { + updateRequestBuilder.request().decRef(); + } + }); - client(randomFrom(nodesWithShards)).prepareIndex("test1") - .setId("1") - .setSource(XContentFactory.jsonBuilder().startObject().endObject()) - .setTimeout(timeout) - .get(); + { + IndexRequestBuilder indexRequestBuilder = client(randomFrom(nodesWithShards)).prepareIndex("test1") + .setId("1") + .setSource(XContentFactory.jsonBuilder().startObject().endObject()) + .setTimeout(timeout); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } - // dynamic mapping updates fail - expectThrows( - MasterNotDiscoveredException.class, - client(randomFrom(nodesWithShards)).prepareIndex("test1") + { + // dynamic mapping updates fail + IndexRequestBuilder indexRequestBuilder = client(randomFrom(nodesWithShards)).prepareIndex("test1") .setId("1") .setSource(XContentFactory.jsonBuilder().startObject().field("new_field", "value").endObject()) - .setTimeout(timeout) - ); + .setTimeout(timeout); + expectThrows(MasterNotDiscoveredException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); + } - // dynamic index creation fails - expectThrows( - MasterNotDiscoveredException.class, - client(randomFrom(nodesWithShards)).prepareIndex("test2") + { + // dynamic index creation fails + IndexRequestBuilder indexRequestBuilder = client(randomFrom(nodesWithShards)).prepareIndex("test2") .setId("1") .setSource(XContentFactory.jsonBuilder().startObject().endObject()) - .setTimeout(timeout) - ); + .setTimeout(timeout); + expectThrows(MasterNotDiscoveredException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); + } - expectThrows( - Exception.class, - client(partitionedNode).prepareIndex("test1") + { + IndexRequestBuilder indexRequestBuilder = client(partitionedNode).prepareIndex("test1") .setId("1") .setSource(XContentFactory.jsonBuilder().startObject().endObject()) - .setTimeout(timeout) - ); - + .setTimeout(timeout); + expectThrows(Exception.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); + } internalCluster().clearDisruptionScheme(true); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleDataNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleDataNodesIT.java index 770ca21fd6898..d26c0348dd52c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleDataNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleDataNodesIT.java @@ -37,36 +37,50 @@ public class SimpleDataNodesIT extends ESIntegTestCase { public void testIndexingBeforeAndAfterDataNodesStart() { internalCluster().startNode(nonDataNode()); indicesAdmin().create(new CreateIndexRequest("test").waitForActiveShards(ActiveShardCount.NONE)).actionGet(); - try { - client().index(new IndexRequest("test").id("1").source(SOURCE, XContentType.JSON).timeout(timeValueSeconds(1))).actionGet(); - fail("no allocation should happen"); - } catch (UnavailableShardsException e) { - // all is well + { + IndexRequest indexRequest = new IndexRequest("test").id("1").source(SOURCE, XContentType.JSON).timeout(timeValueSeconds(1)); + try { + client().index(indexRequest).actionGet(); + fail("no allocation should happen"); + } catch (UnavailableShardsException e) { + // all is well + } finally { + indexRequest.decRef(); + } + + internalCluster().startNode(nonDataNode()); + assertThat( + clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").setLocal(true).get().isTimedOut(), + equalTo(false) + ); } - internalCluster().startNode(nonDataNode()); - assertThat( - clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").setLocal(true).get().isTimedOut(), - equalTo(false) - ); - - // still no shard should be allocated - try { - client().index(new IndexRequest("test").id("1").source(SOURCE, XContentType.JSON).timeout(timeValueSeconds(1))).actionGet(); - fail("no allocation should happen"); - } catch (UnavailableShardsException e) { - // all is well + { + // still no shard should be allocated + IndexRequest indexRequest = new IndexRequest("test").id("1").source(SOURCE, XContentType.JSON).timeout(timeValueSeconds(1)); + try { + client().index(indexRequest).actionGet(); + fail("no allocation should happen"); + } catch (UnavailableShardsException e) { + // all is well + } finally { + indexRequest.decRef(); + } + + // now, start a node data, and see that it gets with shards + internalCluster().startNode(dataNode()); + assertThat( + clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("3").setLocal(true).get().isTimedOut(), + equalTo(false) + ); } - // now, start a node data, and see that it gets with shards - internalCluster().startNode(dataNode()); - assertThat( - clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("3").setLocal(true).get().isTimedOut(), - equalTo(false) - ); - - DocWriteResponse indexResponse = client().index(new IndexRequest("test").id("1").source(SOURCE, XContentType.JSON)).actionGet(); - assertThat(indexResponse.getId(), equalTo("1")); + { + IndexRequest indexRequest = new IndexRequest("test").id("1").source(SOURCE, XContentType.JSON); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + assertThat(indexResponse.getId(), equalTo("1")); + indexRequest.decRef(); + } } public void testShardsAllocatedAfterDataNodesStart() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java index 3b9d3e133b63a..e3f27376c39fd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.cluster.ClusterState; @@ -277,7 +278,11 @@ private void rerouteWithAllocateLocalGateway(Settings commonSettings) throws Exc ); if (closed == false) { - prepareIndex("test").setId("1").setSource("field", "value").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setSource("field", "value") + .setRefreshPolicy(RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } final Index index = resolveIndex("test"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java index ae79c388aa104..37be0a2b8dd1d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java @@ -49,7 +49,7 @@ public void testDecommissionNodeNoReplicas() { ensureGreen("test"); logger.info("--> index some data"); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } indicesAdmin().prepareRefresh().get(); assertHitCount(prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()), 100); @@ -130,7 +130,7 @@ public void testDisablingAllocationFiltering() { logger.info("--> index some data"); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } indicesAdmin().prepareRefresh().get(); assertHitCount(prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()), 100); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java index 3869952bf3b7e..c5df28f0db2ba 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -284,7 +285,9 @@ public void testDelayedMappingPropagationOnPrimary() throws Exception { // this request does not change the cluster state, because mapping is already created, // we don't await and cancel committed publication - ActionFuture docIndexResponse = prepareIndex("index").setId("1").setSource("field", 42).execute(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", 42); + ActionFuture docIndexResponse = indexRequestBuilder.execute(); + indexRequestBuilder.request().decRef(); // Wait a bit to make sure that the reason why we did not get a response // is that cluster state processing is blocked and not just that it takes @@ -373,7 +376,8 @@ public void testDelayedMappingPropagationOnReplica() throws Exception { assertEquals(minVersion, maxVersion); }); - final ActionFuture docIndexResponse = prepareIndex("index").setId("1").setSource("field", 42).execute(); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("index").setId("1").setSource("field", 42); + final ActionFuture docIndexResponse = indexRequestBuilder1.execute(); assertBusy(() -> assertTrue(client().prepareGet("index", "1").get().isExists())); @@ -382,9 +386,8 @@ public void testDelayedMappingPropagationOnReplica() throws Exception { // if the dynamic mapping update is not applied on the replica yet. // this request does not change the cluster state, because the mapping is dynamic, // we need to await and cancel committed publication - ActionFuture dynamicMappingsFut = executeAndCancelCommittedPublication( - prepareIndex("index").setId("2").setSource("field2", 42) - ); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("index").setId("2").setSource("field2", 42); + ActionFuture dynamicMappingsFut = executeAndCancelCommittedPublication(indexRequestBuilder2); // ...and wait for second mapping to be available on master assertBusy(() -> { @@ -413,6 +416,8 @@ public void testDelayedMappingPropagationOnReplica() throws Exception { assertEquals(2, docIndexResponse.get(10, TimeUnit.SECONDS).getShardInfo().getTotal()); // both shards should have succeeded assertThat(dynamicMappingsFut.get(30, TimeUnit.SECONDS).getResult(), equalTo(CREATED)); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java index 012cb826a4403..d3583d24098a2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java @@ -159,6 +159,9 @@ private int indexDocs(String indexName, Object... source) throws InterruptedExce indexRandom(true, false, true, Arrays.asList(builders)); numDocs += numExtraDocs; + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } } return numDocs; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java index 543b0be8ae48d..4ac5b7fe030e1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java @@ -144,6 +144,9 @@ private void indexRandomData() throws Exception { // a case where they are the same (using sync flush), index Random does all this goodness // already indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } } private String findNodeWithShard() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java index 8097bcf7b16d7..003986daf9a1f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java @@ -16,8 +16,10 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -94,9 +96,11 @@ public void testBulkWeirdScenario() throws Exception { ensureGreen(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add(prepareIndex("test").setId("1").setSource("field1", "value1")) - .add(client().prepareUpdate().setIndex("test").setId("1").setDoc("field2", "value2")) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field1", "value1"); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate().setIndex("test").setId("1").setDoc("field2", "value2"); + BulkResponse bulkResponse = bulkRequestBuilder.add(indexRequestBuilder).add(updateRequestBuilder).get(); + indexRequestBuilder.request().decRef(); + updateRequestBuilder.request().decRef(); assertThat(bulkResponse.hasFailures(), equalTo(false)); assertThat(bulkResponse.getItems().length, equalTo(2)); @@ -116,7 +120,7 @@ public void testBulkWeirdScenario() throws Exception { // returns data paths settings of in-sync shard copy private Settings createStaleReplicaScenario(String master) throws Exception { - prepareIndex("test").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); + index("test", null, jsonBuilder().startObject().field("field", "value1").endObject()); refresh(); ClusterState state = clusterAdmin().prepareState().all().get().getState(); List shards = state.routingTable().allShards("test"); @@ -143,7 +147,10 @@ private Settings createStaleReplicaScenario(String master) throws Exception { ensureStableCluster(2, master); logger.info("--> index a document into previous replica shard (that is now primary)"); - client(replicaNode).prepareIndex("test").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); + IndexRequestBuilder indexRequestBuilder = client(replicaNode).prepareIndex("test") + .setSource(jsonBuilder().startObject().field("field", "value1").endObject()); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); logger.info("--> shut down node that has new acknowledged document"); final Settings inSyncDataPathSettings = internalCluster().dataPathSettings(replicaNode); @@ -434,7 +441,7 @@ public void testRemoveAllocationIdOnWriteAfterNodeLeave() throws Exception { ensureYellow("test"); assertEquals(2, clusterAdmin().prepareState().get().getState().metadata().index("test").inSyncAllocationIds(0).size()); logger.info("--> indexing..."); - prepareIndex("test").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); + index("test", null, jsonBuilder().startObject().field("field", "value1").endObject()); assertEquals(1, clusterAdmin().prepareState().get().getState().metadata().index("test").inSyncAllocationIds(0).size()); internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() { @Override @@ -461,7 +468,7 @@ public void testNotWaitForQuorumCopies() throws Exception { logger.info("--> creating index with 1 primary and 2 replicas"); createIndex("test", randomIntBetween(1, 3), 2); ensureGreen("test"); - prepareIndex("test").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); + index("test", null, jsonBuilder().startObject().field("field", "value1").endObject()); logger.info("--> removing 2 nodes from cluster"); internalCluster().stopNode(nodes.get(1)); internalCluster().stopNode(nodes.get(2)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorIT.java index 31e45e64d8afe..0e5f33cf379d4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.cluster.routing.allocation; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.DiskUsageIntegTestCase; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.ShardRouting; @@ -60,7 +61,12 @@ public void testFloodStageExceeded() throws Exception { getTestFileStore(dataNodeName).setTotalSpace(1L); refreshClusterInfo(); assertBusy(() -> { - assertBlocked(prepareIndex(indexName).setId("1").setSource("f", "g"), IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId("1").setSource("f", "g"); + try { + assertBlocked(indexRequestBuilder, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); + } finally { + indexRequestBuilder.request().decRef(); + } assertThat(getIndexBlock(indexName, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE), equalTo("true")); }); @@ -112,7 +118,12 @@ public void testRemoveExistingIndexBlocksWhenDiskThresholdMonitorIsDisabled() th getTestFileStore(dataNodeName).setTotalSpace(1L); refreshClusterInfo(); assertBusy(() -> { - assertBlocked(prepareIndex(indexName).setId("1").setSource("f", "g"), IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId("1").setSource("f", "g"); + try { + assertBlocked(indexRequestBuilder, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); + } finally { + indexRequestBuilder.request().decRef(); + } assertThat(getIndexBlock(indexName, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE), equalTo("true")); }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java index fd5e54631fd7a..ee41aa9a0fd37 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.cluster.routing.allocation.decider; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterInfoServiceUtils; @@ -194,8 +195,14 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception { assertThat("node2 has 2 shards", shardCountByNodeId.get(nodeIds.get(2)), equalTo(2)); } - prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); - assertSearchHits(prepareSearch("test"), "1"); + { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setSource("foo", "bar") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + assertSearchHits(prepareSearch("test"), "1"); + } // Move all nodes above the low watermark so no shard movement can occur, and at least one node above the flood stage watermark so // the index is blocked @@ -207,15 +214,24 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception { ) ); - assertBusy( - () -> assertBlocked(prepareIndex("test").setId("1").setSource("foo", "bar"), IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK) - ); + assertBusy(() -> { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("foo", "bar"); + try { + assertBlocked(indexRequestBuilder, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); + } finally { + indexRequestBuilder.request().decRef(); + } + }); assertFalse(clusterAdmin().prepareHealth("test").setWaitForEvents(Priority.LANGUID).get().isTimedOut()); - // Cannot add further documents - assertBlocked(prepareIndex("test").setId("2").setSource("foo", "bar"), IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); - assertSearchHits(prepareSearch("test"), "1"); + { + // Cannot add further documents + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("2").setSource("foo", "bar"); + assertBlocked(indexRequestBuilder, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); + indexRequestBuilder.request().decRef(); + assertSearchHits(prepareSearch("test"), "1"); + } logger.info("--> index is confirmed read-only, releasing disk space"); @@ -224,10 +240,15 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception { // Attempt to create a new document until DiskUsageMonitor unblocks the index assertBusy(() -> { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("3") + .setSource("foo", "bar") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); try { - prepareIndex("test").setId("3").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + indexRequestBuilder.get(); } catch (ClusterBlockException e) { throw new AssertionError("retrying", e); + } finally { + indexRequestBuilder.request().decRef(); } }); assertSearchHits(prepareSearch("test"), "1", "3"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java index d52498043366a..bcbce4ed389f6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java @@ -115,7 +115,7 @@ public void testClusterSearchShardsWithBlocks() { int docs = between(10, 100); for (int i = 0; i < docs; i++) { - prepareIndex("test-blocks").setId("" + i).setSource("test", "init").get(); + indexDoc("test-blocks", "" + i, "test", "init"); } ensureGreen("test-blocks"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionCleanSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionCleanSettingsIT.java index e8234fb09512b..cda50bd67eebe 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionCleanSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionCleanSettingsIT.java @@ -60,6 +60,9 @@ public void testSearchWithRelocationAndSlowClusterStateProcessing() throws Excep indexRequestBuilderList.add(prepareIndex("test").setSource("{\"int_field\":1}", XContentType.JSON)); } indexRandom(true, indexRequestBuilderList); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilderList) { + indexRequestBuilder.request().decRef(); + } IndicesStoreIntegrationIT.relocateAndBlockCompletion(logger, "test", 0, node_1, node_2); // now search for the documents and see if we get a reply diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java index a0efb81c18668..d9c8c1777e7c7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java @@ -168,11 +168,14 @@ public void testAckedIndexing() throws Exception { } else if (conflictMode == ConflictMode.create) { indexRequestBuilder.setCreate(true); } - - DocWriteResponse response = indexRequestBuilder.get(timeout); - assertThat(response.getResult(), is(oneOf(CREATED, UPDATED))); - ackedDocs.put(id, node); - logger.trace("[{}] indexed id [{}] through node [{}], response [{}]", name, id, node, response); + try { + DocWriteResponse response = indexRequestBuilder.get(timeout); + assertThat(response.getResult(), is(oneOf(CREATED, UPDATED))); + ackedDocs.put(id, node); + logger.trace("[{}] indexed id [{}] through node [{}], response [{}]", name, id, node, response); + } finally { + indexRequestBuilder.request().decRef(); + } } catch (ElasticsearchException e) { exceptedExceptions.add(e); final String docId = id; @@ -292,7 +295,11 @@ public void testRejoinDocumentExistsInAllShardCopies() throws Exception { ensureStableCluster(2, notIsolatedNode); assertFalse(client(notIsolatedNode).admin().cluster().prepareHealth("test").setWaitForYellowStatus().get().isTimedOut()); - DocWriteResponse indexResponse = internalCluster().client(notIsolatedNode).prepareIndex("test").setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = internalCluster().client(notIsolatedNode) + .prepareIndex("test") + .setSource("field", "value"); + DocWriteResponse indexResponse = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertThat(indexResponse.getVersion(), equalTo(1L)); logger.info("Verifying if document exists via node[{}]", notIsolatedNode); @@ -481,15 +488,17 @@ public void testRestartNodeWhileIndexing() throws Exception { threads[i] = new Thread(() -> { while (stopped.get() == false && docID.get() < 5000) { String id = Integer.toString(docID.incrementAndGet()); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id) + .setSource(Map.of("f" + randomIntBetween(1, 10), randomNonNegativeLong()), XContentType.JSON); try { - DocWriteResponse response = prepareIndex(index).setId(id) - .setSource(Map.of("f" + randomIntBetween(1, 10), randomNonNegativeLong()), XContentType.JSON) - .get(); + DocWriteResponse response = indexRequestBuilder.get(); assertThat(response.getResult(), is(oneOf(CREATED, UPDATED))); logger.info("--> index id={} seq_no={}", response.getId(), response.getSeqNo()); ackedDocs.add(response.getId()); } catch (ElasticsearchException ignore) { logger.info("--> fail to index id={}", id); + } finally { + indexRequestBuilder.request().decRef(); } } }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java index a57d8d7f3fbf4..0a86a6d64a638 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.coordination.NoMasterBlockService; import org.elasticsearch.common.settings.Settings; @@ -240,9 +241,18 @@ public void testMappingTimeout() throws Exception { disruption.startDisrupting(); try (BulkRequestBuilder bulk = client().prepareBulk()) { - bulk.add(prepareIndex("test").setId("2").setSource("{ \"f\": 1 }", XContentType.JSON)); - bulk.add(prepareIndex("test").setId("3").setSource("{ \"g\": 1 }", XContentType.JSON)); - bulk.add(prepareIndex("test").setId("4").setSource("{ \"f\": 1 }", XContentType.JSON)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("2").setSource("{ \"f\": 1 }", XContentType.JSON); + bulk.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); + + indexRequestBuilder = prepareIndex("test").setId("3").setSource("{ \"g\": 1 }", XContentType.JSON); + bulk.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); + + indexRequestBuilder = prepareIndex("test").setId("4").setSource("{ \"f\": 1 }", XContentType.JSON); + bulk.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); + BulkResponse bulkResponse = bulk.get(); assertTrue(bulkResponse.hasFailures()); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java index 526921fdc95ba..7c25112b9a3fc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java @@ -222,7 +222,7 @@ public void testMasterFailOverDuringShardSnapshots() throws Exception { final String indexName = "index-one"; createIndex(indexName); - prepareIndex(indexName).setSource("foo", "bar").get(); + indexDoc(indexName, null, "foo", "bar"); blockDataNode(repoName, dataNode); @@ -275,5 +275,8 @@ private void createRandomIndex(String idxName) throws InterruptedException { builders[i] = prepareIndex(idxName).setId(Integer.toString(i)).setSource("field1", "bar " + i); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java index 9ea917518458e..929002c5923ee 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse; import org.elasticsearch.action.support.broadcast.BroadcastResponse; @@ -29,6 +30,8 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static org.elasticsearch.action.DocWriteRequest.OpType; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -55,10 +58,11 @@ public void testIndexActions() throws Exception { logger.info("Running Cluster Health"); ensureGreen(); logger.info("Indexing [type1/1]"); - DocWriteResponse indexResponse = prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource(source("1", "test")) - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertThat(indexResponse.getIndex(), equalTo(getConcreteIndexName())); assertThat(indexResponse.getId(), equalTo("1")); logger.info("Refreshing"); @@ -124,9 +128,13 @@ public void testIndexActions() throws Exception { } logger.info("Index [type1/1]"); - client().index(new IndexRequest("test").id("1").source(source("1", "test"))).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1").source(source("1", "test")); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); logger.info("Index [type1/2]"); - client().index(new IndexRequest("test").id("2").source(source("2", "test2"))).actionGet(); + indexRequest = new IndexRequest("test").id("2").source(source("2", "test2")); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); logger.info("Flushing"); BroadcastResponse flushResult = indicesAdmin().prepareFlush("test").get(); @@ -178,13 +186,22 @@ public void testBulk() throws Exception { ensureGreen(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add(prepareIndex("test").setId("1").setSource(source("1", "test"))) - .add(prepareIndex("test").setId("2").setSource(source("2", "test")).setCreate(true)) - .add(prepareIndex("test").setSource(source("3", "test"))) - .add(prepareIndex("test").setCreate(true).setSource(source("4", "test"))) + List indexRequestBuilders = new ArrayList<>(); + indexRequestBuilders.add(prepareIndex("test").setId("1").setSource(source("1", "test"))); + indexRequestBuilders.add(prepareIndex("test").setId("2").setSource(source("2", "test")).setCreate(true)); + indexRequestBuilders.add(prepareIndex("test").setSource(source("3", "test"))); + indexRequestBuilders.add(prepareIndex("test").setCreate(true).setSource(source("4", "test"))); + indexRequestBuilders.add(prepareIndex("test").setSource("{ xxx }", XContentType.JSON)); + BulkResponse bulkResponse = bulkRequestBuilder.add(indexRequestBuilders.get(0)) + .add(indexRequestBuilders.get(1)) + .add(indexRequestBuilders.get(2)) + .add(indexRequestBuilders.get(3)) .add(client().prepareDelete().setIndex("test").setId("1")) - .add(prepareIndex("test").setSource("{ xxx }", XContentType.JSON)) // failure + .add(indexRequestBuilders.get(4)) // failure .get(); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } assertThat(bulkResponse.hasFailures(), equalTo(true)); assertThat(bulkResponse.getItems().length, equalTo(6)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/document/ShardInfoIT.java b/server/src/internalClusterTest/java/org/elasticsearch/document/ShardInfoIT.java index eab3f21d858f9..67e1a6f324175 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/document/ShardInfoIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/document/ShardInfoIT.java @@ -15,7 +15,9 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.test.ESIntegTestCase; @@ -33,7 +35,9 @@ public class ShardInfoIT extends ESIntegTestCase { public void testIndexAndDelete() throws Exception { prepareIndex(1); - DocWriteResponse indexResponse = prepareIndex("idx").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("idx").setSource("{}", XContentType.JSON); + DocWriteResponse indexResponse = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertShardInfo(indexResponse); DeleteResponse deleteResponse = client().prepareDelete("idx", indexResponse.getId()).get(); assertShardInfo(deleteResponse); @@ -41,7 +45,9 @@ public void testIndexAndDelete() throws Exception { public void testUpdate() throws Exception { prepareIndex(1); - UpdateResponse updateResponse = client().prepareUpdate("idx", "1").setDoc("{}", XContentType.JSON).setDocAsUpsert(true).get(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("idx", "1").setDoc("{}", XContentType.JSON).setDocAsUpsert(true); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertShardInfo(updateResponse); } @@ -49,7 +55,9 @@ public void testBulkWithIndexAndDeleteItems() throws Exception { prepareIndex(1); try (BulkRequestBuilder indexBulkRequestBuilder = client().prepareBulk()) { for (int i = 0; i < 10; i++) { - indexBulkRequestBuilder.add(prepareIndex("idx").setSource("{}", XContentType.JSON)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("idx").setSource("{}", XContentType.JSON); + indexBulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } BulkResponse bulkResponse = indexBulkRequestBuilder.get(); @@ -73,9 +81,11 @@ public void testBulkWithUpdateItems() throws Exception { prepareIndex(1); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { for (int i = 0; i < 10; i++) { - bulkRequestBuilder.add( - client().prepareUpdate("idx", Integer.toString(i)).setDoc("{}", XContentType.JSON).setDocAsUpsert(true) - ); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("idx", Integer.toString(i)) + .setDoc("{}", XContentType.JSON) + .setDocAsUpsert(true); + bulkRequestBuilder.add(updateRequestBuilder); + updateRequestBuilder.request().decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java index 8c6abc3e14cd8..4425ae0ffbc63 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; @@ -84,7 +85,7 @@ public Settings onNodeStopped(String nodeName) { internalCluster().startNode(dataPathSettings); logger.info("--> indexing a simple document"); - prepareIndex(indexName).setId("1").setSource("field1", "value1").get(); + indexDoc(indexName, "1", "field1", "value1"); logger.info("--> restarting the node without the data role"); ex = expectThrows( @@ -152,7 +153,9 @@ public void testFailsToStartIfUpgradedTooFar() { public void testUpgradeDataFolder() throws IOException, InterruptedException { String node = internalCluster().startNode(); prepareCreate("test").get(); - indexRandom(true, prepareIndex("test").setId("1").setSource("{}", XContentType.JSON)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("{}", XContentType.JSON); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); String nodeId = clusterAdmin().prepareState().get().getState().nodes().getMasterNodeId(); final Settings dataPathSettings = internalCluster().dataPathSettings(node); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeRepurposeCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeRepurposeCommandIT.java index e6839044e2100..439372690ce1e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeRepurposeCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeRepurposeCommandIT.java @@ -35,7 +35,7 @@ public void testRepurpose() throws Exception { prepareCreate(indexName, indexSettings(1, 0)).get(); logger.info("--> indexing a simple document"); - prepareIndex(indexName).setId("1").setSource("field1", "value1").get(); + indexDoc(indexName, "1", "field1", "value1"); ensureGreen(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/explain/ExplainActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/explain/ExplainActionIT.java index cdb418182fff2..979d0f7168a98 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/explain/ExplainActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/explain/ExplainActionIT.java @@ -42,7 +42,7 @@ public void testSimple() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSettings(Settings.builder().put("index.refresh_interval", -1))); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("field", "value1").get(); + indexDoc("test", "1", "field", "value1"); ExplainResponse response = client().prepareExplain(indexOrAlias(), "1").setQuery(QueryBuilders.matchAllQuery()).get(); assertNotNull(response); @@ -99,11 +99,11 @@ public void testExplainWithFields() throws Exception { ); ensureGreen("test"); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject().startObject("obj1").field("field1", "value1").field("field2", "value2").endObject().endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject().startObject("obj1").field("field1", "value1").field("field2", "value2").endObject().endObject() + ); refresh(); ExplainResponse response = client().prepareExplain(indexOrAlias(), "1") @@ -157,11 +157,11 @@ public void testExplainWithSource() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen("test"); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject().startObject("obj1").field("field1", "value1").field("field2", "value2").endObject().endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject().startObject("obj1").field("field1", "value1").field("field2", "value2").endObject().endObject() + ); refresh(); ExplainResponse response = client().prepareExplain(indexOrAlias(), "1") @@ -194,7 +194,7 @@ public void testExplainWithFilteredAlias() { ); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value1").get(); + indexDoc("test", "1", "field1", "value1", "field2", "value1"); refresh(); ExplainResponse response = client().prepareExplain("alias1", "1").setQuery(QueryBuilders.matchAllQuery()).get(); @@ -211,7 +211,7 @@ public void testExplainWithFilteredAliasFetchSource() { ); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value1").get(); + indexDoc("test", "1", "field1", "value1", "field2", "value1"); refresh(); ExplainResponse response = client().prepareExplain("alias1", "1") @@ -238,7 +238,7 @@ public void testExplainDateRangeInQueryString() { String aMonthAgo = DateTimeFormatter.ISO_LOCAL_DATE.format(now.minusMonths(1)); String aMonthFromNow = DateTimeFormatter.ISO_LOCAL_DATE.format(now.plusMonths(1)); - prepareIndex("test").setId("1").setSource("past", aMonthAgo, "future", aMonthFromNow).get(); + indexDoc("test", "1", "past", aMonthAgo, "future", aMonthFromNow); refresh(); @@ -288,8 +288,8 @@ public void testQueryRewrite() { .get(); ensureGreen("twitter"); - prepareIndex("twitter").setId("1").setSource("user", "user1", "followers", new String[] { "user2", "user3" }).get(); - prepareIndex("twitter").setId("2").setSource("user", "user2", "followers", new String[] { "user1" }).get(); + indexDoc("twitter", "1", "user", "user1", "followers", new String[] { "user2", "user3" }); + indexDoc("twitter", "2", "user", "user2", "followers", new String[] { "user1" }); refresh(); TermsQueryBuilder termsLookupQuery = QueryBuilders.termsLookupQuery("user", new TermsLookup("twitter", "2", "followers")); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 5f3b854b74fb4..bdf3ed138b287 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -10,10 +10,12 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -127,7 +129,7 @@ public void testSimpleOpenClose() throws Exception { ); logger.info("--> indexing a simple document"); - prepareIndex("test").setId("1").setSource("field1", "value1").get(); + indexDoc("test", "1", "field1", "value1"); logger.info("--> closing test index..."); assertAcked(indicesAdmin().prepareClose("test")); @@ -141,14 +143,14 @@ public void testSimpleOpenClose() throws Exception { logger.info("--> trying to index into a closed index ..."); try { - prepareIndex("test").setId("1").setSource("field1", "value1").get(); + indexDoc("test", "1", "field1", "value1"); fail(); } catch (IndexClosedException e) { // all is well } logger.info("--> creating another index (test2) by indexing into it"); - prepareIndex("test2").setId("1").setSource("field1", "value1").get(); + indexDoc("test2", "1", "field1", "value1"); logger.info("--> verifying that the state is green"); ensureGreen(); @@ -187,7 +189,7 @@ public void testSimpleOpenClose() throws Exception { logger.info("--> trying to index into a closed index ..."); try { - prepareIndex("test").setId("1").setSource("field1", "value1").get(); + indexDoc("test", "1", "field1", "value1"); fail(); } catch (IndexClosedException e) { // all is well @@ -212,7 +214,7 @@ public void testSimpleOpenClose() throws Exception { assertThat(getResponse.isExists(), equalTo(true)); logger.info("--> indexing a simple document"); - prepareIndex("test").setId("2").setSource("field1", "value1").get(); + indexDoc("test", "2", "field1", "value1"); } public void testJustMasterNode() throws Exception { @@ -251,7 +253,7 @@ public void testJustMasterNodeAndJustDataNode() { logger.info("--> create an index"); indicesAdmin().prepareCreate("test").get(); - prepareIndex("test").setSource("field1", "value1").get(); + indexDoc("test", null, "field1", "value1"); } public void testTwoNodesSingleDoc() throws Exception { @@ -261,7 +263,7 @@ public void testTwoNodesSingleDoc() throws Exception { internalCluster().startNodes(2); logger.info("--> indexing a simple document"); - prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1"); logger.info("--> waiting for green status"); ClusterHealthResponse health = clusterAdmin().prepareHealth() @@ -365,7 +367,7 @@ public void testRecoverBrokenIndexMetadata() throws Exception { logger.info("--> starting one node"); internalCluster().startNode(); logger.info("--> indexing a simple document"); - prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1"); logger.info("--> waiting for green status"); if (usually()) { ensureYellow(); @@ -442,7 +444,7 @@ public void testRecoverMissingAnalyzer() throws Exception { } }""").get(); logger.info("--> indexing a simple document"); - prepareIndex("test").setId("1").setSource("field1", "value one").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value one"); logger.info("--> waiting for green status"); if (usually()) { ensureYellow(); @@ -490,7 +492,7 @@ public void testRecoverMissingAnalyzer() throws Exception { public void testArchiveBrokenClusterSettings() throws Exception { logger.info("--> starting one node"); internalCluster().startNode(); - prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1"); logger.info("--> waiting for green status"); if (usually()) { ensureYellow(); @@ -584,4 +586,13 @@ public Settings onNodeStopped(String nodeName) throws Exception { } }); } + + private DocWriteResponse indexDocImmediate(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/QuorumGatewayIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/QuorumGatewayIT.java index 15a72e3534b50..0134fdc0fbb2f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/QuorumGatewayIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/QuorumGatewayIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -42,11 +43,11 @@ public void testQuorumRecovery() throws Exception { final NumShards test = getNumShards("test"); logger.info("--> indexing..."); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("field", "value1").endObject()); // We don't check for failures in the flush response: if we do we might get the following: // FlushNotAllowedEngineException[[test][1] recovery is in progress, flush [COMMIT_TRANSLOG] is not allowed] flush(); - prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("field", "value2").endObject()).get(); + index("test", "2", jsonBuilder().startObject().field("field", "value2").endObject()); refresh(); for (int i = 0; i < 10; i++) { @@ -73,10 +74,11 @@ public void doAfterNodes(int numNodes, final Client activeClient) throws Excepti }, 30, TimeUnit.SECONDS); logger.info("--> one node is closed -- index 1 document into the remaining nodes"); - activeClient.prepareIndex("test") + IndexRequestBuilder indexRequestBuilder = activeClient.prepareIndex("test") .setId("3") - .setSource(jsonBuilder().startObject().field("field", "value3").endObject()) - .get(); + .setSource(jsonBuilder().startObject().field("field", "value3").endObject()); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNoFailures(activeClient.admin().indices().prepareRefresh().get()); for (int i = 0; i < 10; i++) { assertHitCount(activeClient.prepareSearch().setSize(0).setQuery(matchAllQuery()), 3L); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 3baabe4cc888e..b53699ec8c108 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.coordination.ElectionSchedulerFactory; @@ -96,21 +97,11 @@ public void testOneNodeRecoverFromGateway() throws Exception { ); assertAcked(prepareCreate("test").setMapping(mapping)); - prepareIndex("test").setId("10990239") - .setSource(jsonBuilder().startObject().startArray("appAccountIds").value(14).value(179).endArray().endObject()) - .get(); - prepareIndex("test").setId("10990473") - .setSource(jsonBuilder().startObject().startArray("appAccountIds").value(14).endArray().endObject()) - .get(); - prepareIndex("test").setId("10990513") - .setSource(jsonBuilder().startObject().startArray("appAccountIds").value(14).value(179).endArray().endObject()) - .get(); - prepareIndex("test").setId("10990695") - .setSource(jsonBuilder().startObject().startArray("appAccountIds").value(14).endArray().endObject()) - .get(); - prepareIndex("test").setId("11026351") - .setSource(jsonBuilder().startObject().startArray("appAccountIds").value(14).endArray().endObject()) - .get(); + index("test", "10990239", jsonBuilder().startObject().startArray("appAccountIds").value(14).value(179).endArray().endObject()); + index("test", "10990473", jsonBuilder().startObject().startArray("appAccountIds").value(14).endArray().endObject()); + index("test", "10990513", jsonBuilder().startObject().startArray("appAccountIds").value(14).value(179).endArray().endObject()); + index("test", "10990695", jsonBuilder().startObject().startArray("appAccountIds").value(14).endArray().endObject()); + index("test", "11026351", jsonBuilder().startObject().startArray("appAccountIds").value(14).endArray().endObject()); refresh(); assertHitCount(prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)), 2); @@ -265,9 +256,9 @@ public void testSingleNodeNoFlush() throws Exception { public void testSingleNodeWithFlush() throws Exception { internalCluster().startNode(); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("field", "value1").endObject()); flush(); - prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("field", "value2").endObject()).get(); + index("test", "2", jsonBuilder().startObject().field("field", "value2").endObject()); refresh(); assertHitCount(prepareSearch().setSize(0).setQuery(matchAllQuery()), 2); @@ -302,9 +293,9 @@ public void testTwoNodeFirstNodeCleared() throws Exception { final String firstNode = internalCluster().startNode(); internalCluster().startNode(); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("field", "value1").endObject()); flush(); - prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("field", "value2").endObject()).get(); + index("test", "2", jsonBuilder().startObject().field("field", "value2").endObject()); refresh(); logger.info("Running Cluster Health (wait for the shards to startup)"); @@ -352,9 +343,9 @@ public void testLatestVersionLoaded() throws Exception { Settings node2DataPathSettings = internalCluster().dataPathSettings(nodes.get(1)); assertAcked(indicesAdmin().prepareCreate("test")); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("field", "value1").endObject()); indicesAdmin().prepareFlush().get(); - prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("field", "value2").endObject()).get(); + index("test", "2", jsonBuilder().startObject().field("field", "value2").endObject()); indicesAdmin().prepareRefresh().get(); logger.info("--> running cluster_health (wait for the shards to startup)"); @@ -371,7 +362,7 @@ public void testLatestVersionLoaded() throws Exception { internalCluster().stopRandomDataNode(); logger.info("--> one node is closed - start indexing data into the second one"); - prepareIndex("test").setId("3").setSource(jsonBuilder().startObject().field("field", "value3").endObject()).get(); + index("test", "3", jsonBuilder().startObject().field("field", "value3").endObject()); // TODO: remove once refresh doesn't fail immediately if there a master block: // https://github.com/elastic/elasticsearch/issues/9997 // clusterAdmin().prepareHealth("test").setWaitForYellowStatus().get(); @@ -456,7 +447,9 @@ public void testReuseInFileBasedPeerRecovery() throws Exception { logger.info("--> indexing docs"); int numDocs = randomIntBetween(1, 1024); for (int i = 0; i < numDocs; i++) { - client(primaryNode).prepareIndex("test").setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = client(primaryNode).prepareIndex("test").setSource("field", "value"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } client(primaryNode).admin().indices().prepareFlush("test").setForce(true).get(); @@ -489,7 +482,9 @@ public void testReuseInFileBasedPeerRecovery() throws Exception { public Settings onNodeStopped(String nodeName) throws Exception { // index some more documents; we expect to reuse the files that already exist on the replica for (int i = 0; i < moreDocs; i++) { - client(primaryNode).prepareIndex("test").setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = client(primaryNode).prepareIndex("test").setSource("field", "value"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } // prevent a sequence-number-based recovery from being possible diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java index e7988d447571a..402c0b8b1a34a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.gateway; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingNodesHelper; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -37,6 +38,7 @@ import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; @@ -85,7 +87,7 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { String nodeWithReplica = internalCluster().startDataOnlyNode(); Settings nodeWithReplicaSettings = internalCluster().dataPathSettings(nodeWithReplica); ensureGreen(indexName); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -93,7 +95,7 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { ); indicesAdmin().prepareFlush(indexName).get(); if (randomBoolean()) { - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), false, randomBoolean(), @@ -148,7 +150,7 @@ public void testRecentPrimaryInformation() throws Exception { Settings nodeWithReplicaSettings = internalCluster().dataPathSettings(nodeWithReplica); ensureGreen(indexName); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), false, randomBoolean(), @@ -156,7 +158,7 @@ public void testRecentPrimaryInformation() throws Exception { ); internalCluster().stopNode(nodeWithReplica); if (randomBoolean()) { - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), false, randomBoolean(), @@ -180,7 +182,7 @@ public void testRecentPrimaryInformation() throws Exception { String newNode = internalCluster().startDataOnlyNode(); recoveryStarted.await(); // Index more documents and flush to destroy sync_id and remove the retention lease (as file_based_recovery_threshold reached). - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -231,14 +233,14 @@ public void testFullClusterRestartPerformNoopRecovery() throws Exception { ) ); ensureGreen(indexName); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, between(200, 500)).mapToObj(n -> prepareIndex(indexName).setSource("f", "v")).toList() ); indicesAdmin().prepareFlush(indexName).get(); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), false, randomBoolean(), @@ -277,7 +279,7 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception { ) ); ensureGreen(indexName); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -289,7 +291,7 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception { internalCluster().stopNode(nodeWithLowerMatching); ensureGreen(indexName); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), false, randomBoolean(), @@ -300,7 +302,7 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception { Settings nodeWithHigherMatchingSettings = internalCluster().dataPathSettings(nodeWithHigherMatching); internalCluster().stopNode(nodeWithHigherMatching); if (usually()) { - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), false, randomBoolean(), @@ -334,7 +336,7 @@ public void testDoNotCancelRecoveryForBrokenNode() throws Exception { .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms") ) ); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -380,7 +382,7 @@ public void testPeerRecoveryForClosedIndices() throws Exception { .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms") .build() ); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -443,4 +445,19 @@ private void assertNoOpRecoveries(String indexName) { } } } + + private void indexRandomAndDecRefRequests( + boolean forceRefresh, + boolean dummyDocuments, + boolean maybeFlush, + List builders + ) throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, maybeFlush, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java index 27e63e5614744..066221d02524e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.IndexWriter; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.UUIDs; @@ -41,6 +42,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -166,7 +168,7 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { String nodeWithReplica = internalCluster().startDataOnlyNode(); Settings nodeWithReplicaSettings = internalCluster().dataPathSettings(nodeWithReplica); ensureGreen(indexName); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -223,7 +225,7 @@ public void testFullClusterRestartPerformNoopRecovery() throws Exception { ) ); ensureGreen(indexName); - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -261,7 +263,7 @@ public void testSimulateRecoverySourceOnOldNode() throws Exception { createIndex(indexName, 1, 0); ensureGreen(indexName); if (randomBoolean()) { - indexRandom( + indexRandomAndDecRefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -323,4 +325,19 @@ private void ensureGlobalCheckpointAdvancedAndSynced(String indexName) throws Ex } }); } + + private void indexRandomAndDecRefRequests( + boolean forceRefresh, + boolean dummyDocuments, + boolean maybeFlush, + List builders + ) throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, maybeFlush, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java index c9809574002c8..ab094d62e0d27 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.common.Strings; @@ -81,7 +82,7 @@ public void testSimpleGet() { assertThat(response.isExists(), equalTo(false)); logger.info("--> index doc 1"); - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").get(); + indexDoc("test", "1", "field1", "value1", "field2", "value2"); logger.info("--> non realtime get 1"); response = client().prepareGet(indexOrAlias(), "1").setRealtime(false).get(); @@ -168,7 +169,7 @@ public void testSimpleGet() { assertThat(response.getField("field2"), nullValue()); logger.info("--> update doc 1"); - prepareIndex("test").setId("1").setSource("field1", "value1_1", "field2", "value2_1").get(); + indexDoc("test", "1", "field1", "value1_1", "field2", "value2_1"); logger.info("--> realtime get 1"); response = client().prepareGet(indexOrAlias(), "1").get(); @@ -178,7 +179,7 @@ public void testSimpleGet() { assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2_1")); logger.info("--> update doc 1 again"); - prepareIndex("test").setId("1").setSource("field1", "value1_2", "field2", "value2_2").get(); + indexDoc("test", "1", "field1", "value1_2", "field2", "value2_2"); response = client().prepareGet(indexOrAlias(), "1").get(); assertThat(response.isExists(), equalTo(true)); @@ -202,7 +203,7 @@ public void testGetWithAliasPointingToMultipleIndices() { } else { indicesAdmin().prepareCreate("index3").addAlias(new Alias("alias1").indexRouting("1").writeIndex(true)).get(); } - DocWriteResponse indexResponse = prepareIndex("index1").setId("id").setSource(Collections.singletonMap("foo", "bar")).get(); + DocWriteResponse indexResponse = index("index1", "id", Collections.singletonMap("foo", "bar")); assertThat(indexResponse.status().getStatus(), equalTo(RestStatus.CREATED.getStatus())); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, client().prepareGet("alias1", "_alias_id")); @@ -226,7 +227,7 @@ public void testSimpleMultiGet() throws Exception { assertThat(response.getResponses()[0].getResponse().isExists(), equalTo(false)); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } response = client().prepareMultiGet() @@ -288,7 +289,7 @@ public void testGetDocWithMultivaluedFields() throws Exception { assertThat(response.isExists(), equalTo(false)); assertThat(response.isExists(), equalTo(false)); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().array("field", "1", "2").endObject()).get(); + index("test", "1", jsonBuilder().startObject().array("field", "1", "2").endObject()); response = client().prepareGet("test", "1").setStoredFields("field").get(); assertThat(response.isExists(), equalTo(true)); @@ -319,7 +320,7 @@ public void testGetWithVersion() { assertThat(response.isExists(), equalTo(false)); logger.info("--> index doc 1"); - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").get(); + indexDoc("test", "1", "field1", "value1", "field2", "value2"); // From translog: @@ -363,7 +364,7 @@ public void testGetWithVersion() { } logger.info("--> index doc 1 again, so increasing the version"); - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").get(); + indexDoc("test", "1", "field1", "value1", "field2", "value2"); // From translog: @@ -418,7 +419,7 @@ public void testMultiGetWithVersion() throws Exception { assertThat(response.getResponses()[0].getResponse().isExists(), equalTo(false)); for (int i = 0; i < 3; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } // Version from translog @@ -468,7 +469,7 @@ public void testMultiGetWithVersion() throws Exception { assertThat(response.getResponses()[2].getFailure().getFailure(), instanceOf(VersionConflictEngineException.class)); for (int i = 0; i < 3; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } // Version from translog @@ -541,9 +542,7 @@ public void testGetFieldsNonLeafField() throws Exception { .setSettings(Settings.builder().put("index.refresh_interval", -1)) ); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject()); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, @@ -618,7 +617,9 @@ public void testGetFieldsComplexField() throws Exception { logger.info("indexing documents"); - prepareIndex("my-index").setId("1").setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("my-index").setId("1").setSource(source, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); logger.info("checking real time retrieval"); @@ -713,7 +714,11 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException { assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); - prepareIndex("test").setId("1").setRouting("routingValue").setId("1").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setRouting("routingValue") + .setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); String[] fieldsList = { "_routing" }; // before refresh - document is only in translog @@ -742,7 +747,9 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { "text": "some text." } """; - prepareIndex("test").setId("1").setSource(doc, XContentType.JSON).setRouting("1").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource(doc, XContentType.JSON).setRouting("1"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); String[] fieldsList = { "_routing" }; // before refresh - document is only in translog assertGetFieldsAlwaysWorks(indexOrAlias(), "1", fieldsList, "1"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java index c9432ede04911..fcd4669e54c50 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.get.TransportGetFromTranslogAction; import org.elasticsearch.action.get.TransportGetFromTranslogAction.Response; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -44,7 +45,11 @@ public void testGetFromTranslog() throws Exception { // There hasn't been any switches from unsafe to safe map assertThat(response.segmentGeneration(), equalTo(-1L)); - var indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(RefreshPolicy.NONE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setSource("field1", "value1") + .setRefreshPolicy(RefreshPolicy.NONE); + var indexResponse = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); response = getFromTranslog(indexOrAlias(), "1"); assertNotNull(response.getResult()); assertThat(response.getResult().isExists(), equalTo(true)); @@ -57,7 +62,7 @@ public void testGetFromTranslog() throws Exception { assertThat(response.getResult().isExists(), equalTo(false)); assertThat(response.segmentGeneration(), equalTo(-1L)); - indexResponse = prepareIndex("test").setSource("field1", "value2").get(); + indexResponse = indexDoc("test", null, "field1", "value2"); response = getFromTranslog(indexOrAlias(), indexResponse.getId()); assertNotNull(response.getResult()); assertThat(response.getResult().isExists(), equalTo(true)); @@ -70,11 +75,11 @@ public void testGetFromTranslog() throws Exception { assertThat(response.segmentGeneration(), equalTo(-1L)); // After two refreshes the LiveVersionMap switches back to append-only and stops tracking IDs // Refreshing with empty LiveVersionMap doesn't cause the switch, see {@link LiveVersionMap.Maps#shouldInheritSafeAccess()}. - prepareIndex("test").setSource("field1", "value3").get(); + indexDoc("test", null, "field1", "value3"); refresh("test"); refresh("test"); // An optimized index operation marks the maps as unsafe - prepareIndex("test").setSource("field1", "value4").get(); + indexDoc("test", null, "field1", "value4"); response = getFromTranslog(indexOrAlias(), "non-existent"); assertNull(response.getResult()); assertThat(response.segmentGeneration(), greaterThan(0L)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java index b8c9c42657d3f..1e55b20bcb261 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java @@ -48,7 +48,7 @@ public void testShardMultiGetFromTranslog() throws Exception { // Do a single get to enable storing locations in translog. Otherwise, we could get unwanted refreshes that // prune the LiveVersionMap and would make the test fail/flaky. - var indexResponse = prepareIndex("test").setId("0").setSource("field1", "value2").get(); + var indexResponse = indexDoc("test", "0", "field1", "value2"); client().prepareGet("test", indexResponse.getId()).get(); var mgetIds = List.of("1", "2", "3"); @@ -65,7 +65,9 @@ public void testShardMultiGetFromTranslog() throws Exception { try (var bulkRequest = client().prepareBulk()) { var idsToIndex = randomSubsetOf(2, mgetIds); for (String id : idsToIndex) { - bulkRequest.add(new IndexRequest("test").id(id).source("field1", "value1")); + IndexRequest indexRequest = new IndexRequest("test").id(id).source("field1", "value1"); + bulkRequest.add(indexRequest); + indexRequest.decRef(); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); var bulkResponse = bulkRequest.get(); @@ -107,7 +109,7 @@ public void testShardMultiGetFromTranslog() throws Exception { } assertThat(response.segmentGeneration(), equalTo(-1L)); - indexResponse = prepareIndex("test").setSource("field1", "value2").get(); + indexResponse = indexDoc("test", null, "field1", "value2"); response = getFromTranslog(indexOrAlias(), List.of(indexResponse.getId())); multiGetShardResponse = response.multiGetShardResponse(); assertThat(getLocations(multiGetShardResponse).size(), equalTo(1)); @@ -131,11 +133,11 @@ public void testShardMultiGetFromTranslog() throws Exception { assertThat(response.segmentGeneration(), equalTo(-1L)); // After two refreshes the LiveVersionMap switches back to append-only and stops tracking IDs // Refreshing with empty LiveVersionMap doesn't cause the switch, see {@link LiveVersionMap.Maps#shouldInheritSafeAccess()}. - prepareIndex("test").setSource("field1", "value3").get(); + indexDoc("test", null, "field1", "value3"); refresh("test"); refresh("test"); // An optimized index operation marks the maps as unsafe - prepareIndex("test").setSource("field1", "value4").get(); + indexDoc("test", null, "field1", "value4"); response = getFromTranslog(indexOrAlias(), List.of("non-existent")); multiGetShardResponse = response.multiGetShardResponse(); assertThat(getLocations(multiGetShardResponse).size(), equalTo(1)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java index 216d5e25218e3..871494233609d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java @@ -71,10 +71,7 @@ public void testFinalPipelineCantChangeDestination() { {"processors": [{"changing_dest": {}}]}"""); clusterAdmin().putPipeline(new PutPipelineRequest("final_pipeline", finalPipelineBody, XContentType.JSON)).actionGet(); - final IllegalStateException e = expectThrows( - IllegalStateException.class, - prepareIndex("index").setId("1").setSource(Map.of("field", "value")) - ); + final IllegalStateException e = expectThrows(IllegalStateException.class, () -> index("index", "1", Map.of("field", "value"))); assertThat( e, hasToString( @@ -91,10 +88,7 @@ public void testFinalPipelineCantRerouteDestination() { {"processors": [{"reroute": {}}]}"""); clusterAdmin().putPipeline(new PutPipelineRequest("final_pipeline", finalPipelineBody, XContentType.JSON)).actionGet(); - final IllegalStateException e = expectThrows( - IllegalStateException.class, - prepareIndex("index").setId("1").setSource(Map.of("field", "value")) - ); + final IllegalStateException e = expectThrows(IllegalStateException.class, () -> index("index", "1", Map.of("field", "value"))); assertThat( e, hasToString( @@ -118,10 +112,7 @@ public void testFinalPipelineOfOldDestinationIsNotInvoked() { {"processors": [{"final": {"exists":"no_such_field"}}]}"""); clusterAdmin().putPipeline(new PutPipelineRequest("final_pipeline", finalPipelineBody, XContentType.JSON)).actionGet(); - DocWriteResponse indexResponse = prepareIndex("index").setId("1") - .setSource(Map.of("field", "value")) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + DocWriteResponse indexResponse = indexWithImmediateRefresh("index", "1", Map.of("field", "value")); assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse(prepareSearch("target"), response -> { assertEquals(1, response.getHits().getTotalHits().value); @@ -144,10 +135,7 @@ public void testFinalPipelineOfNewDestinationIsInvoked() { {"processors": [{"final": {}}]}"""); clusterAdmin().putPipeline(new PutPipelineRequest("final_pipeline", finalPipelineBody, XContentType.JSON)).actionGet(); - DocWriteResponse indexResponse = prepareIndex("index").setId("1") - .setSource(Map.of("field", "value")) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + DocWriteResponse indexResponse = indexWithImmediateRefresh("index", "1", Map.of("field", "value")); assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse(prepareSearch("target"), response -> { assertEquals(1, response.getHits().getTotalHits().value); @@ -170,10 +158,7 @@ public void testDefaultPipelineOfNewDestinationIsNotInvoked() { {"processors": [{"final": {}}]}"""); clusterAdmin().putPipeline(new PutPipelineRequest("target_default_pipeline", targetPipeline, XContentType.JSON)).actionGet(); - DocWriteResponse indexResponse = prepareIndex("index").setId("1") - .setSource(Map.of("field", "value")) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + DocWriteResponse indexResponse = indexWithImmediateRefresh("index", "1", Map.of("field", "value")); assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse(prepareSearch("target"), response -> { assertEquals(1, response.getHits().getTotalHits().value); @@ -196,10 +181,7 @@ public void testDefaultPipelineOfRerouteDestinationIsInvoked() { {"processors": [{"final": {}}]}"""); clusterAdmin().putPipeline(new PutPipelineRequest("target_default_pipeline", targetPipeline, XContentType.JSON)).actionGet(); - DocWriteResponse indexResponse = prepareIndex("index").setId("1") - .setSource(Map.of("field", "value")) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + DocWriteResponse indexResponse = indexWithImmediateRefresh("index", "1", Map.of("field", "value")); assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse(prepareSearch("target"), response -> { assertEquals(1, response.getHits().getTotalHits().value); @@ -224,7 +206,7 @@ public void testAvoidIndexingLoop() { IllegalStateException exception = expectThrows( IllegalStateException.class, - prepareIndex("index").setId("1").setSource(Map.of("dest", "index")).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + () -> indexWithImmediateRefresh("index", "1", Map.of("dest", "index")) ); assertThat( exception.getMessage(), @@ -239,7 +221,7 @@ public void testFinalPipeline() { // this asserts that the final_pipeline was used, without us having to actually create the pipeline etc. final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - prepareIndex("index").setId("1").setSource(Map.of("field", "value")) + () -> index("index", "1", Map.of("field", "value")) ); assertThat(e, hasToString(containsString("pipeline with id [final_pipeline] does not exist"))); } @@ -258,6 +240,7 @@ public void testRequestPipelineAndFinalPipeline() { index.setPipeline("request_pipeline"); index.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); final DocWriteResponse response = index.get(); + index.request().decRef(); assertThat(response.status(), equalTo(RestStatus.CREATED)); final GetRequestBuilder get = client().prepareGet("index", "1"); final GetResponse getResponse = get.get(); @@ -285,6 +268,7 @@ public void testDefaultAndFinalPipeline() { index.setSource(Map.of("field", "value")); index.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); final DocWriteResponse response = index.get(); + index.request().decRef(); assertThat(response.status(), equalTo(RestStatus.CREATED)); final GetRequestBuilder get = client().prepareGet("index", "1"); final GetResponse getResponse = get.get(); @@ -332,6 +316,7 @@ public void testDefaultAndFinalPipelineFromTemplates() { index.setSource(Map.of("field", "value")); index.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); final DocWriteResponse response = index.get(); + index.request().decRef(); assertThat(response.status(), equalTo(RestStatus.CREATED)); final GetRequestBuilder get = client().prepareGet("index", "1"); final GetResponse getResponse = get.get(); @@ -366,7 +351,7 @@ public void testHighOrderFinalPipelinePreferred() throws IOException { // this asserts that the high_order_final_pipeline was selected, without us having to actually create the pipeline etc. final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - prepareIndex("index").setId("1").setSource(Map.of("field", "value")) + () -> index("index", "1", Map.of("field", "value")) ); assertThat(e, hasToString(containsString("pipeline with id [high_order_final_pipeline] does not exist"))); } @@ -531,4 +516,15 @@ public boolean isAsync() { } } + private DocWriteResponse indexWithImmediateRefresh(String index, String id, Map source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id) + .setSource(source) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + try { + return indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java index f225674215bb2..c95e638de8a20 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.cluster.metadata.MappingMetadata; @@ -37,7 +38,10 @@ public class HiddenIndexIT extends ESIntegTestCase { public void testHiddenIndexSearch() { assertAcked(indicesAdmin().prepareCreate("hidden-index").setSettings(Settings.builder().put("index.hidden", true).build()).get()); - prepareIndex("hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("hidden-index").setSource("foo", "bar") + .setRefreshPolicy(RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); // default not visible to wildcard expansion assertResponse( @@ -67,7 +71,9 @@ public void testHiddenIndexSearch() { // implicit based on use of pattern starting with . and a wildcard assertAcked(indicesAdmin().prepareCreate(".hidden-index").setSettings(Settings.builder().put("index.hidden", true).build()).get()); - prepareIndex(".hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexRequestBuilder = prepareIndex(".hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertResponse(prepareSearch(randomFrom(".*", ".hidden-*")).setSize(1000).setQuery(QueryBuilders.matchAllQuery()), response -> { boolean matchedHidden = Arrays.stream(response.getHits().getHits()).anyMatch(hit -> ".hidden-index".equals(hit.getIndex())); assertTrue(matchedHidden); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexRequestBuilderIT.java index 71f0c75efa026..976429472a833 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexRequestBuilderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexRequestBuilderIT.java @@ -35,6 +35,9 @@ public void testSetSource() throws InterruptedException, ExecutionException { prepareIndex("test").setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}")), XContentType.JSON), prepareIndex("test").setSource(map) }; indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ElasticsearchAssertions.assertHitCount( prepareSearch("test").setQuery(QueryBuilders.termQuery("test_field", "foobar")), builders.length @@ -42,11 +45,14 @@ public void testSetSource() throws InterruptedException, ExecutionException { } public void testOddNumberOfSourceObjects() { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); try { - prepareIndex("test").setSource("test_field", "foobar", new Object()); + indexRequestBuilder.setSource("test_field", "foobar", new Object()); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("The number of object passed must be even but was [3]")); + } finally { + indexRequestBuilder.request().decRef(); } } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java index aca13ecb3b0e7..7858735f88538 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java @@ -70,9 +70,7 @@ public void testIndexSort() { .putList("index.sort.field", "date", "numeric_dv", "keyword_dv") ).setMapping(TEST_MAPPING).get(); for (int i = 0; i < 20; i++) { - prepareIndex("test").setId(Integer.toString(i)) - .setSource("numeric_dv", randomInt(), "keyword_dv", randomAlphaOfLengthBetween(10, 20)) - .get(); + indexDoc("test", Integer.toString(i), "numeric_dv", randomInt(), "keyword_dv", randomAlphaOfLengthBetween(10, 20)); } flushAndRefresh(); ensureYellow(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java index e0e3be492023b..a8e15067b9a66 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java @@ -110,6 +110,7 @@ public void testWriteIndexingPressureMetricsAreIncremented() throws Exception { totalRequestSize += request.ramBytesUsed(); assertTrue(request.ramBytesUsed() > request.source().length()); bulkRequest.add(request); + request.decRef(); } final long bulkRequestSize = bulkRequest.ramBytesUsed(); @@ -166,6 +167,7 @@ public void testWriteIndexingPressureMetricsAreIncremented() throws Exception { final long secondBulkRequestSize = secondBulkRequest.ramBytesUsed(); final long secondBulkShardRequestSize = request.ramBytesUsed(); final long secondBulkOps = secondBulkRequest.numberOfActions(); + request.decRef(); assertBusy(() -> { assertThat( @@ -247,6 +249,7 @@ public void testWriteCanBeRejectedAtCoordinatingLevel() throws Exception { totalRequestSize += request.ramBytesUsed(); assertTrue(request.ramBytesUsed() > request.source().length()); bulkRequest.add(request); + request.decRef(); } final long bulkRequestSize = bulkRequest.ramBytesUsed(); @@ -316,6 +319,7 @@ public void testWriteCanBeRejectedAtPrimaryLevel() throws Exception { totalRequestSize += request.ramBytesUsed(); assertTrue(request.ramBytesUsed() > request.source().length()); bulkRequest.add(request); + request.decRef(); } final long bulkShardRequestSize = totalRequestSize; restartNodesWithSettings( @@ -389,6 +393,7 @@ public void testWritesWillSucceedIfBelowThreshold() throws Exception { .source(Collections.singletonMap("key", randomAlphaOfLength(500))); totalRequestSize += request.ramBytesUsed(); responses.add(client(coordinatingOnlyNode).index(request)); + request.decRef(); } replicaRelease.close(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java index 578c1adf3143a..e9fc49972104b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java @@ -14,7 +14,9 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Requests; import org.elasticsearch.common.settings.Settings; @@ -56,7 +58,11 @@ public void createTestIndex() { } public void testIndex() { - DocWriteResponse index = prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setSource("foo", "bar") + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + DocWriteResponse index = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertEquals(RestStatus.CREATED, index.status()); assertFalse("request shouldn't have forced a refresh", index.forcedRefresh()); assertSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "bar")), "1"); @@ -64,7 +70,9 @@ public void testIndex() { public void testDelete() throws InterruptedException, ExecutionException { // Index normally - indexRandom(true, prepareIndex("test").setId("1").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "bar")), "1"); // Now delete with blockUntilRefresh @@ -76,49 +84,65 @@ public void testDelete() throws InterruptedException, ExecutionException { public void testUpdate() throws InterruptedException, ExecutionException { // Index normally - indexRandom(true, prepareIndex("test").setId("1").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "bar")), "1"); - // Update with RefreshPolicy.WAIT_UNTIL - UpdateResponse update = client().prepareUpdate("test", "1") - .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz") - .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) - .get(); - assertEquals(2, update.getVersion()); - assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); - assertSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "baz")), "1"); - - // Upsert with RefreshPolicy.WAIT_UNTIL - update = client().prepareUpdate("test", "2") - .setDocAsUpsert(true) - .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "cat") - .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) - .get(); - assertEquals(1, update.getVersion()); - assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); - assertSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "cat")), "2"); - - // Update-becomes-delete with RefreshPolicy.WAIT_UNTIL - update = client().prepareUpdate("test", "2") - .setScript(new Script(ScriptType.INLINE, "mockscript", "delete_plz", emptyMap())) - .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) - .get(); - assertEquals(2, update.getVersion()); - assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); - assertNoSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "cat"))); + { + // Update with RefreshPolicy.WAIT_UNTIL + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz") + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + UpdateResponse update = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + assertEquals(2, update.getVersion()); + assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); + assertSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "baz")), "1"); + } + + { + // Upsert with RefreshPolicy.WAIT_UNTIL + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", "2") + .setDocAsUpsert(true) + .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "cat") + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + UpdateResponse update = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + assertEquals(1, update.getVersion()); + assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); + assertSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "cat")), "2"); + } + + { + // Update-becomes-delete with RefreshPolicy.WAIT_UNTIL + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", "2") + .setScript(new Script(ScriptType.INLINE, "mockscript", "delete_plz", emptyMap())) + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + UpdateResponse update = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + assertEquals(2, update.getVersion()); + assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); + assertNoSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "cat"))); + } } public void testBulk() { // Index by bulk with RefreshPolicy.WAIT_UNTIL try (BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL)) { - bulk.add(prepareIndex("test").setId("1").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("foo", "bar"); + bulk.add(indexRequestBuilder); assertBulkSuccess(bulk.get()); + indexRequestBuilder.request().decRef(); assertSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "bar")), "1"); } // Update by bulk with RefreshPolicy.WAIT_UNTIL try (BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL)) { - bulk.add(client().prepareUpdate("test", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz")); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz"); + bulk.add(updateRequestBuilder); + updateRequestBuilder.request().decRef(); assertBulkSuccess(bulk.get()); assertSearchHits(prepareSearch("test").setQuery(matchQuery("foo", "baz")), "1"); } @@ -143,10 +167,11 @@ public void testBulk() { */ public void testNoRefreshInterval() throws InterruptedException, ExecutionException { updateIndexSettings(Settings.builder().put("index.refresh_interval", -1), "test"); - ActionFuture index = prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource("foo", "bar") - .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) - .execute(); + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + ActionFuture index = indexRequestBuilder.execute(); + indexRequestBuilder.request().decRef(); while (false == index.isDone()) { indicesAdmin().prepareRefresh("test").get(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java index 2a48ddc5769ce..afa5151b4c3ed 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java @@ -36,10 +36,10 @@ public void testMergesHappening() throws Exception { final int numDocs = scaledRandomIntBetween(100, 1000); try (BulkRequestBuilder request = client().prepareBulk()) { for (int j = 0; j < numDocs; ++j) { - request.add( - new IndexRequest("test").id(Long.toString(id++)) - .source(jsonBuilder().startObject().field("l", randomLong()).endObject()) - ); + IndexRequest indexRequest = new IndexRequest("test").id(Long.toString(id++)) + .source(jsonBuilder().startObject().field("l", randomLong()).endObject()); + request.add(indexRequest); + indexRequest.decRef(); } BulkResponse response = request.get(); refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java index acfc38ca12f89..10d8f961a7eee 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.engine; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; @@ -161,13 +162,17 @@ static IndexingResult indexDocs(int numRequests, int numThreads) throws Exceptio indexers[i] = new Thread(() -> { phaser.arriveAndAwaitAdvance(); while (completedRequests.incrementAndGet() <= numRequests) { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); try { - final DocWriteResponse resp = prepareIndex("test").setSource("{}", XContentType.JSON).get(); + indexRequestBuilder.setSource("{}", XContentType.JSON); + final DocWriteResponse resp = indexRequestBuilder.get(); numSuccess.incrementAndGet(); assertThat(resp.status(), equalTo(RestStatus.CREATED)); } catch (IllegalArgumentException e) { numFailure.incrementAndGet(); assertThat(e.getMessage(), containsString("Number of documents in the index can't exceed [" + maxDocs.get() + "]")); + } finally { + indexRequestBuilder.request().decRef(); } } }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java index 55e90d4398201..4e1ab98615c1e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java @@ -35,7 +35,7 @@ public void testEagerGlobalOrdinalsFieldDataLoading() throws Exception { ); ensureGreen(); - prepareIndex("test").setId("1").setSource("name", "name").get(); + indexDoc("test", "1", "name", "name"); indicesAdmin().prepareRefresh("test").get(); ClusterStatsResponse response = clusterAdmin().prepareClusterStats().get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java index c1f06aeceebde..2ff8c8028e061 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java @@ -32,7 +32,7 @@ public void testDynamicTemplateCopyTo() throws Exception { int recordCount = between(1, 200); for (int i = 0; i < recordCount * 2; i++) { - prepareIndex("test-idx").setId(Integer.toString(i)).setSource("test_field", "test " + i, "even", i % 2 == 0).get(); + indexDoc("test-idx", Integer.toString(i), "test_field", "test " + i, "even", i % 2 == 0); } indicesAdmin().prepareRefresh("test-idx").get(); @@ -65,7 +65,7 @@ public void testDynamicObjectCopyTo() throws Exception { .endObject() ); assertAcked(indicesAdmin().prepareCreate("test-idx").setMapping(mapping)); - prepareIndex("test-idx").setId("1").setSource("foo", "bar").get(); + indexDoc("test-idx", "1", "foo", "bar"); indicesAdmin().prepareRefresh("test-idx").get(); assertHitCount(prepareSearch("test-idx").setQuery(QueryBuilders.termQuery("root.top.child", "bar")), 1L); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index d35e275b5515c..52c73592f66eb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; @@ -63,9 +64,9 @@ protected Collection> nodePlugins() { public void testConflictingDynamicMappings() { // we don't use indexRandom because the order of requests is important here createIndex("index"); - prepareIndex("index").setId("1").setSource("foo", 3).get(); + indexDoc("index", "1", "foo", 3); try { - prepareIndex("index").setId("2").setSource("foo", "bar").get(); + indexDoc("index", "2", "foo", "bar"); fail("Indexing request should have failed!"); } catch (DocumentParsingException e) { // general case, the parsing code complains that it can't parse "bar" as a "long" @@ -81,13 +82,17 @@ public void testConflictingDynamicMappings() { public void testConflictingDynamicMappingsBulk() { // we don't use indexRandom because the order of requests is important here createIndex("index"); - prepareIndex("index").setId("1").setSource("foo", 3).get(); + indexDoc("index", "1", "foo", 3); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add(prepareIndex("index").setId("1").setSource("foo", 3)).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("foo", 3); + BulkResponse bulkResponse = bulkRequestBuilder.add(indexRequestBuilder).get(); + indexRequestBuilder.request().decRef(); assertFalse(bulkResponse.hasFailures()); } try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add(prepareIndex("index").setId("2").setSource("foo", "bar")).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("2").setSource("foo", "bar"); + BulkResponse bulkResponse = bulkRequestBuilder.add(indexRequestBuilder).get(); + indexRequestBuilder.request().decRef(); assertTrue(bulkResponse.hasFailures()); } } @@ -113,10 +118,7 @@ public void testConcurrentDynamicUpdates() throws Throwable { public void run() { try { startLatch.await(); - assertEquals( - DocWriteResponse.Result.CREATED, - prepareIndex("index").setId(id).setSource("field" + id, "bar").get().getResult() - ); + assertEquals(DocWriteResponse.Result.CREATED, indexDoc("index", id, "field" + id, "bar").getResult()); } catch (Exception e) { error.compareAndSet(null, e); } @@ -166,7 +168,7 @@ public void testPreflightCheckAvoidsMaster() throws InterruptedException, IOExce XContentType.JSON ) .get(); - prepareIndex("index").setId("1").setSource("nested1", Map.of("foo", "bar"), "nested2", Map.of("foo", "bar")).get(); + indexDoc("index", "1", "nested1", Map.of("foo", "bar"), "nested2", Map.of("foo", "bar")); final CountDownLatch masterBlockedLatch = new CountDownLatch(1); final CountDownLatch indexingCompletedLatch = new CountDownLatch(1); @@ -187,13 +189,14 @@ public void onFailure(Exception e) { }); masterBlockedLatch.await(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("2"); try { assertThat( - expectThrows(IllegalArgumentException.class, prepareIndex("index").setId("2").setSource("nested3", Map.of("foo", "bar"))) - .getMessage(), + expectThrows(IllegalArgumentException.class, indexRequestBuilder.setSource("nested3", Map.of("foo", "bar"))).getMessage(), Matchers.containsString("Limit of nested fields [2] has been exceeded") ); } finally { + indexRequestBuilder.request().decRef(); indexingCompletedLatch.countDown(); } } @@ -201,7 +204,7 @@ public void onFailure(Exception e) { public void testTotalFieldsLimitForDynamicMappingsUpdateCheckedAtDocumentParseTime() throws InterruptedException { createIndex("index", Settings.builder().put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2).build()); ensureGreen("index"); - prepareIndex("index").setId("1").setSource("field1", "value1").get(); + indexDoc("index", "1", "field1", "value1"); final CountDownLatch masterBlockedLatch = new CountDownLatch(1); final CountDownLatch indexingCompletedLatch = new CountDownLatch(1); @@ -223,13 +226,18 @@ public void onFailure(Exception e) { masterBlockedLatch.await(); try { - Exception e = expectThrows(DocumentParsingException.class, prepareIndex("index").setId("2").setSource("field2", "value2")); - assertThat(e.getMessage(), Matchers.containsString("failed to parse")); - assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); - assertThat( - e.getCause().getMessage(), - Matchers.containsString("Limit of total fields [2] has been exceeded while adding new fields [1]") - ); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("2"); + try { + Exception e = expectThrows(DocumentParsingException.class, indexRequestBuilder.setSource("field2", "value2")); + assertThat(e.getMessage(), Matchers.containsString("failed to parse")); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + assertThat( + e.getCause().getMessage(), + Matchers.containsString("Limit of total fields [2] has been exceeded while adding new fields [1]") + ); + } finally { + indexRequestBuilder.request().decRef(); + } } finally { indexingCompletedLatch.countDown(); } @@ -265,11 +273,10 @@ public void testTotalFieldsLimitWithRuntimeFields() { { // introduction of a new object with 2 new sub-fields fails - Exception exc = expectThrows( - DocumentParsingException.class, - prepareIndex("index1").setId("1") - .setSource("field3", "value3", "my_object2", Map.of("new_field1", "value1", "new_field2", "value2")) - ); + final IndexRequestBuilder indexRequestBuilder = prepareIndex("index1").setId("1") + .setSource("field3", "value3", "my_object2", Map.of("new_field1", "value1", "new_field2", "value2")); + Exception exc = expectThrows(DocumentParsingException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertThat(exc.getMessage(), Matchers.containsString("failed to parse")); assertThat(exc.getCause(), instanceOf(IllegalArgumentException.class)); assertThat( @@ -280,7 +287,7 @@ public void testTotalFieldsLimitWithRuntimeFields() { { // introduction of a new single field succeeds - prepareIndex("index1").setId("2").setSource("field3", "value3", "new_field4", 100).get(); + indexDoc("index1", "2", "field3", "value3", "new_field4", 100); } { @@ -295,8 +302,7 @@ public void testTotalFieldsLimitWithRuntimeFields() { """, XContentType.JSON)); // introduction of a new object with 2 new sub-fields succeeds - prepareIndex("index1").setId("1") - .setSource("field3", "value3", "my_object2", Map.of("new_field1", "value1", "new_field2", "value2")); + indexDoc("index1", "1", "field3", "value3", "my_object2", Map.of("new_field1", "value1", "new_field2", "value2")); } } @@ -304,7 +310,7 @@ public void testMappingVersionAfterDynamicMappingUpdate() throws Exception { createIndex("test"); final ClusterService clusterService = internalCluster().clusterService(); final long previousVersion = clusterService.state().metadata().index("test").getMappingVersion(); - prepareIndex("test").setId("1").setSource("field", "text").get(); + indexDoc("test", "1", "field", "text"); assertBusy(() -> assertThat(clusterService.state().metadata().index("test").getMappingVersion(), equalTo(1 + previousVersion))); } @@ -367,7 +373,10 @@ public void testBulkRequestWithDynamicTemplates() throws Exception { Randomness.shuffle(requests); try (BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { - requests.forEach(bulkRequest::add); + requests.forEach(request -> { + bulkRequest.add(request); + request.decRef(); + }); final BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertFalse(bulkResponse.hasFailures()); } @@ -420,14 +429,15 @@ public void testBulkRequestWithNotFoundDynamicTemplate() throws Exception { mappings.endObject(); try (BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { - bulkRequest.add( - new IndexRequest("test").id("1") - .source(XContentFactory.jsonBuilder().startObject().field("my_location", "41.12,-71.34").endObject()) - .setDynamicTemplates(Map.of("my_location", "foo_bar")), - new IndexRequest("test").id("2") - .source(XContentFactory.jsonBuilder().startObject().field("address.location", "41.12,-71.34").endObject()) - .setDynamicTemplates(Map.of("address.location", "bar_foo")) - ); + IndexRequest indexRequest1 = new IndexRequest("test").id("1") + .source(XContentFactory.jsonBuilder().startObject().field("my_location", "41.12,-71.34").endObject()) + .setDynamicTemplates(Map.of("my_location", "foo_bar")); + IndexRequest indexRequest2 = new IndexRequest("test").id("2") + .source(XContentFactory.jsonBuilder().startObject().field("address.location", "41.12,-71.34").endObject()) + .setDynamicTemplates(Map.of("address.location", "bar_foo")); + bulkRequest.add(indexRequest1, indexRequest2); + indexRequest1.decRef(); + indexRequest2.decRef(); final BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet(); assertTrue(bulkItemResponses.hasFailures()); assertThat(bulkItemResponses.getItems()[0].getFailure().getCause(), instanceOf(DocumentParsingException.class)); @@ -458,6 +468,7 @@ public void testDynamicRuntimeNoConflicts() { try (BulkRequest bulkRequest = new BulkRequest()) { for (IndexRequest doc : docs) { bulkRequest.add(doc); + doc.decRef(); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet(); @@ -498,6 +509,7 @@ public void testDynamicRuntimeObjectFields() { try (BulkRequest bulkRequest = new BulkRequest()) { for (IndexRequest doc : docs) { bulkRequest.add(doc); + doc.decRef(); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet(); @@ -509,7 +521,7 @@ public void testDynamicRuntimeObjectFields() { assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("obj.runtime.one", "one")), 1); assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("obj.runtime.one.two", "1")), 1); - Exception exception = expectThrows(DocumentParsingException.class, prepareIndex("test").setSource("obj.runtime", "value")); + Exception exception = expectThrows(DocumentParsingException.class, () -> indexDoc("test", null, "obj.runtime", "value")); assertThat( exception.getMessage(), containsString("object mapping for [obj.runtime] tried to parse field [runtime] as object, but found a concrete value") @@ -538,22 +550,16 @@ public void testDynamicRuntimeObjectFields() { // the parent object has been mapped dynamic:true, hence the field gets indexed // we use a fixed doc id here to make sure this document and the one we sent later with a conflicting type // target the same shard where we are sure the mapping update has been applied - assertEquals( - RestStatus.CREATED, - prepareIndex("test").setSource("obj.runtime.dynamic.number", 1) - .setId("id") - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get() - .status() - ); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource("obj.runtime.dynamic.number", 1) + .setId("id") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + assertEquals(RestStatus.CREATED, indexRequestBuilder.get().status()); + indexRequestBuilder.request().decRef(); assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("obj.runtime.dynamic.number", 1)), 1); // a doc with the same field but a different type causes a conflict - Exception e = expectThrows( - DocumentParsingException.class, - prepareIndex("test").setId("id").setSource("obj.runtime.dynamic.number", "string") - ); + Exception e = expectThrows(DocumentParsingException.class, () -> indexDoc("test", "id", "obj.runtime.dynamic.number", "string")); assertThat( e.getMessage(), containsString( @@ -579,6 +585,7 @@ public void testSubobjectsFalseAtRoot() throws Exception { IndexRequest request = new IndexRequest("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source("host.name", "localhost", "host.id", 111, "time", 100, "time.max", 1000); DocWriteResponse indexResponse = client().index(request).actionGet(); + request.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); assertBusy(() -> { @@ -624,6 +631,7 @@ public void testSubobjectsFalse() throws Exception { 1000 ); DocWriteResponse indexResponse = client().index(request).actionGet(); + request.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); assertBusy(() -> { @@ -661,11 +669,15 @@ public void testKnnSubObject() throws Exception { } }""").get()); - client().index(new IndexRequest("test").source("mapped_obj.vector", Randomness.get().doubles(3, 0.0, 5.0).toArray())).get(); - - client().index( - new IndexRequest("test").source("obj.vector", Randomness.get().doubles(MIN_DIMS_FOR_DYNAMIC_FLOAT_MAPPING, 0.0, 5.0).toArray()) - ).get(); + IndexRequest indexRequest = new IndexRequest("test").source("mapped_obj.vector", Randomness.get().doubles(3, 0.0, 5.0).toArray()); + client().index(indexRequest).get(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").source( + "obj.vector", + Randomness.get().doubles(MIN_DIMS_FOR_DYNAMIC_FLOAT_MAPPING, 0.0, 5.0).toArray() + ); + client().index(indexRequest).get(); + indexRequest.decRef(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java index a22910ab9c4eb..b9e6d2e2cfb69 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java @@ -8,7 +8,9 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.unit.DistanceUnit; @@ -45,7 +47,7 @@ public void testMultiFields() throws Exception { assertThat(titleFields.get("not_analyzed"), notNullValue()); assertThat(((Map) titleFields.get("not_analyzed")).get("type").toString(), equalTo("keyword")); - prepareIndex("my-index").setId("1").setSource("title", "Multi fields").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("my-index", "1", "title", "Multi fields"); assertHitCount(prepareSearch("my-index").setQuery(matchQuery("title", "multi")), 1); assertHitCount(prepareSearch("my-index").setQuery(matchQuery("title.not_analyzed", "Multi fields")), 1); @@ -64,7 +66,7 @@ public void testMultiFields() throws Exception { assertThat(titleFields.get("uncased"), notNullValue()); assertThat(((Map) titleFields.get("uncased")).get("analyzer").toString(), equalTo("whitespace")); - prepareIndex("my-index").setId("1").setSource("title", "Multi fields").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("my-index", "1", "title", "Multi fields"); assertHitCount(prepareSearch("my-index").setQuery(matchQuery("title.uncased", "Multi")), 1); } @@ -88,7 +90,7 @@ public void testGeoPointMultiField() throws Exception { assertThat(bField.get("type").toString(), equalTo("keyword")); GeoPoint point = new GeoPoint(51, 19); - prepareIndex("my-index").setId("1").setSource("a", point.toString()).setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("my-index", "1", "a", point.toString()); assertHitCount( prepareSearch("my-index").setSize(0) .setQuery(constantScoreQuery(geoDistanceQuery("a").point(51, 19).distance(50, DistanceUnit.KILOMETERS))), @@ -114,7 +116,7 @@ public void testCompletionMultiField() throws Exception { assertThat(bField.size(), equalTo(1)); assertThat(bField.get("type").toString(), equalTo("keyword")); - prepareIndex("my-index").setId("1").setSource("a", "complete me").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("my-index", "1", "a", "complete me"); assertHitCount(prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "complete me")), 1L); } @@ -135,7 +137,7 @@ public void testIpMultiField() throws Exception { assertThat(bField.size(), equalTo(1)); assertThat(bField.get("type").toString(), equalTo("keyword")); - prepareIndex("my-index").setId("1").setSource("a", "127.0.0.1").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("my-index", "1", "a", "127.0.0.1"); assertHitCount(prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "127.0.0.1")), 1L); } @@ -192,4 +194,13 @@ private XContentBuilder createPutMappingSource() throws IOException { .endObject(); } + private DocWriteResponse indexDocImmediate(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java index 66a35328954e1..4567bba62bf4a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java @@ -32,7 +32,7 @@ public void setUp() throws Exception { super.setUp(); createIndex("test"); ensureGreen(); - prepareIndex("index").setId("1").setSource("field", "value").get(); + indexDoc("index", "1", "field", "value"); refresh(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java index 70983e5abfb96..641766225ee82 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java @@ -43,6 +43,9 @@ public void setUp() throws Exception { public void testZeroTermsQuery() throws ExecutionException, InterruptedException { List indexRequests = getIndexRequests(); indexRandom(true, false, indexRequests); + for (IndexRequestBuilder indexRequestBuilder : indexRequests) { + indexRequestBuilder.request().decRef(); + } MatchPhraseQueryBuilder baseQuery = matchPhraseQuery("name", "the who").analyzer("standard_stopwords"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java index 9c4473297ba7b..2f8e1c6d22a3a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.seqno; import org.apache.lucene.store.AlreadyClosedException; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -57,7 +58,9 @@ public void testGlobalCheckpointSyncWithAsyncDurability() throws Exception { for (int j = 0; j < 10; j++) { final String id = Integer.toString(j); - prepareIndex("test").setId(id).setSource("{\"foo\": " + id + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(id).setSource("{\"foo\": " + id + "}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } assertBusy(() -> { @@ -157,7 +160,10 @@ private void runGlobalCheckpointSyncTest( } for (int j = 0; j < numberOfDocuments; j++) { final String id = Integer.toString(index * numberOfDocuments + j); - prepareIndex("test").setId(id).setSource("{\"foo\": " + id + "}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(id) + .setSource("{\"foo\": " + id + "}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } try { barrier.await(); @@ -223,7 +229,9 @@ public void testPersistGlobalCheckpoint() throws Exception { } int numDocs = randomIntBetween(1, 20); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } ensureGreen("test"); assertBusy(() -> { @@ -252,7 +260,9 @@ public void testPersistLocalCheckpoint() { logger.info("numDocs {}", numDocs); long maxSeqNo = 0; for (int i = 0; i < numDocs; i++) { - maxSeqNo = prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get().getSeqNo(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON); + maxSeqNo = indexRequestBuilder.get().getSeqNo(); + indexRequestBuilder.request().decRef(); logger.info("got {}", maxSeqNo); } for (IndicesService indicesService : internalCluster().getDataNodeInstances(IndicesService.class)) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/GlobalCheckpointListenersIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/GlobalCheckpointListenersIT.java index b38198a98b5a5..aa54746ec0bfa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/GlobalCheckpointListenersIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/GlobalCheckpointListenersIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.shard; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexService; import org.elasticsearch.indices.IndicesService; @@ -63,7 +64,9 @@ public void accept(final long g, final Exception e) { } }, null); - prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertBusy(() -> assertThat(globalCheckpoint.get(), equalTo((long) index))); // adding a listener expecting a lower global checkpoint should fire immediately final AtomicLong immediateGlobalCheckpint = new AtomicLong(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index 0f40b49e096a4..59408e2c38335 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -13,8 +13,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterInfoServiceUtils; import org.elasticsearch.cluster.ClusterState; @@ -149,7 +151,7 @@ public void testLockTryingToDelete() throws Exception { public void testDurableFlagHasEffect() { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).get(); + indexDoc("test", "1", "{}", XContentType.JSON, null); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); @@ -172,7 +174,7 @@ public void testDurableFlagHasEffect() { setDurability(shard, Translog.Durability.REQUEST); assertThat(needsSync, falseWith(translog)); setDurability(shard, Translog.Durability.ASYNC); - prepareIndex("test").setId("2").setSource("{}", XContentType.JSON).get(); + indexDoc("test", "2", "{}", XContentType.JSON, null); assertThat(needsSync, trueWith(translog)); setDurability(shard, Translog.Durability.REQUEST); client().prepareDelete("test", "1").get(); @@ -183,21 +185,25 @@ public void testDurableFlagHasEffect() { assertTrue(translog.syncNeeded()); setDurability(shard, Translog.Durability.REQUEST); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); assertNoFailures( - bulkRequestBuilder.add(prepareIndex("test").setId("3").setSource("{}", XContentType.JSON)) + bulkRequestBuilder.add(indexRequestBuilder.setId("3").setSource("{}", XContentType.JSON)) .add(client().prepareDelete("test", "1")) .get() ); + indexRequestBuilder.request().decRef(); } assertThat(needsSync, falseWith(translog)); setDurability(shard, Translog.Durability.ASYNC); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); assertNoFailures( - bulkRequestBuilder.add(prepareIndex("test").setId("4").setSource("{}", XContentType.JSON)) + bulkRequestBuilder.add(indexRequestBuilder.setId("4").setSource("{}", XContentType.JSON)) .add(client().prepareDelete("test", "3")) .get() ); + indexRequestBuilder.request().decRef(); } setDurability(shard, Translog.Durability.REQUEST); assertThat(needsSync, trueWith(translog)); @@ -225,7 +231,7 @@ public void testIndexDirIsDeletedWhenShardRemoved() throws Exception { Settings idxSettings = Settings.builder().put(IndexMetadata.SETTING_DATA_PATH, idxPath).build(); createIndex("test", idxSettings); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); + indexDoc("test", "1", "{}", XContentType.JSON, IMMEDIATE); assertHitCount(client().prepareSearch("test"), 1L); indicesAdmin().prepareDelete("test").get(); assertAllIndicesRemovedAndDeletionCompleted(Collections.singleton(getInstanceFromNode(IndicesService.class))); @@ -235,7 +241,7 @@ public void testIndexDirIsDeletedWhenShardRemoved() throws Exception { public void testExpectedShardSizeIsPresent() throws InterruptedException { assertAcked(indicesAdmin().prepareCreate("test").setSettings(indexSettings(1, 0))); for (int i = 0; i < 50; i++) { - prepareIndex("test").setSource("{}", XContentType.JSON).get(); + indexDoc("test", null, "{}", XContentType.JSON, null); } ensureGreen("test"); InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class); @@ -258,7 +264,9 @@ public void testIndexCanChangeCustomDataPath() throws Exception { logger.info("--> creating index [{}] with data_path [{}]", index, indexDataPath); createIndex(index, Settings.builder().put(IndexMetadata.SETTING_DATA_PATH, indexDataPath.toAbsolutePath().toString()).build()); - prepareIndex(index).setId("1").setSource("foo", "bar").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId("1").setSource("foo", "bar").setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); ensureGreen(index); assertHitCount(client().prepareSearch(index).setSize(0), 1L); @@ -333,7 +341,7 @@ public void testMaybeFlush() throws Exception { .build() ) .get(); - prepareIndex("test").setId("0").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); + indexDoc("test", "0", "{}", XContentType.JSON, randomBoolean() ? IMMEDIATE : NONE); assertFalse(shard.shouldPeriodicallyFlush()); shard.applyIndexOperationOnPrimary( Versions.MATCH_ANY, @@ -348,7 +356,7 @@ public void testMaybeFlush() throws Exception { final Translog translog = getTranslog(shard); assertEquals(2, translog.stats().getUncommittedOperations()); assertThat(shard.flushStats().getTotal(), equalTo(0L)); - prepareIndex("test").setId("2").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); + indexDoc("test", "2", "{}", XContentType.JSON, randomBoolean() ? IMMEDIATE : NONE); assertThat(shard.getLastKnownGlobalCheckpoint(), equalTo(2L)); assertBusy(() -> { // this is async assertFalse(shard.shouldPeriodicallyFlush()); @@ -451,7 +459,7 @@ public void testStressMaybeFlushOrRollTranslogGeneration() throws Exception { settings = Settings.builder().put("index.translog.generation_threshold_size", "117b").build(); } indicesAdmin().prepareUpdateSettings("test").setSettings(settings).get(); - prepareIndex("test").setId("0").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); + indexDoc("test", "0", "{}", XContentType.JSON, randomBoolean() ? IMMEDIATE : NONE); assertFalse(shard.shouldPeriodicallyFlush()); final AtomicBoolean running = new AtomicBoolean(true); final int numThreads = randomIntBetween(2, 4); @@ -474,7 +482,7 @@ public void testStressMaybeFlushOrRollTranslogGeneration() throws Exception { final CheckedRunnable check; if (flush) { final FlushStats initialStats = shard.flushStats(); - prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).get(); + indexDoc("test", "1", "{}", XContentType.JSON, null); check = () -> { assertFalse(shard.shouldPeriodicallyFlush()); final FlushStats currentStats = shard.flushStats(); @@ -499,7 +507,7 @@ public void testStressMaybeFlushOrRollTranslogGeneration() throws Exception { }; } else { final long generation = getTranslog(shard).currentFileGeneration(); - prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).get(); + indexDoc("test", "1", "{}", XContentType.JSON, null); check = () -> { assertFalse(shard.shouldRollTranslogGeneration()); assertEquals(generation + 1, getTranslog(shard).currentFileGeneration()); @@ -520,7 +528,7 @@ public void testFlushStats() throws Exception { indicesAdmin().prepareUpdateSettings("test").setSettings(settings).get(); final int numDocs = between(10, 100); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get(); + indexDoc("test", Integer.toString(i), "{}", XContentType.JSON, null); } // A flush stats may include the new total count but the old period count - assert eventually. assertBusy(() -> { @@ -531,7 +539,7 @@ public void testFlushStats() throws Exception { settings = Settings.builder().put("index.translog.flush_threshold_size", (String) null).build(); indicesAdmin().prepareUpdateSettings("test").setSettings(settings).get(); - prepareIndex("test").setId(UUIDs.randomBase64UUID()).setSource("{}", XContentType.JSON).get(); + indexDoc("test", UUIDs.randomBase64UUID(), "{}", XContentType.JSON, null); indicesAdmin().prepareFlush("test").setForce(randomBoolean()).setWaitIfOngoing(true).get(); final FlushStats flushStats = indicesAdmin().prepareStats("test").clear().setFlush(true).get().getTotal().flush; assertThat(flushStats.getTotal(), greaterThan(flushStats.getPeriodic())); @@ -543,9 +551,9 @@ public void testShardHasMemoryBufferOnTranslogRecover() throws Throwable { IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService indexService = indicesService.indexService(resolveIndex("test")); IndexShard shard = indexService.getShardOrNull(0); - prepareIndex("test").setId("0").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + indexDoc("test", "0", "{\"foo\" : \"bar\"}", XContentType.JSON, null); client().prepareDelete("test", "0").get(); - prepareIndex("test").setId("1").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); + indexDoc("test", "1", "{\"foo\" : \"bar\"}", XContentType.JSON, IMMEDIATE); CheckedFunction wrapper = directoryReader -> directoryReader; shard.close("simon says", false); @@ -659,7 +667,7 @@ public void testInvalidateIndicesRequestCacheWhenRollbackEngine() throws Excepti final SearchRequest countRequest = new SearchRequest("test").source(new SearchSourceBuilder().size(0)); final long numDocs = between(10, 20); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get(); + indexDoc("test", Integer.toString(i), "{}", XContentType.JSON, null); if (randomBoolean()) { shard.refresh("test"); } @@ -681,7 +689,7 @@ public void testInvalidateIndicesRequestCacheWhenRollbackEngine() throws Excepti final long moreDocs = between(10, 20); for (int i = 0; i < moreDocs; i++) { - prepareIndex("test").setId(Long.toString(i + numDocs)).setSource("{}", XContentType.JSON).get(); + indexDoc("test", Long.toString(i + numDocs), "{}", XContentType.JSON, null); if (randomBoolean()) { shard.refresh("test"); } @@ -705,7 +713,7 @@ public void testShardChangesWithDefaultDocType() throws Exception { int numOps = between(1, 10); for (int i = 0; i < numOps; i++) { if (randomBoolean()) { - prepareIndex("index").setId(randomFrom("1", "2")).setSource("{}", XContentType.JSON).get(); + indexDoc("index", randomFrom("1", "2"), "{}", XContentType.JSON, null); } else { client().prepareDelete("index", randomFrom("1", "2")).get(); } @@ -782,4 +790,17 @@ private static void assertAllIndicesRemovedAndDeletionCompleted(Iterable assertFalse(indicesService.hasUncompletedPendingDeletes()), 1, TimeUnit.MINUTES); } } + + private void indexDoc(String index, String id, String source, XContentType contentType, WriteRequest.RefreshPolicy refreshPolicy) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source, contentType); + if (refreshPolicy != null) { + indexRequestBuilder.setRefreshPolicy(refreshPolicy); + } + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java index 6c691c0a14440..ca6926dae4c28 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java @@ -124,7 +124,7 @@ public void testCorruptIndex() throws Exception { numDocs += numExtraDocs; - indexRandom(false, false, false, Arrays.asList(builders)); + indexRandomAndDecRefRequests(false, false, false, Arrays.asList(builders)); flush(indexName); } @@ -284,7 +284,7 @@ public void testCorruptTranslogTruncation() throws Exception { for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex(indexName).setSource("foo", "bar"); } - indexRandom(false, false, false, Arrays.asList(builders)); + indexRandomAndDecRefRequests(false, false, false, Arrays.asList(builders)); flush(indexName); disableTranslogFlush(indexName); @@ -295,7 +295,7 @@ public void testCorruptTranslogTruncation() throws Exception { for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex(indexName).setSource("foo", "bar"); } - indexRandom(false, false, false, Arrays.asList(builders)); + indexRandomAndDecRefRequests(false, false, false, Arrays.asList(builders)); RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand(); MockTerminal terminal = MockTerminal.create(); @@ -474,7 +474,7 @@ public void testCorruptTranslogTruncationOfReplica() throws Exception { for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex(indexName).setSource("foo", "bar"); } - indexRandom(false, false, false, Arrays.asList(builders)); + indexRandomAndDecRefRequests(false, false, false, Arrays.asList(builders)); flush(indexName); disableTranslogFlush(indexName); // having no extra docs is an interesting case for seq no based recoveries - test it more often @@ -484,7 +484,7 @@ public void testCorruptTranslogTruncationOfReplica() throws Exception { for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex(indexName).setSource("foo", "bar"); } - indexRandom(false, false, false, Arrays.asList(builders)); + indexRandomAndDecRefRequests(false, false, false, Arrays.asList(builders)); final int totalDocs = numDocsToKeep + numDocsToTruncate; // sample the replica node translog dirs @@ -651,4 +651,19 @@ private SeqNoStats getSeqNoStats(String index, int shardId) { final ShardStats[] shardStats = indicesAdmin().prepareStats(index).get().getIndices().get(index).getShards(); return shardStats[shardId].getSeqNoStats(); } + + private void indexRandomAndDecRefRequests( + boolean forceRefresh, + boolean dummyDocuments, + boolean maybeFlush, + List builders + ) throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, maybeFlush, builders); + } finally { + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java index 199a397f52ad2..9a8a93102b2e8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -93,7 +94,7 @@ private void runTestAutomaticRefresh(final IntToLongFunction count) throws Inter int numDocs = scaledRandomIntBetween(25, 100); totalNumDocs.set(numDocs); CountDownLatch indexingDone = new CountDownLatch(numDocs); - prepareIndex("test").setId("0").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + index("test", "0", "{\"foo\" : \"bar\"}", XContentType.JSON); indexingDone.countDown(); // one doc is indexed above blocking IndexShard shard = indexService.getShard(0); PlainActionFuture future = new PlainActionFuture<>(); @@ -126,7 +127,9 @@ private void runTestAutomaticRefresh(final IntToLongFunction count) throws Inter started.await(); assertThat(count.applyAsLong(totalNumDocs.get()), equalTo(1L)); for (int i = 1; i < numDocs; i++) { - prepareIndex("test").setId("" + i).setSource("{\"foo\" : \"bar\"}", XContentType.JSON).execute(new ActionListener<>() { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("" + i) + .setSource("{\"foo\" : \"bar\"}", XContentType.JSON); + indexRequestBuilder.execute(ActionListener.runAfter(new ActionListener<>() { @Override public void onResponse(DocWriteResponse indexResponse) { indexingDone.countDown(); @@ -137,7 +140,7 @@ public void onFailure(Exception e) { indexingDone.countDown(); throw new AssertionError(e); } - }); + }, () -> indexRequestBuilder.request().decRef())); } indexingDone.await(); t.join(); @@ -155,7 +158,7 @@ public void testPendingRefreshWithIntervalChange() throws Exception { IndexService indexService = createIndex("test", builder.build()); assertFalse(indexService.getIndexSettings().isExplicitRefresh()); ensureGreen(); - prepareIndex("test").setId("0").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + index("test", "0", "{\"foo\" : \"bar\"}", XContentType.JSON); IndexShard shard = indexService.getShard(0); scheduleRefresh(shard, false); assertTrue(shard.isSearchIdle()); @@ -163,7 +166,7 @@ public void testPendingRefreshWithIntervalChange() throws Exception { // async on purpose to make sure it happens concurrently indicesAdmin().prepareRefresh().execute(ActionListener.running(refreshLatch::countDown)); assertHitCount(client().prepareSearch(), 1); - prepareIndex("test").setId("1").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + index("test", "1", "{\"foo\" : \"bar\"}", XContentType.JSON); scheduleRefresh(shard, false); assertTrue(shard.hasRefreshPending()); @@ -180,7 +183,7 @@ public void testPendingRefreshWithIntervalChange() throws Exception { // We need to ensure a `scheduledRefresh` triggered by the internal refresh setting update is executed before we index a new doc; // otherwise, it will compete to call `Engine#maybeRefresh` with the `scheduledRefresh` that we are going to verify. ensureNoPendingScheduledRefresh(indexService.getThreadPool()); - prepareIndex("test").setId("2").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + index("test", "2", "{\"foo\" : \"bar\"}", XContentType.JSON); scheduleRefresh(shard, true); assertFalse(shard.hasRefreshPending()); assertTrue(shard.isSearchIdle()); @@ -280,15 +283,11 @@ public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException { assertEquals( RestStatus.CREATED, - prepareIndex(idleIndex).setSource("keyword", "idle", "@timestamp", "2021-05-10T19:00:03.765Z", "routing_field", "aaa") - .get() - .status() + indexDoc(idleIndex, null, "keyword", "idle", "@timestamp", "2021-05-10T19:00:03.765Z", "routing_field", "aaa").status() ); assertEquals( RestStatus.CREATED, - prepareIndex(activeIndex).setSource("keyword", "active", "@timestamp", "2021-05-12T20:07:12.112Z", "routing_field", "aaa") - .get() - .status() + indexDoc(activeIndex, null, "keyword", "active", "@timestamp", "2021-05-12T20:07:12.112Z", "routing_field", "aaa").status() ); assertEquals(RestStatus.OK, indicesAdmin().prepareRefresh(idleIndex, activeIndex).get().getStatus()); @@ -352,8 +351,8 @@ public void testSearchIdleExistsQueryMatchOneIndex() throws InterruptedException "type=keyword" ); - assertEquals(RestStatus.CREATED, prepareIndex(idleIndex).setSource("keyword", "idle").get().status()); - assertEquals(RestStatus.CREATED, prepareIndex(activeIndex).setSource("keyword", "active", "unmapped", "bbb").get().status()); + assertEquals(RestStatus.CREATED, indexDoc(idleIndex, null, "keyword", "idle").status()); + assertEquals(RestStatus.CREATED, indexDoc(activeIndex, null, "keyword", "active", "unmapped", "bbb").status()); assertEquals(RestStatus.OK, indicesAdmin().prepareRefresh(idleIndex, activeIndex).get().getStatus()); waitUntil( @@ -396,4 +395,22 @@ private static void assertIdleShardsRefreshStats(final IndicesStatsResponse befo assertTrue(refreshStatsAfter.containsAll(refreshStatsBefore)); assertTrue(refreshStatsBefore.containsAll(refreshStatsAfter)); } + + private void index(String index, String id, String source, XContentType contentType) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source, contentType).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + protected final DocWriteResponse indexDoc(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java index a99f6c4340941..8d94a8d8d12f2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -162,7 +162,7 @@ public void testCorruptFileAndRecover() throws InterruptedException, IOException for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex("test").setSource("field", "value"); } - indexRandom(true, builders); + indexRandomAndDecrefRequests(true, builders); ensureGreen(); // double flush to create safe commit in case of async durability assertAllSuccessful(indicesAdmin().prepareFlush().setForce(true).get()); @@ -269,7 +269,7 @@ public void testCorruptPrimaryNoReplica() throws ExecutionException, Interrupted for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex("test").setSource("field", "value"); } - indexRandom(true, builders); + indexRandomAndDecrefRequests(true, builders); ensureGreen(); // double flush to create safe commit in case of async durability assertAllSuccessful(indicesAdmin().prepareFlush().setForce(true).get()); @@ -398,7 +398,7 @@ public void testCorruptionOnNetworkLayer() throws InterruptedException { for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex("test").setSource("field", "value"); } - indexRandom(true, builders); + indexRandomAndDecrefRequests(true, builders); ensureGreen(); assertAllSuccessful(indicesAdmin().prepareFlush().setForce(true).get()); // we have to flush at least once here since we don't corrupt the translog @@ -537,7 +537,7 @@ public void testCorruptFileThenSnapshotAndRestore() throws InterruptedException, for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex("test").setSource("field", "value"); } - indexRandom(true, builders); + indexRandomAndDecrefRequests(true, builders); ensureGreen(); assertAllSuccessful(indicesAdmin().prepareFlush().setForce(true).get()); // we have to flush at least once here since we don't corrupt the translog @@ -604,7 +604,7 @@ public void testReplicaCorruption() throws Exception { for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex("test").setSource("field", "value"); } - indexRandom(true, builders); + indexRandomAndDecrefRequests(true, builders); ensureGreen(); assertAllSuccessful(indicesAdmin().prepareFlush().setForce(true).get()); // we have to flush at least once here since we don't corrupt the translog @@ -762,4 +762,14 @@ private List getShuffledDataNodes() { return list; })); } + + private void indexRandomAndDecrefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java index 9618dcf761be9..207f7737080cc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java @@ -63,6 +63,9 @@ public void testCorruptTranslogFiles() throws Exception { } indexRandom(false, false, false, Arrays.asList(builders)); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } final Path translogPath = internalCluster().getInstance(IndicesService.class) .indexService(resolveIndex("test")) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java index b7b0c0bdddf8c..b65bfad966ba9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.TransportShardBulkAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.plugins.Plugin; @@ -91,7 +92,9 @@ public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, I for (int i = 0; i < numDocs; i++) { XContentBuilder doc = null; doc = jsonBuilder().startObject().field("foo", "bar").endObject(); - bulkBuilder.add(client.prepareIndex("index").setSource(doc)); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex("index").setSource(doc); + bulkBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } BulkResponse bulkResponse = bulkBuilder.get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java index e66357b5fbfe9..2df17affff38e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java @@ -48,9 +48,13 @@ public void testAutoGenerateIdNoDuplicates() throws Exception { logger.info("indexing [{}] docs", numOfDocs); List builders = new ArrayList<>(numOfDocs); for (int j = 0; j < numOfDocs; j++) { - builders.add(prepareIndex("test").setSource("field", "value_" + j)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource("field", "value_" + j); + builders.add(indexRequestBuilder); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } logger.info("verifying indexed content"); int numOfChecks = randomIntBetween(8, 12); for (int j = 0; j < numOfChecks; j++) { @@ -106,15 +110,15 @@ public void testCreatedFlag() throws Exception { createIndex("test"); ensureGreen(); - DocWriteResponse indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_1").get(); + DocWriteResponse indexResponse = indexDoc("test", "1", "field1", "value1_1"); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); - indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_2").get(); + indexResponse = indexDoc("test", "1", "field1", "value1_2"); assertEquals(DocWriteResponse.Result.UPDATED, indexResponse.getResult()); client().prepareDelete("test", "1").get(); - indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_2").get(); + indexResponse = indexDoc("test", "1", "field1", "value1_2"); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); } @@ -123,14 +127,14 @@ public void testCreatedFlagWithFlush() throws Exception { createIndex("test"); ensureGreen(); - DocWriteResponse indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_1").get(); + DocWriteResponse indexResponse = indexDoc("test", "1", "field1", "value1_1"); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); client().prepareDelete("test", "1").get(); flush(); - indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_2").get(); + indexResponse = indexDoc("test", "1", "field1", "value1_2"); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); } @@ -172,11 +176,12 @@ public void testCreatedFlagWithExternalVersioning() throws Exception { createIndex("test"); ensureGreen(); - DocWriteResponse indexResponse = prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource("field1", "value1_1") .setVersion(123) - .setVersionType(VersionType.EXTERNAL) - .get(); + .setVersionType(VersionType.EXTERNAL); + DocWriteResponse indexResponse = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); } @@ -184,7 +189,9 @@ public void testCreateFlagWithBulk() { createIndex("test"); ensureGreen(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add(prepareIndex("test").setId("1").setSource("field1", "value1_1")).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field1", "value1_1"); + BulkResponse bulkResponse = bulkRequestBuilder.add(indexRequestBuilder).get(); + indexRequestBuilder.request().decRef(); assertThat(bulkResponse.hasFailures(), equalTo(false)); assertThat(bulkResponse.getItems().length, equalTo(1)); IndexResponse indexResponse = bulkResponse.getItems()[0].getResponse(); @@ -207,7 +214,7 @@ public void testCreateIndexWithLongName() { } try { - prepareIndex(randomAlphaOfLengthBetween(min, max).toLowerCase(Locale.ROOT)).setSource("foo", "bar").get(); + indexDoc(randomAlphaOfLengthBetween(min, max).toLowerCase(Locale.ROOT), null, "foo", "bar"); fail("exception should have been thrown on too-long index name"); } catch (InvalidIndexNameException e) { assertThat( @@ -219,11 +226,14 @@ public void testCreateIndexWithLongName() { try { // Catch chars that are more than a single byte - prepareIndex( + indexDoc( randomAlphaOfLength(MetadataCreateIndexService.MAX_INDEX_NAME_BYTES - 1).toLowerCase(Locale.ROOT) + "Ϟ".toLowerCase( Locale.ROOT - ) - ).setSource("foo", "bar").get(); + ), + null, + "foo", + "bar" + ); fail("exception should have been thrown on too-long index name"); } catch (InvalidIndexNameException e) { assertThat( @@ -262,7 +272,7 @@ public void testInvalidIndexName() { } public void testDocumentWithBlankFieldName() { - Exception e = expectThrows(DocumentParsingException.class, prepareIndex("test").setId("1").setSource("", "value1_2")); + Exception e = expectThrows(DocumentParsingException.class, () -> indexDoc("test", "1", "", "value1_2")); assertThat(e.getMessage(), containsString("failed to parse")); assertThat(e.getCause().getMessage(), containsString("field name cannot be an empty string")); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java index 67e8d2fd75d65..87daa2116863c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.index.IndexNotFoundException; @@ -83,9 +84,15 @@ public void testIndexNameDateMathExpressions() { String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; String dateMathExp3 = "<.marvel-{now/d-2d}>"; - prepareIndex(dateMathExp1).setId("1").setSource("{}", XContentType.JSON).get(); - prepareIndex(dateMathExp2).setId("2").setSource("{}", XContentType.JSON).get(); - prepareIndex(dateMathExp3).setId("3").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(dateMathExp1).setId("1").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex(dateMathExp2).setId("2").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex(dateMathExp3).setId("3").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); dateSensitiveGet(prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3), response -> { @@ -147,9 +154,15 @@ public void testAutoCreateIndexWithDateMathExpression() { String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; String dateMathExp3 = "<.marvel-{now/d-2d}>"; - prepareIndex(dateMathExp1).setId("1").setSource("{}", XContentType.JSON).get(); - prepareIndex(dateMathExp2).setId("2").setSource("{}", XContentType.JSON).get(); - prepareIndex(dateMathExp3).setId("3").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(dateMathExp1).setId("1").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex(dateMathExp2).setId("2").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex(dateMathExp3).setId("3").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); dateSensitiveGet(prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3), response -> { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java index 1c715beb04356..881cd7d21d8e9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.indices; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -96,7 +97,9 @@ public void testDeletesAloneCanTriggerRefresh() throws Exception { IndexService indexService = createIndex("index", indexSettings(1, 0).put("index.refresh_interval", -1).build()); IndexShard shard = indexService.getShard(0); for (int i = 0; i < 100; i++) { - prepareIndex("index").setId(Integer.toString(i)).setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId(Integer.toString(i)).setSource("field", "value"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } // Force merge so we know all merges are done before we start deleting: BaseBroadcastResponse r = client().admin().indices().prepareForceMerge().setMaxNumSegments(1).get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index d1462ef8da3dc..20b2c27964d7b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; @@ -308,7 +309,9 @@ public void testWildcardBehaviour() throws Exception { verify(getSettings(indices).setIndicesOptions(options), false); assertAcked(prepareCreate("foobar")); - prepareIndex("foobar").setId("1").setSource("k", "v").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("foobar").setId("1").setSource("k", "v").setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); // Verify defaults for wildcards, with one wildcard expression and one existing index indices = new String[] { "foo*" }; @@ -394,7 +397,9 @@ public void testWildcardBehaviourSnapshotRestore() throws Exception { public void testAllMissingLenient() throws Exception { createIndex("test1"); - prepareIndex("test1").setId("1").setSource("k", "v").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test1").setId("1").setSource("k", "v").setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCount(prepareSearch("test2").setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(matchAllQuery()), 0L); assertHitCount(prepareSearch("test2", "test3").setQuery(matchAllQuery()).setIndicesOptions(IndicesOptions.lenientExpandOpen()), 0L); // you should still be able to run empty searches without things blowing up diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java index 62e6cb59994b2..f7171ed4994ec 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.indices; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.client.internal.Client; @@ -49,7 +50,7 @@ public void testCacheAggs() throws Exception { .setMapping("f", "type=date") .setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)) ); - indexRandom( + indexRandomAndDecRefRequests( true, client.prepareIndex("index").setSource("f", "2014-03-10T00:00:00.000Z"), client.prepareIndex("index").setSource("f", "2014-05-13T00:00:00.000Z") @@ -116,7 +117,7 @@ public void testQueryRewrite() throws Exception { .put("index.number_of_routing_shards", 5) ) ); - indexRandom( + indexRandomAndDecRefRequests( true, client.prepareIndex("index").setId("1").setRouting("1").setSource("s", "2016-03-19"), client.prepareIndex("index").setId("2").setRouting("1").setSource("s", "2016-03-20"), @@ -186,7 +187,7 @@ public void testQueryRewriteMissingValues() throws Exception { .setMapping("s", "type=date") .setSettings(indexSettings(1, 0).put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)) ); - indexRandom( + indexRandomAndDecRefRequests( true, client.prepareIndex("index").setId("1").setSource("s", "2016-03-19"), client.prepareIndex("index").setId("2").setSource("s", "2016-03-20"), @@ -253,7 +254,7 @@ public void testQueryRewriteDates() throws Exception { .setMapping("d", "type=date") .setSettings(indexSettings(1, 0).put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)) ); - indexRandom( + indexRandomAndDecRefRequests( true, client.prepareIndex("index").setId("1").setSource("d", "2014-01-01T00:00:00"), client.prepareIndex("index").setId("2").setSource("d", "2014-02-01T00:00:00"), @@ -325,7 +326,7 @@ public void testQueryRewriteDatesWithNow() throws Exception { assertAcked(indicesAdmin().prepareCreate("index-3").setMapping("d", "type=date").setSettings(settings).get()); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); DateFormatter formatter = DateFormatter.forPattern("strict_date_optional_time"); - indexRandom( + indexRandomAndDecRefRequests( true, client.prepareIndex("index-1").setId("1").setSource("d", formatter.format(now)), client.prepareIndex("index-1").setId("2").setSource("d", formatter.format(now.minusDays(1))), @@ -408,7 +409,7 @@ public void testCanCache() throws Exception { .put("index.number_of_routing_shards", 2) .build(); assertAcked(indicesAdmin().prepareCreate("index").setMapping("s", "type=date").setSettings(settings).get()); - indexRandom( + indexRandomAndDecRefRequests( true, client.prepareIndex("index").setId("1").setRouting("1").setSource("s", "2016-03-19"), client.prepareIndex("index").setId("2").setRouting("1").setSource("s", "2016-03-20"), @@ -527,7 +528,12 @@ public void testCacheWithFilteredAlias() { .addAlias(new Alias("last_week").filter(QueryBuilders.rangeQuery("created_at").gte("now-7d/d"))) ); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); - client.prepareIndex("index").setId("1").setRouting("1").setSource("created_at", DateTimeFormatter.ISO_LOCAL_DATE.format(now)).get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex("index") + .setId("1") + .setRouting("1") + .setSource("created_at", DateTimeFormatter.ISO_LOCAL_DATE.format(now)); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache BroadcastResponse forceMergeResponse = indicesAdmin().prepareForceMerge("index").setFlush(true).get(); ElasticsearchAssertions.assertAllSuccessful(forceMergeResponse); @@ -579,7 +585,7 @@ public void testProfileDisableCache() throws Exception { .setMapping("k", "type=keyword") .setSettings(indexSettings(1, 0).put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)) ); - indexRandom(true, client.prepareIndex("index").setSource("k", "hello")); + indexRandomAndDecRefRequests(true, client.prepareIndex("index").setSource("k", "hello")); ensureSearchable("index"); int expectedHits = 0; @@ -620,4 +626,13 @@ private static void assertCacheState(Client client, String index, long expectedH ); } + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/flush/FlushIT.java index 17b18bf9af1ee..116dcd72940b7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/flush/FlushIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.Settings; @@ -49,7 +50,9 @@ public void testWaitIfOngoing() throws InterruptedException { final int numIters = scaledRandomIntBetween(10, 30); for (int i = 0; i < numIters; i++) { for (int j = 0; j < 10; j++) { - prepareIndex("test").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } final CountDownLatch latch = new CountDownLatch(10); final CopyOnWriteArrayList errors = new CopyOnWriteArrayList<>(); @@ -87,7 +90,9 @@ public void testRejectIllegalFlushParameters() { createIndex("test"); int numDocs = randomIntBetween(0, 10); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } assertThat( expectThrows(ValidationException.class, indicesAdmin().flush(new FlushRequest().force(true).waitIfOngoing(false))).getMessage(), @@ -121,7 +126,7 @@ public void testFlushOnInactive() throws Exception { ensureGreen(indexName); int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { - prepareIndex(indexName).setSource("f", "v").get(); + indexDoc(indexName, null, "f", "v"); } if (randomBoolean()) { internalCluster().restartNode(randomFrom(dataNodes)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java index ec5d9876b7703..e9666f3a50864 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.core.Strings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -58,7 +59,8 @@ public void testConcurrentDynamicMapping() throws Exception { for (int j = 0; j < numDocs; j++) { Map source = new HashMap<>(); source.put(fieldName, "test-user"); - prepareIndex("test").setId(Integer.toString(currentID++)).setSource(source).execute(new ActionListener() { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(currentID++)).setSource(source); + indexRequestBuilder.execute(ActionListener.runAfter(new ActionListener<>() { @Override public void onResponse(DocWriteResponse response) { latch.countDown(); @@ -69,7 +71,7 @@ public void onFailure(Exception e) { throwable.add(e); latch.countDown(); } - }); + }, () -> indexRequestBuilder.request().decRef())); } latch.await(); assertThat(throwable, emptyIterable()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java index 937addb473f8b..c46a9eca5c7ec 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.indices.mapping; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -58,7 +59,9 @@ public void testBWCMalformedDynamicTemplate() { .put("index.version.created", IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0)) ).setMapping(mapping) ); - prepareIndex(indexName).setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setSource("{\"foo\" : \"bar\"}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNoFailures((indicesAdmin().prepareRefresh(indexName)).get()); assertHitCount(prepareSearch(indexName), 1); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index 70cd143686dc8..0532ed78e4240 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -80,6 +80,9 @@ public void testDynamicUpdates() throws Exception { ); } indexRandom(true, false, indexRequests); + for (IndexRequestBuilder indexRequestBuilder : indexRequests) { + indexRequestBuilder.request().decRef(); + } logger.info("checking all the documents are there"); BroadcastResponse refreshResponse = indicesAdmin().prepareRefresh().get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java index dd29823f8076f..414a0b67b9aa2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java @@ -49,6 +49,9 @@ public void testNoopRequestBreaker() throws Exception { reqs.add(client.prepareIndex("cb-test").setId(Long.toString(id)).setSource("test", id)); } indexRandom(true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } // A cardinality aggregation uses BigArrays and thus the REQUEST breaker client.prepareSearch("cb-test").setQuery(matchAllQuery()).addAggregation(cardinality("card").field("test")).get().decRef(); @@ -66,6 +69,9 @@ public void testNoopFielddataBreaker() throws Exception { reqs.add(client.prepareIndex("cb-test").setId(Long.toString(id)).setSource("test", id)); } indexRandom(true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } // Sorting using fielddata and thus the FIELDDATA breaker client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get().decRef(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java index 55851ef7b9402..856f66c7d5165 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -128,6 +128,9 @@ public void testMemoryBreaker() throws Exception { reqs.add(client.prepareIndex("cb-test").setId(Long.toString(id)).setSource("test", "value" + id)); } indexRandom(true, false, true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } // clear field data cache (thus setting the loaded field data back to 0) clearFieldData(); @@ -191,6 +194,9 @@ public void testRamAccountingTermsEnum() throws Exception { reqs.add(client.prepareIndex("ramtest").setId(Long.toString(id)).setSource("test", "value" + id)); } indexRandom(true, false, true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } // execute a search that loads field data (sorting on the "test" field) client.prepareSearch("ramtest").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get().decRef(); @@ -241,6 +247,9 @@ public void testRequestBreaker() throws Exception { reqs.add(client.prepareIndex("cb-test").setId(Long.toString(id)).setSource("test", id)); } indexRandom(true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } // A cardinality aggregation uses BigArrays and thus the REQUEST breaker try { @@ -274,6 +283,9 @@ public void testAggTookTooMuch() throws Exception { reqs.add(client.prepareIndex("cb-test").setId(Long.toString(id)).setSource("test", id)); } indexRandom(true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } // A terms aggregation on the "test" field should trip the bucket circuit breaker try { @@ -370,6 +382,7 @@ public void testLimitsRequestSize() { IndexRequest indexRequest = new IndexRequest("index").id(Integer.toString(i)); indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "num", i); bulkRequest.add(indexRequest); + indexRequest.decRef(); } updateClusterSettings( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java index 22f987cc855cc..07fed3a8521f1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastResponse; @@ -125,12 +126,15 @@ public void testBreakerWithRandomExceptions() throws IOException, InterruptedExc numDocs = between(10, 100); } for (int i = 0; i < numDocs; i++) { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); try { - prepareIndex("test").setId("" + i) + indexRequestBuilder.setId("" + i) .setTimeout(TimeValue.timeValueSeconds(1)) .setSource("test-str", randomUnicodeOfLengthBetween(5, 25), "test-num", i) .get(); - } catch (ElasticsearchException ex) {} + } catch (ElasticsearchException ex) {} finally { + indexRequestBuilder.request().decRef(); + } } logger.info("Start Refresh"); // don't assert on failures here diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java index 779072272e59a..ee6d4483c5cff 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java @@ -41,11 +41,11 @@ public void testPrimaryRelocationWhileIndexing() throws Exception { @Override public void run() { while (finished.get() == false && numAutoGenDocs.get() < 10_000) { - DocWriteResponse indexResponse = prepareIndex("test").setId("id").setSource("field", "value").get(); + DocWriteResponse indexResponse = indexDoc("test", "id", "field", "value"); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); DeleteResponse deleteResponse = client().prepareDelete("test", "id").get(); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); - prepareIndex("test").setSource("auto", true).get(); + indexDoc("test", null, "auto", true); numAutoGenDocs.incrementAndGet(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index bd400f9f0f6a1..616908eee0931 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -467,6 +467,7 @@ public void testCancelNewShardRecoveryAndUsesExistingShardCopy() throws Exceptio ); } indexRandom(randomBoolean(), docs); + decRefIndexRequests(docs); logger.info("--> start node B"); // force a shard recovery from nodeA to nodeB @@ -936,6 +937,7 @@ private IndicesStatsResponse createAndPopulateIndex(String name, int nodeCount, } indexRandom(true, docs); + decRefIndexRequests(docs); flush(); assertHitCount(prepareSearch(name).setSize(0), numDocs); return indicesAdmin().prepareStats(name).get(); @@ -1000,6 +1002,7 @@ public void testHistoryRetention() throws Exception { requests.add(prepareIndex(indexName).setSource("{}", XContentType.JSON)); } indexRandom(true, requests); + decRefIndexRequests(requests); if (randomBoolean()) { flush(indexName); } @@ -1019,7 +1022,10 @@ public void testHistoryRetention() throws Exception { final int numNewDocs = scaledRandomIntBetween(25, 250); for (int i = 0; i < numNewDocs; i++) { - prepareIndex(indexName).setSource("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setSource("{}", XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } // Flush twice to update the safe commit's local checkpoint assertThat(indicesAdmin().prepareFlush(indexName).setForce(true).execute().get().getFailedShards(), equalTo(0)); @@ -1051,7 +1057,10 @@ public void testDoNotInfinitelyWaitForMapping() { indicesAdmin().preparePutMapping("test").setSource("test_field", "type=text,analyzer=test_analyzer").get(); int numDocs = between(1, 10); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setId("u" + i).setSource(singletonMap("test_field", Integer.toString(i)), XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("u" + i) + .setSource(singletonMap("test_field", Integer.toString(i)), XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } Semaphore recoveryBlocked = new Semaphore(1); for (DiscoveryNode node : clusterService().state().nodes()) { @@ -1147,12 +1156,11 @@ public void testRecoverLocallyUpToGlobalCheckpoint() throws Exception { ); ensureGreen(indexName); int numDocs = randomIntBetween(0, 100); - indexRandom( - randomBoolean(), - false, - randomBoolean(), - IntStream.range(0, numDocs).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + List indexRequestBuilders = IntStream.range(0, numDocs) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), false, randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); indicesAdmin().prepareRefresh(indexName).get(); // avoid refresh when we are failing a shard String failingNode = randomFrom(nodes); PlainActionFuture startRecoveryRequestFuture = new PlainActionFuture<>(); @@ -1238,12 +1246,11 @@ public void testUsesFileBasedRecoveryIfRetentionLeaseMissing() throws Exception .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "12h") .build() ); - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, between(0, 100)).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + List indexRequestBuilders = IntStream.range(0, between(0, 100)) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); ensureGreen(indexName); final ShardId shardId = new ShardId(resolveIndex(indexName), 0); @@ -1302,12 +1309,11 @@ public void testUsesFileBasedRecoveryIfRetentionLeaseAheadOfGlobalCheckpoint() t .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "12h") .build() ); - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, between(0, 100)).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + List indexRequestBuilders = IntStream.range(0, between(0, 100)) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); ensureGreen(indexName); final ShardId shardId = new ShardId(resolveIndex(indexName), 0); @@ -1333,12 +1339,11 @@ public Settings onNodeStopped(String nodeName) throws Exception { .isTimedOut() ); - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, between(1, 100)).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + List indexRequestBuilders = IntStream.range(0, between(0, 100)) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); // We do not guarantee that the replica can recover locally all the way to its own global checkpoint before starting // to recover from the primary, so we must be careful not to perform an operations-based recovery if this would require @@ -1386,12 +1391,11 @@ public void testUsesFileBasedRecoveryIfOperationsBasedRecoveryWouldBeUnreasonabl logger.info("--> performing ops-based recoveries up to [{}%] of docs", reasonableOperationsBasedRecoveryProportion * 100.0); createIndex(indexName, settings.build()); - indexRandom( - randomBoolean(), - false, - randomBoolean(), - IntStream.range(0, between(0, 100)).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + List indexRequestBuilders = IntStream.range(0, between(0, 100)) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), false, randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); ensureGreen(indexName); flush(indexName); @@ -1466,13 +1470,11 @@ public Settings onNodeStopped(String nodeName) throws Exception { * * ==> it is unreasonable to recover the replica using a seqno-based recovery */ - - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, newDocCount).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + List indexRequestBuilders = IntStream.range(0, newDocCount) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); flush(indexName); @@ -1509,12 +1511,11 @@ public void testDoesNotCopyOperationsInSafeCommit() throws Exception { String indexName = "test-index"; createIndex(indexName, indexSettings(1, 0).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build()); - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, between(0, 100)).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + List indexRequestBuilders = IntStream.range(0, between(0, 100)) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); final ShardId shardId = new ShardId(resolveIndex(indexName), 0); final DiscoveryNodes discoveryNodes = clusterService().state().nodes(); @@ -1528,12 +1529,11 @@ public void testDoesNotCopyOperationsInSafeCommit() throws Exception { assertBusy(() -> assertThat(primary.getLastSyncedGlobalCheckpoint(), equalTo(maxSeqNoBeforeRecovery))); assertThat(indicesAdmin().prepareFlush(indexName).get().getFailedShards(), is(0)); // makes a safe commit - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, between(0, 100)).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + indexRequestBuilders = IntStream.range(0, between(0, 100)) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); setReplicaCount(1, indexName); ensureGreen(indexName); @@ -1585,12 +1585,11 @@ public void testRepeatedRecovery() throws Exception { indexName, indexSettings(randomIntBetween(1, 6), 1).put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "200ms").build() ); - indexRandom( - randomBoolean(), - false, - randomBoolean(), - IntStream.range(0, randomIntBetween(0, 10)).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + List indexRequestBuilders = IntStream.range(0, between(0, 10)) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), false, randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); assertThat(indicesAdmin().prepareFlush(indexName).get().getFailedShards(), equalTo(0)); @@ -1613,12 +1612,11 @@ public void testRepeatedRecovery() throws Exception { ensureGreen(indexName); logger.info("--> index more documents"); - indexRandom( - randomBoolean(), - false, - randomBoolean(), - IntStream.range(0, randomIntBetween(0, 10)).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + indexRequestBuilders = IntStream.range(0, between(0, 10)) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), false, randomBoolean(), indexRequestBuilders); + decRefIndexRequests(indexRequestBuilders); logger.info("--> add replicas again"); setReplicaCount(1, indexName); @@ -1638,6 +1636,7 @@ public void testAllocateEmptyPrimaryResetsGlobalCheckpoint() throws Exception { .mapToObj(n -> prepareIndex(indexName).setSource("foo", "bar")) .toList(); indexRandom(randomBoolean(), true, true, indexRequests); + decRefIndexRequests(indexRequests); ensureGreen(); internalCluster().stopRandomDataNode(); internalCluster().stopRandomDataNode(); @@ -1680,13 +1679,14 @@ public void testPeerRecoveryTrimsLocalTranslog() throws Exception { for (int i = 0; i < indexers.length; i++) { indexers[i] = new Thread(() -> { while (stopped.get() == false) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName); try { - DocWriteResponse response = prepareIndex(indexName).setSource( - Map.of("f" + randomIntBetween(1, 10), randomNonNegativeLong()), - XContentType.JSON - ).get(); + indexRequestBuilder.setSource(Map.of("f" + randomIntBetween(1, 10), randomNonNegativeLong()), XContentType.JSON); + DocWriteResponse response = indexRequestBuilder.get(); assertThat(response.getResult(), is(oneOf(CREATED, UPDATED))); - } catch (ElasticsearchException ignored) {} + } catch (ElasticsearchException ignored) {} finally { + indexRequestBuilder.request().decRef(); + } } }); } @@ -1730,6 +1730,7 @@ public void testReservesBytesDuringPeerRecoveryPhaseOne() throws Exception { .mapToObj(n -> prepareIndex(indexName).setSource("foo", "bar")) .toList(); indexRandom(randomBoolean(), true, true, indexRequests); + decRefIndexRequests(indexRequests); assertThat(indicesAdmin().prepareFlush(indexName).get().getFailedShards(), equalTo(0)); ClusterState clusterState = clusterAdmin().prepareState().get().getState(); @@ -1803,6 +1804,7 @@ public void testWaitForClusterStateToBeAppliedOnSourceNode() throws Exception { .mapToObj(n -> prepareIndex(indexName).setSource("foo", "bar")) .toList(); indexRandom(randomBoolean(), true, true, indexRequests); + decRefIndexRequests(indexRequests); assertThat(indicesAdmin().prepareFlush(indexName).get().getFailedShards(), equalTo(0)); final var replicaNode = internalCluster().startDataOnlyNode(); @@ -1873,6 +1875,7 @@ public void testDeleteIndexDuringFinalization() throws Exception { .mapToObj(n -> prepareIndex(indexName).setSource("foo", "bar")) .toList(); indexRandom(randomBoolean(), true, true, indexRequests); + decRefIndexRequests(indexRequests); assertThat(indicesAdmin().prepareFlush(indexName).get().getFailedShards(), equalTo(0)); final var replicaNode = internalCluster().startDataOnlyNode(); @@ -1900,7 +1903,8 @@ public void testDeleteIndexDuringFinalization() throws Exception { // Process the TRANSLOG_OPS response on the replica (avoiding failing it due to a concurrent delete) but // before sending the response back send another document to the primary, advancing the GCP to prevent the replica // being marked as in-sync (NB below we delay the replica write until after the index is deleted) - prepareIndex(indexName).setSource("foo", "baz").execute(ActionListener.noop()); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setSource("foo", "baz"); + indexRequestBuilder.execute(ActionListener.running(() -> indexRequestBuilder.request().decRef())); primaryIndexShard.addGlobalCheckpointListener( globalCheckpointBeforeRecovery + 1, @@ -2052,4 +2056,16 @@ private long getLocalCheckpointOfSafeCommit(IndexCommit safeIndexCommit) throws return localCheckpointTracker.getPersistedCheckpoint(); } } + + private void decRefIndexRequests(IndexRequestBuilder[] indexRequestBuilders) { + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } + } + + private void decRefIndexRequests(List indexRequestBuilders) { + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java index 56fffa682a36d..5879a343b9611 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java @@ -52,9 +52,7 @@ public void testSimpleUpdateNumberOfReplicas() throws Exception { assertThat(clusterHealth.getIndices().get("test").getActiveShards(), equalTo(numShards.totalNumShards)); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)) - .setSource(jsonBuilder().startObject().field("value", "test" + i).endObject()) - .get(); + index("test", Integer.toString(i), jsonBuilder().startObject().field("value", "test" + i).endObject()); } refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java index 165b776f1ebc1..c42c75edb553f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java @@ -8,9 +8,9 @@ package org.elasticsearch.indices.settings; -import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; @@ -375,14 +375,18 @@ public void testOpenCloseUpdateSettings() throws Exception { public void testEngineGCDeletesSetting() throws Exception { createIndex("test"); - prepareIndex("test").setId("1").setSource("f", 1).setVersionType(VersionType.EXTERNAL).setVersion(1).get(); - client().prepareDelete("test", "1").setVersionType(VersionType.EXTERNAL).setVersion(2).get(); - // delete is still in cache this should fail - ActionRequestBuilder builder = prepareIndex("test").setId("1") - .setSource("f", 3) + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setSource("f", 1) .setVersionType(VersionType.EXTERNAL) .setVersion(1); - expectThrows(VersionConflictEngineException.class, builder); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + client().prepareDelete("test", "1").setVersionType(VersionType.EXTERNAL).setVersion(2).get(); + // delete is still in cache this should fail + indexRequestBuilder = prepareIndex("test").setId("1").setSource("f", 3).setVersionType(VersionType.EXTERNAL).setVersion(1); + ; + expectThrows(VersionConflictEngineException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertAcked(indicesAdmin().prepareUpdateSettings("test").setSettings(Settings.builder().put("index.gc_deletes", 0))); @@ -395,7 +399,9 @@ public void testEngineGCDeletesSetting() throws Exception { } // delete should not be in cache - prepareIndex("test").setId("1").setSource("f", 2).setVersionType(VersionType.EXTERNAL).setVersion(1); + indexRequestBuilder = prepareIndex("test").setId("1").setSource("f", 2).setVersionType(VersionType.EXTERNAL).setVersion(1); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } public void testUpdateSettingsWithBlocks() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java index 77cdc2e99977d..8d9dea1d7cdcb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -113,7 +114,7 @@ public void testCloseIndex() throws Exception { createIndex(indexName); final int nbDocs = randomIntBetween(0, 50); - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), false, randomBoolean(), @@ -132,7 +133,7 @@ public void testCloseAlreadyClosedIndex() throws Exception { createIndex(indexName); if (randomBoolean()) { - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), false, randomBoolean(), @@ -175,7 +176,7 @@ public void testConcurrentClose() throws InterruptedException { createIndex(indexName); final int nbDocs = randomIntBetween(10, 50); - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), false, randomBoolean(), @@ -243,7 +244,7 @@ public void testCloseWhileDeletingIndices() throws Exception { final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); createIndex(indexName); if (randomBoolean()) { - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), false, randomBoolean(), @@ -347,7 +348,7 @@ public void testCloseIndexWaitForActiveShards() throws Exception { createIndex(indexName, 2, 0); final int nbDocs = randomIntBetween(0, 50); - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), false, randomBoolean(), @@ -372,7 +373,7 @@ public void testNoopPeerRecoveriesWhenIndexClosed() throws Exception { createIndex(indexName, indexSettings(1, numberOfReplicas).put("index.routing.rebalance.enable", "none").build()); int iterations = between(1, 3); for (int iter = 0; iter < iterations; iter++) { - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -407,7 +408,7 @@ public void testRecoverExistingReplica() throws Exception { clusterService().state().nodes().getDataNodes().values().stream().map(DiscoveryNode::getName).collect(Collectors.toSet()) ); createIndex(indexName, indexSettings(1, 1).put("index.routing.allocation.include._name", String.join(",", dataNodes)).build()); - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -423,7 +424,9 @@ public Settings onNodeStopped(String nodeName) throws Exception { Client client = client(dataNodes.get(0)); int moreDocs = randomIntBetween(1, 50); for (int i = 0; i < moreDocs; i++) { - client.prepareIndex(indexName).setSource("num", i).get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName).setSource("num", i); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } assertAcked(indicesAdmin().prepareClose(indexName)); return super.onNodeStopped(nodeName); @@ -448,7 +451,7 @@ public void testRelocatedClosedIndexIssue() throws Exception { final List dataNodes = internalCluster().startDataOnlyNodes(2); // allocate shard to first data node createIndex(indexName, indexSettings(1, 0).put("index.routing.allocation.include._name", dataNodes.get(0)).build()); - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -467,7 +470,7 @@ public void testResyncPropagatePrimaryTerm() throws Exception { internalCluster().ensureAtLeastNumDataNodes(3); final String indexName = "closed_indices_promotion"; createIndex(indexName, 1, 2); - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -496,7 +499,7 @@ public void testSearcherId() throws Exception { final String indexName = "test_commit_id"; final int numberOfShards = randomIntBetween(1, 5); createIndex(indexName, numberOfShards, 0); - indexRandom( + indexRandomAndDecrefRequests( randomBoolean(), randomBoolean(), randomBoolean(), @@ -634,4 +637,19 @@ void assertNoFileBasedRecovery(String indexName) { } } } + + private void indexRandomAndDecrefRequests( + boolean forceRefresh, + boolean dummyDocuments, + boolean maybeFlush, + List builders + ) throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, maybeFlush, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java index b66a0b0f3be44..94eeda3785656 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.indices.state; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -93,10 +94,13 @@ public void testCloseWhileRelocatingShards() throws Exception { nbDocs = scaledRandomIntBetween(1, 100); logger.debug("creating index {} with {} documents", indexName, nbDocs); createIndex(indexName); - indexRandom( - randomBoolean(), - IntStream.range(0, nbDocs).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).toList() - ); + List builders = IntStream.range(0, nbDocs) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .toList(); + indexRandom(randomBoolean(), builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } } default -> { logger.debug("creating index {} with background indexing", indexName); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java index 61bb48b7f7583..eab8bc729f7e5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java @@ -259,11 +259,14 @@ public void testOpenCloseWithDocs() throws IOException, ExecutionException, Inte assertAcked(indicesAdmin().prepareCreate("test").setMapping(mapping)); ensureGreen(); int docs = between(10, 100); - IndexRequestBuilder[] builder = new IndexRequestBuilder[docs]; + IndexRequestBuilder[] builders = new IndexRequestBuilder[docs]; for (int i = 0; i < docs; i++) { - builder[i] = prepareIndex("test").setId("" + i).setSource("test", "init"); + builders[i] = prepareIndex("test").setId("" + i).setSource("test", "init"); + } + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); } - indexRandom(true, builder); if (randomBoolean()) { indicesAdmin().prepareFlush("test").setForce(true).execute().get(); } @@ -281,7 +284,7 @@ public void testOpenCloseIndexWithBlocks() { int docs = between(10, 100); for (int i = 0; i < docs; i++) { - prepareIndex("test").setId("" + i).setSource("test", "init").get(); + indexDoc("test", "" + i, "test", "init"); } for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE)) { @@ -337,7 +340,7 @@ public void testTranslogStats() throws Exception { final int nbDocs = randomIntBetween(0, 50); int uncommittedOps = 0; for (long i = 0; i < nbDocs; i++) { - final DocWriteResponse indexResponse = prepareIndex(indexName).setId(Long.toString(i)).setSource("field", i).get(); + final DocWriteResponse indexResponse = indexDoc(indexName, Long.toString(i), "field", i); assertThat(indexResponse.status(), is(RestStatus.CREATED)); if (rarely()) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java index b5448498f0ce9..b9cc31b99bc8b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java @@ -43,7 +43,7 @@ public void testSimpleOpenClose() { ); logger.info("--> indexing a simple document"); - prepareIndex("test").setId("1").setSource("field1", "value1").get(); + indexDoc("test", "1", "field1", "value1"); logger.info("--> closing test index..."); assertAcked(indicesAdmin().prepareClose("test")); @@ -54,7 +54,7 @@ public void testSimpleOpenClose() { logger.info("--> trying to index into a closed index ..."); try { - prepareIndex("test").setId("1").setSource("field1", "value1").get(); + indexDoc("test", "1", "field1", "value1"); fail(); } catch (IndexClosedException e) { // all is well @@ -76,7 +76,7 @@ public void testSimpleOpenClose() { ); logger.info("--> indexing a simple document"); - prepareIndex("test").setId("1").setSource("field1", "value1").get(); + indexDoc("test", "1", "field1", "value1"); } public void testFastCloseAfterCreateContinuesCreateAfterOpen() { @@ -111,7 +111,7 @@ public void testFastCloseAfterCreateContinuesCreateAfterOpen() { ); logger.info("--> indexing a simple document"); - prepareIndex("test").setId("1").setSource("field1", "value1").get(); + indexDoc("test", "1", "field1", "value1"); } public void testConsistencyAfterIndexCreationFailure() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 0b386de16ebbf..d246d6cdfdaf1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -134,8 +134,8 @@ public void testFieldDataStats() { .setMapping("field", "type=text,fielddata=true", "field2", "type=text,fielddata=true") ); ensureGreen(); - prepareIndex("test").setId("1").setSource("field", "value1", "field2", "value1").get(); - prepareIndex("test").setId("2").setSource("field", "value2", "field2", "value2").get(); + indexDoc("test", "1", "field", "value1", "field2", "value1"); + indexDoc("test", "2", "field", "value2", "field2", "value2"); indicesAdmin().prepareRefresh().get(); NodesStatsResponse nodesStats = clusterAdmin().prepareNodesStats("data:true").setIndices(true).get(); @@ -238,8 +238,8 @@ public void testClearAllCaches() throws Exception { ); ensureGreen(); clusterAdmin().prepareHealth().setWaitForGreenStatus().get(); - prepareIndex("test").setId("1").setSource("field", "value1").get(); - prepareIndex("test").setId("2").setSource("field", "value2").get(); + indexDoc("test", "1", "field", "value1"); + indexDoc("test", "2", "field", "value2"); indicesAdmin().prepareRefresh().get(); NodesStatsResponse nodesStats = clusterAdmin().prepareNodesStats("data:true").setIndices(true).get(); @@ -334,6 +334,9 @@ public void testQueryCache() throws Exception { .setSource(jsonBuilder().startObject().field("common", "field").field("str_value", "s" + i).endObject()); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } numDocs += pageDocs; boolean allHaveDocs = true; @@ -378,6 +381,9 @@ public void testQueryCache() throws Exception { .setSource(jsonBuilder().startObject().field("common", "field").field("str_value", "s" + i).endObject()); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } refresh(); assertBusy(() -> { assertThat( @@ -454,7 +460,7 @@ public void testNonThrottleStats() throws Exception { sb.append(termUpto++); sb.append(" some random text that keeps repeating over and over again hambone"); } - prepareIndex("test").setId("" + termUpto).setSource("field" + (i % 10), sb.toString()).get(); + indexDoc("test", "" + termUpto, "field" + (i % 10), sb.toString()); } refresh(); stats = indicesAdmin().prepareStats().get(); @@ -490,7 +496,7 @@ public void testThrottleStats() throws Exception { sb.append(' '); sb.append(termUpto++); } - prepareIndex("test").setId("" + termUpto).setSource("field" + (i % 10), sb.toString()).get(); + indexDoc("test", "" + termUpto, "field" + (i % 10), sb.toString()); if (i % 2 == 0) { refresh(); } @@ -516,9 +522,9 @@ public void testSimpleStats() throws Exception { createIndex("test1", "test2"); ensureGreen(); - prepareIndex("test1").setId(Integer.toString(1)).setSource("field", "value").get(); - prepareIndex("test1").setId(Integer.toString(2)).setSource("field", "value").get(); - prepareIndex("test2").setId(Integer.toString(1)).setSource("field", "value").get(); + indexDoc("test1", Integer.toString(1), "field", "value"); + indexDoc("test1", Integer.toString(2), "field", "value"); + indexDoc("test2", Integer.toString(1), "field", "value"); refresh(); NumShards test1 = getNumShards("test1"); @@ -607,27 +613,15 @@ public void testSimpleStats() throws Exception { // index failed try { - prepareIndex("test1").setId(Integer.toString(1)) - .setSource("field", "value") - .setVersion(1) - .setVersionType(VersionType.EXTERNAL) - .get(); + indexDocWithExternalVersion("test1", Integer.toString(1), 1, "field", "value"); fail("Expected a version conflict"); } catch (VersionConflictEngineException e) {} try { - prepareIndex("test2").setId(Integer.toString(1)) - .setSource("field", "value") - .setVersion(1) - .setVersionType(VersionType.EXTERNAL) - .get(); + indexDocWithExternalVersion("test2", Integer.toString(1), 1, "field", "value"); fail("Expected a version conflict"); } catch (VersionConflictEngineException e) {} try { - prepareIndex("test2").setId(Integer.toString(1)) - .setSource("field", "value") - .setVersion(1) - .setVersionType(VersionType.EXTERNAL) - .get(); + indexDocWithExternalVersion("test2", Integer.toString(1), 1, "field", "value"); fail("Expected a version conflict"); } catch (VersionConflictEngineException e) {} @@ -659,7 +653,7 @@ public void testMergeStats() { assertThat(stats.getTotal().getSearch(), nullValue()); for (int i = 0; i < 20; i++) { - prepareIndex("test_index").setId(Integer.toString(i)).setSource("field", "value").get(); + indexDoc("test_index", Integer.toString(i), "field", "value"); indicesAdmin().prepareFlush().get(); } indicesAdmin().prepareForceMerge().setMaxNumSegments(1).get(); @@ -717,9 +711,9 @@ public void testAllFlags() throws Exception { ensureGreen(); - prepareIndex("test_index").setId(Integer.toString(1)).setSource("field", "value").get(); - prepareIndex("test_index").setId(Integer.toString(2)).setSource("field", "value").get(); - prepareIndex("test_index_2").setId(Integer.toString(1)).setSource("field", "value").get(); + indexDoc("test_index", Integer.toString(1), "field", "value"); + indexDoc("test_index", Integer.toString(2), "field", "value"); + indexDoc("test_index_2", Integer.toString(1), "field", "value"); indicesAdmin().prepareRefresh().get(); IndicesStatsRequestBuilder builder = indicesAdmin().prepareStats(); @@ -848,9 +842,9 @@ public void testMultiIndex() throws Exception { ensureGreen(); - prepareIndex("test1").setId(Integer.toString(1)).setSource("field", "value").get(); - prepareIndex("test1").setId(Integer.toString(2)).setSource("field", "value").get(); - prepareIndex("test2").setId(Integer.toString(1)).setSource("field", "value").get(); + indexDoc("test1", Integer.toString(1), "field", "value"); + indexDoc("test1", Integer.toString(2), "field", "value"); + indexDoc("test2", Integer.toString(1), "field", "value"); refresh(); int numShards1 = getNumShards("test1").totalNumShards; @@ -905,8 +899,10 @@ public void testCompletionFieldsParam() throws Exception { }""")); ensureGreen(); - prepareIndex("test1").setId(Integer.toString(1)).setSource(""" - {"bar":"bar","baz":"baz"}""", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test1").setId(Integer.toString(1)).setSource(""" + {"bar":"bar","baz":"baz"}""", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); IndicesStatsRequestBuilder builder = indicesAdmin().prepareStats(); @@ -948,7 +944,7 @@ public void testGroupsParam() throws Exception { ensureGreen(); - prepareIndex("test1").setId(Integer.toString(1)).setSource("foo", "bar").get(); + indexDoc("test1", Integer.toString(1), "foo", "bar"); refresh(); prepareSearch("_all").setStats("bar", "baz").get().decRef(); @@ -1084,12 +1080,16 @@ public void testFilterCacheStats() throws Exception { .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "200ms") .build(); assertAcked(prepareCreate("index").setSettings(settings).get()); - indexRandom( - false, - true, - prepareIndex("index").setId("1").setSource("foo", "bar"), - prepareIndex("index").setId("2").setSource("foo", "baz") - ); + { + List builders = List.of( + prepareIndex("index").setId("1").setSource("foo", "bar"), + prepareIndex("index").setId("2").setSource("foo", "baz") + ); + indexRandom(false, true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } persistGlobalCheckpoint("index"); // Need to persist the global checkpoint for the soft-deletes retention MP. refresh(); ensureGreen(); @@ -1151,11 +1151,16 @@ public void testFilterCacheStats() throws Exception { assertThat(response.getTotal().queryCache.getCacheSize(), equalTo(0L)); assertThat(response.getTotal().queryCache.getCacheCount(), greaterThan(0L)); - indexRandom( - true, - prepareIndex("index").setId("1").setSource("foo", "bar"), - prepareIndex("index").setId("2").setSource("foo", "baz") - ); + { + List builders = List.of( + prepareIndex("index").setId("1").setSource("foo", "bar"), + prepareIndex("index").setId("2").setSource("foo", "baz") + ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } assertBusy(() -> { assertNoFailures(prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz")))); @@ -1186,8 +1191,9 @@ public void testBulkStats() throws Exception { ensureGreen(); try (BulkRequest request1 = new BulkRequest()) { for (int i = 0; i < 20; ++i) { - request1.add(new IndexRequest(index).source(Collections.singletonMap("key", "value" + i))) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexRequest indexRequest = new IndexRequest(index).source(Collections.singletonMap("key", "value" + i)); + request1.add(indexRequest).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequest.decRef(); } BulkResponse bulkResponse = client().bulk(request1).get(); assertThat(bulkResponse.hasFailures(), equalTo(false)); @@ -1252,7 +1258,9 @@ public void testConcurrentIndexingAndStatsRequests() throws BrokenBarrierExcepti } while (stop.get() == false) { final String id = Integer.toString(idGenerator.incrementAndGet()); - final DocWriteResponse response = prepareIndex("test").setId(id).setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(id).setSource("{}", XContentType.JSON); + final DocWriteResponse response = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED)); } }); @@ -1322,17 +1330,20 @@ public void testWriteLoadIsCaptured() throws Exception { assertBusy(() -> { final int numDocs = randomIntBetween(15, 25); final List> indexRequestFutures = new ArrayList<>(numDocs); + List buildersToDecRef = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { - indexRequestFutures.add( - prepareIndex(indexName).setId(Integer.toString(idGenerator.incrementAndGet())) - .setSource("{}", XContentType.JSON) - .execute() - ); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId(Integer.toString(idGenerator.incrementAndGet())) + .setSource("{}", XContentType.JSON); + buildersToDecRef.add(indexRequestBuilder); + indexRequestFutures.add(indexRequestBuilder.execute()); } for (ActionFuture indexRequestFuture : indexRequestFutures) { assertThat(indexRequestFuture.get().getResult(), equalTo(DocWriteResponse.Result.CREATED)); } + for (IndexRequestBuilder builder : buildersToDecRef) { + builder.request().decRef(); + } final IndicesStatsResponse statsResponseAfterIndexing = indicesAdmin().prepareStats(indexName).get(); final IndexStats indexStatsAfterIndexing = statsResponseAfterIndexing.getIndices().get(indexName); @@ -1357,4 +1368,13 @@ private void persistGlobalCheckpoint(String index) throws Exception { } } } + + private DocWriteResponse indexDocWithExternalVersion(String index, String id, long version, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setVersion(version).setVersionType(VersionType.EXTERNAL).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index b28d5a8af9e1d..7ec43a094835f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -18,6 +19,7 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.common.ParsingException; @@ -148,7 +150,7 @@ public void testSimpleIndexTemplateTests() throws Exception { assertThat(response.getIndexTemplates(), hasSize(2)); // index something into test_index, will match on both templates - prepareIndex("test_index").setId("1").setSource("field1", "value1", "field2", "value 2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test_index", "1", "field1", "value1", "field2", "value 2"); ensureGreen(); assertResponse( @@ -161,7 +163,7 @@ public void testSimpleIndexTemplateTests() throws Exception { } ); - prepareIndex("text_index").setId("1").setSource("field1", "value1", "field2", "value 2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("text_index", "1", "field1", "value1", "field2", "value 2"); ensureGreen(); // now only match on one template (template_1) @@ -484,11 +486,11 @@ public void testIndexTemplateWithAliases() throws Exception { assertAcked(prepareCreate("test_index")); ensureGreen(); - prepareIndex("test_index").setId("1").setSource("type", "type1", "field", "A value").get(); - prepareIndex("test_index").setId("2").setSource("type", "type2", "field", "B value").get(); - prepareIndex("test_index").setId("3").setSource("type", "typeX", "field", "C value").get(); - prepareIndex("test_index").setId("4").setSource("type", "typeY", "field", "D value").get(); - prepareIndex("test_index").setId("5").setSource("type", "typeZ", "field", "E value").get(); + indexDoc("test_index", "1", "type", "type1", "field", "A value"); + indexDoc("test_index", "2", "type", "type2", "field", "B value"); + indexDoc("test_index", "3", "type", "typeX", "field", "C value"); + indexDoc("test_index", "4", "type", "typeY", "field", "D value"); + indexDoc("test_index", "5", "type", "typeZ", "field", "E value"); GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases().setIndices("test_index").get(); assertThat(getAliasesResponse.getAliases().size(), equalTo(1)); @@ -540,8 +542,8 @@ public void testIndexTemplateWithAliasesInSource() { assertThat(getAliasesResponse.getAliases().size(), equalTo(1)); assertThat(getAliasesResponse.getAliases().get("test_index").size(), equalTo(1)); - prepareIndex("test_index").setId("1").setSource("field", "value1").get(); - prepareIndex("test_index").setId("2").setSource("field", "value2").get(); + indexDoc("test_index", "1", "field", "value1"); + indexDoc("test_index", "2", "field", "value2"); refresh(); assertHitCount(prepareSearch("test_index"), 2L); @@ -576,8 +578,8 @@ public void testIndexTemplateWithAliasesSource() { assertThat(getAliasesResponse.getAliases().size(), equalTo(1)); assertThat(getAliasesResponse.getAliases().get("test_index").size(), equalTo(3)); - prepareIndex("test_index").setId("1").setSource("field", "value1").get(); - prepareIndex("test_index").setId("2").setSource("field", "value2").get(); + indexDoc("test_index", "1", "field", "value1"); + indexDoc("test_index", "2", "field", "value2"); refresh(); assertHitCount(prepareSearch("test_index"), 2L); @@ -716,9 +718,13 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio .addAlias(new Alias("alias4").filter(termQuery("field", "value"))) .get(); - client().prepareIndex("a1").setId("test").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = client().prepareIndex("a1").setId("test").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse response = bulkRequestBuilder.add(new IndexRequest("a2").id("test").source("{}", XContentType.JSON)).get(); + IndexRequest indexRequest = new IndexRequest("a2").id("test").source("{}", XContentType.JSON); + BulkResponse response = bulkRequestBuilder.add(indexRequest).get(); + indexRequest.decRef(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getIndex(), equalTo("a2")); @@ -734,10 +740,14 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio // So the aliases defined in the index template for this index will not fail // even though the fields in the alias fields don't exist yet and indexing into // an index that doesn't exist yet will succeed - prepareIndex("b1").setId("test").setSource("{}", XContentType.JSON).get(); + indexRequestBuilder = prepareIndex("b1").setId("test").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse response = bulkRequestBuilder.add(new IndexRequest("b2").id("test").source("{}", XContentType.JSON)).get(); + IndexRequest indexRequest = new IndexRequest("b2").id("test").source("{}", XContentType.JSON); + BulkResponse response = bulkRequestBuilder.add(indexRequest).get(); + indexRequest.decRef(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getId(), equalTo("test")); @@ -838,9 +848,9 @@ public void testMultipleTemplate() throws IOException { ) .get(); - prepareIndex("ax").setId("1").setSource("field1", "value1", "field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("ax", "1", "field1", "value1", "field2", "value2"); - prepareIndex("bx").setId("1").setSource("field1", "value1", "field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("bx", "1", "field1", "value1", "field2", "value2"); ensureGreen(); @@ -986,7 +996,7 @@ public void testIndexTemplatesWithSameSubfield() { """, XContentType.JSON) .get(); - prepareIndex("test").setSource().get(); + indexDoc("test", null); FieldCapabilitiesResponse fieldCapabilitiesResponse = client().prepareFieldCaps("test").setFields("*location").get(); { Map field = fieldCapabilitiesResponse.getField("kwm.source.geo.location"); @@ -1003,4 +1013,13 @@ public void testIndexTemplatesWithSameSubfield() { assertTrue(fieldCapabilities.isAggregatable()); } } + + private DocWriteResponse indexDocImmediate(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java index e7568d2f93e81..a25206c0f050e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java @@ -62,7 +62,11 @@ public void testAsyncProcessorImplementation() { try (BulkRequest bulkRequest = new BulkRequest()) { int numDocs = randomIntBetween(8, 256); for (int i = 0; i < numDocs; i++) { - bulkRequest.add(new IndexRequest("foobar").id(Integer.toString(i)).source("{}", XContentType.JSON).setPipeline("_id")); + IndexRequest indexRequest = new IndexRequest("foobar").id(Integer.toString(i)) + .source("{}", XContentType.JSON) + .setPipeline("_id"); + bulkRequest.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems().length, equalTo(numDocs)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java index d82fb640bc5e6..dcfc1650dd989 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.GetPipelineResponse; @@ -146,6 +147,7 @@ public void testBulkWithIngestFailures() throws Exception { IndexRequest indexRequest = new IndexRequest("index").id(Integer.toString(i)).setPipeline("_id"); indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", i % 2 == 0); bulkRequest.add(indexRequest); + indexRequest.decRef(); } BulkResponse response = client().bulk(bulkRequest).actionGet(); @@ -196,10 +198,12 @@ public void testBulkWithUpsert() throws Exception { IndexRequest indexRequest = new IndexRequest("index").id("1").setPipeline("_id"); indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "val1"); bulkRequest.add(indexRequest); + indexRequest.decRef(); UpdateRequest updateRequest = new UpdateRequest("index", "2"); updateRequest.doc("{}", Requests.INDEX_CONTENT_TYPE); updateRequest.upsert("{\"field1\":\"upserted_val\"}", XContentType.JSON).upsertRequest().setPipeline("_id"); bulkRequest.add(updateRequest); + updateRequest.decRef(); BulkResponse response = client().bulk(bulkRequest).actionGet(); @@ -233,14 +237,24 @@ public void test() throws Exception { assertThat(getResponse.pipelines().size(), equalTo(1)); assertThat(getResponse.pipelines().get(0).getId(), equalTo("_id")); - prepareIndex("test").setId("1").setPipeline("_id").setSource("field", "value", "fail", false).get(); + { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setPipeline("_id") + .setSource("field", "value", "fail", false); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } Map doc = client().prepareGet("test", "1").get().getSourceAsMap(); assertThat(doc.get("field"), equalTo("value")); assertThat(doc.get("processed"), equalTo(true)); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - bulkRequestBuilder.add(prepareIndex("test").setId("2").setSource("field", "value2", "fail", false).setPipeline("_id")).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("2") + .setSource("field", "value2", "fail", false) + .setPipeline("_id"); + bulkRequestBuilder.add(indexRequestBuilder).get(); + indexRequestBuilder.request().decRef(); } doc = client().prepareGet("test", "2").get().getSourceAsMap(); assertThat(doc.get("field"), equalTo("value2")); @@ -293,9 +307,9 @@ public void testWithDedicatedMaster() throws Exception { clusterAdmin().putPipeline(putPipelineRequest).get(); try (BulkRequestBuilder bulkRequestBuilder = client(masterOnlyNode).prepareBulk()) { - BulkItemResponse item = bulkRequestBuilder.add( - prepareIndex("test").setSource("field", "value2", "drop", true).setPipeline("_id") - ).get().getItems()[0]; + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource("field", "value2", "drop", true).setPipeline("_id"); + BulkItemResponse item = bulkRequestBuilder.add(indexRequestBuilder).get().getItems()[0]; + indexRequestBuilder.request().decRef(); assertFalse(item.isFailed()); assertEquals("auto-generated", item.getResponse().getId()); } @@ -353,12 +367,13 @@ public void testPipelineOriginHeader() throws Exception { clusterAdmin().putPipeline(putPipelineRequest).get(); } + IndexRequest indexRequest = new IndexRequest("test"); Exception e = expectThrows(Exception.class, () -> { - IndexRequest indexRequest = new IndexRequest("test"); indexRequest.source("{}", XContentType.JSON); indexRequest.setPipeline("1"); client().index(indexRequest).get(); }); + indexRequest.decRef(); IngestProcessorException ingestException = (IngestProcessorException) e.getCause(); assertThat(ingestException.getHeader("processor_type"), equalTo(List.of("fail"))); assertThat(ingestException.getHeader("pipeline_origin"), equalTo(List.of("3", "2", "1"))); @@ -426,7 +441,9 @@ public void testPipelineProcessorOnFailure() throws Exception { clusterAdmin().putPipeline(putPipelineRequest).get(); } - prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).setPipeline("1").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).setPipeline("1"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Map inserted = client().prepareGet("test", "1").get().getSourceAsMap(); assertThat(inserted.get("readme"), equalTo("pipeline with id [3] is a bad pipeline")); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java index d0d126a5439e6..02d1825fa2fce 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java @@ -96,7 +96,9 @@ public void testIngestStatsNamesAndTypes() throws IOException { // index a single document through the pipeline try (BulkRequest bulkRequest = new BulkRequest()) { - bulkRequest.add(new IndexRequest("index1").id("1").source("{}", XContentType.JSON).setPipeline("pipeline1")); + IndexRequest indexRequest = new IndexRequest("index1").id("1").source("{}", XContentType.JSON).setPipeline("pipeline1"); + bulkRequest.add(indexRequest); + indexRequest.decRef(); client().bulk(bulkRequest).actionGet(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/mget/SimpleMgetIT.java b/server/src/internalClusterTest/java/org/elasticsearch/mget/SimpleMgetIT.java index 03e8edbf1173d..e2a67319b1337 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/mget/SimpleMgetIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/mget/SimpleMgetIT.java @@ -8,16 +8,19 @@ package org.elasticsearch.mget; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.get.MultiGetItemResponse; import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -37,10 +40,7 @@ public class SimpleMgetIT extends ESIntegTestCase { public void testThatMgetShouldWorkWithOneIndexMissing() throws IOException { createIndex("test"); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("foo", "bar").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + indexImmediate("test", "1", jsonBuilder().startObject().field("foo", "bar").endObject()); MultiGetResponse mgetResponse = client().prepareMultiGet() .add(new MultiGetRequest.Item("test", "1")) @@ -74,10 +74,7 @@ public void testThatMgetShouldWorkWithMultiIndexAlias() throws IOException { assertAcked(prepareCreate("test").addAlias(new Alias("multiIndexAlias"))); assertAcked(prepareCreate("test2").addAlias(new Alias("multiIndexAlias"))); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("foo", "bar").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + indexImmediate("test", "1", jsonBuilder().startObject().field("foo", "bar").endObject()); MultiGetResponse mgetResponse = client().prepareMultiGet() .add(new MultiGetRequest.Item("test", "1")) @@ -113,10 +110,7 @@ public void testThatMgetShouldWorkWithAliasRouting() throws IOException { ) ); - prepareIndex("alias1").setId("1") - .setSource(jsonBuilder().startObject().field("foo", "bar").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + indexImmediate("alias1", "1", jsonBuilder().startObject().field("foo", "bar").endObject()); MultiGetResponse mgetResponse = client().prepareMultiGet().add(new MultiGetRequest.Item("alias1", "1")).get(); assertEquals(1, mgetResponse.getResponses().length); @@ -139,7 +133,10 @@ public void testThatSourceFilteringIsSupported() throws Exception { .endObject() ); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource(sourceBytesRef, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)) + .setSource(sourceBytesRef, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } MultiGetRequestBuilder request = client().prepareMultiGet(); @@ -186,11 +183,12 @@ public void testThatRoutingPerDocumentIsSupported() throws Exception { final String id = routingKeyForShard("test", 0); final String routingOtherShard = routingKeyForShard("test", 1); - prepareIndex("test").setId(id) + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(id) .setRefreshPolicy(IMMEDIATE) .setRouting(routingOtherShard) - .setSource(jsonBuilder().startObject().field("foo", "bar").endObject()) - .get(); + .setSource(jsonBuilder().startObject().field("foo", "bar").endObject()); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); MultiGetResponse mgetResponse = client().prepareMultiGet() .add(new MultiGetRequest.Item(indexOrAlias(), id).routing(routingOtherShard)) @@ -210,4 +208,13 @@ public void testThatRoutingPerDocumentIsSupported() throws Exception { private static String indexOrAlias() { return randomBoolean() ? "test" : "alias"; } + + private DocWriteResponse indexImmediate(String index, String id, XContentBuilder source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverIT.java index 6b30cfd8e8535..45b1fcadb1a2c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverIT.java @@ -36,29 +36,33 @@ public class DocumentParsingObserverIT extends ESIntegTestCase { public void testDocumentIsReportedUponBulk() throws IOException { hasWrappedParser = false; - client().index( - new IndexRequest(TEST_INDEX_NAME).id("1").source(jsonBuilder().startObject().field("test", "I am sam i am").endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest(TEST_INDEX_NAME).id("1") + .source(jsonBuilder().startObject().field("test", "I am sam " + "i am").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertTrue(hasWrappedParser); // there are more assertions in a TestDocumentParsingObserver hasWrappedParser = false; // the format of the request does not matter - client().index( - new IndexRequest(TEST_INDEX_NAME).id("2").source(cborBuilder().startObject().field("test", "I am sam i am").endObject()) - ).actionGet(); + indexRequest = new IndexRequest(TEST_INDEX_NAME).id("2") + .source(cborBuilder().startObject().field("test", "I am sam i am").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertTrue(hasWrappedParser); // there are more assertions in a TestDocumentParsingObserver hasWrappedParser = false; - // white spaces does not matter - client().index(new IndexRequest(TEST_INDEX_NAME).id("3").source(""" + indexRequest = new IndexRequest(TEST_INDEX_NAME).id("3").source(""" { "test": "I am sam i am" } - """, XContentType.JSON)).actionGet(); + """, XContentType.JSON); + // white spaces does not matter + client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertTrue(hasWrappedParser); // there are more assertions in a TestDocumentParsingObserver } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java index 1e67a38c76017..fa13fd792bbb1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java @@ -49,11 +49,11 @@ public void testFullRollingRestart() throws Exception { final String healthTimeout = "1m"; for (int i = 0; i < 1000; i++) { - prepareIndex("test").setId(Long.toString(i)).setSource(Map.of("test", "value" + i)).get(); + index("test", Long.toString(i), Map.of("test", "value" + i)); } flush(); for (int i = 1000; i < 2000; i++) { - prepareIndex("test").setId(Long.toString(i)).setSource(Map.of("test", "value" + i)).get(); + index("test", Long.toString(i), Map.of("test", "value" + i)); } logger.info("--> now start adding nodes"); @@ -165,7 +165,7 @@ public void testNoRebalanceOnRollingRestart() throws Exception { ).get(); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Long.toString(i)).setSource(Map.of("test", "value" + i)).get(); + index("test", Long.toString(i), Map.of("test", "value" + i)); } ensureGreen(); ClusterState state = clusterAdmin().prepareState().get().getState(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java index 0e14d80aaa0cd..49ee08520c250 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java @@ -122,13 +122,13 @@ public void testSimpleRelocationNoIndexing() { logger.info("--> index 10 docs"); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } logger.info("--> flush so we have an actual index"); indicesAdmin().prepareFlush().get(); logger.info("--> index more docs so we have something in the translog"); for (int i = 10; i < 20; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } logger.info("--> verifying count"); @@ -337,12 +337,12 @@ public void indexShardStateChanged( clusterAdmin().prepareReroute().add(new MoveAllocationCommand("test", 0, nodes[fromNode], nodes[toNode])).get(); logger.debug("--> index [{}] documents", builders1.size()); - indexRandom(false, true, builders1); + indexRandomAndDecrefRequests(false, true, builders1); // wait for shard to reach post recovery postRecoveryShards.acquire(1); logger.debug("--> index [{}] documents", builders2.size()); - indexRandom(true, true, builders2); + indexRandomAndDecrefRequests(true, true, builders2); // verify cluster was finished. assertFalse( @@ -386,7 +386,7 @@ public void testCancellationCleansTempFiles() throws Exception { for (int i = 0; i < numDocs; i++) { requests.add(prepareIndex(indexName).setSource("{}", XContentType.JSON)); } - indexRandom(true, requests); + indexRandomAndDecrefRequests(true, requests); assertFalse(clusterAdmin().prepareHealth().setWaitForNodes("3").setWaitForGreenStatus().get().isTimedOut()); flush(); @@ -489,7 +489,7 @@ public void testIndexSearchAndRelocateConcurrently() throws Exception { ids.add(id); docs[i] = prepareIndex("test").setId(id).setSource("field1", English.intToEnglish(i)); } - indexRandom(true, docs); + indexRandomAndDecrefRequests(true, docs); assertHitCount(prepareSearch("test"), numDocs); logger.info(" --> moving index to new nodes"); @@ -504,7 +504,7 @@ public void testIndexSearchAndRelocateConcurrently() throws Exception { ids.add(id); docs[i] = prepareIndex("test").setId(id).setSource("field1", English.intToEnglish(numDocs + i)); } - indexRandom(true, docs); + indexRandomAndDecrefRequests(true, docs); logger.info(" --> waiting for relocation to complete"); ensureGreen(TimeValue.timeValueSeconds(60), "test"); // move all shards to the new nodes (it waits on relocation) @@ -534,16 +534,18 @@ public void testRelocateWhileWaitingForRefresh() { logger.info("--> index 10 docs"); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } logger.info("--> flush so we have an actual index"); indicesAdmin().prepareFlush().get(); logger.info("--> index more docs so we have something in the translog"); + List buildersToDecRef = new ArrayList<>(); for (int i = 10; i < 20; i++) { - prepareIndex("test").setId(Integer.toString(i)) + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)) .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL) - .setSource("field", "value" + i) - .execute(); + .setSource("field", "value" + i); + buildersToDecRef.add(indexRequestBuilder); + indexRequestBuilder.execute(); } logger.info("--> start another node"); @@ -567,6 +569,9 @@ public void testRelocateWhileWaitingForRefresh() { logger.info("--> verifying count"); indicesAdmin().prepareRefresh().get(); assertHitCount(prepareSearch("test").setSize(0), 20); + for (IndexRequestBuilder builder : buildersToDecRef) { + builder.request().decRef(); + } } public void testRelocateWhileContinuouslyIndexingAndWaitingForRefresh() throws Exception { @@ -582,19 +587,22 @@ public void testRelocateWhileContinuouslyIndexingAndWaitingForRefresh() throws E logger.info("--> index 10 docs"); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).get(); + indexDoc("test", Integer.toString(i), "field", "value" + i); } logger.info("--> flush so we have an actual index"); indicesAdmin().prepareFlush().get(); logger.info("--> index more docs so we have something in the translog"); final List> pendingIndexResponses = new ArrayList<>(); + List buildersToDecRef = new ArrayList<>(); for (int i = 10; i < 20; i++) { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); pendingIndexResponses.add( - prepareIndex("test").setId(Integer.toString(i)) + indexRequestBuilder.setId(Integer.toString(i)) .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL) .setSource("field", "value" + i) .execute() ); + buildersToDecRef.add(indexRequestBuilder); } logger.info("--> start another node"); @@ -611,12 +619,14 @@ public void testRelocateWhileContinuouslyIndexingAndWaitingForRefresh() throws E .execute(); logger.info("--> index 100 docs while relocating"); for (int i = 20; i < 120; i++) { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); pendingIndexResponses.add( - prepareIndex("test").setId(Integer.toString(i)) + indexRequestBuilder.setId(Integer.toString(i)) .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL) .setSource("field", "value" + i) .execute() ); + buildersToDecRef.add(indexRequestBuilder); } relocationListener.actionGet(); clusterHealthResponse = clusterAdmin().prepareHealth() @@ -631,6 +641,9 @@ public void testRelocateWhileContinuouslyIndexingAndWaitingForRefresh() throws E indicesAdmin().prepareRefresh().get(); assertTrue(pendingIndexResponses.stream().allMatch(ActionFuture::isDone)); }, 1, TimeUnit.MINUTES); + for (IndexRequestBuilder builder : buildersToDecRef) { + builder.request().decRef(); + } assertHitCount(prepareSearch("test").setSize(0), 120); } @@ -720,4 +733,29 @@ public void sendRequest( } } } + + private void indexRandomAndDecrefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + indexRandomAndDecrefRequests(forceRefresh, Arrays.asList(builders)); + } + + private void indexRandomAndDecrefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecrefRequests(boolean forceRefresh, boolean dummyDocuments, List builders) + throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/SimpleRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/SimpleRecoveryIT.java index baa721cbbabd2..5a56a0475aa4e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/SimpleRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/SimpleRecoveryIT.java @@ -41,12 +41,16 @@ public void testSimpleRecovery() throws Exception { NumShards numShards = getNumShards("test"); - client().index(new IndexRequest("test").id("1").source(source("1", "test"), XContentType.JSON)).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1").source(source("1", "test"), XContentType.JSON); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); BroadcastResponse flushResponse = indicesAdmin().flush(new FlushRequest("test")).actionGet(); assertThat(flushResponse.getTotalShards(), equalTo(numShards.totalNumShards)); assertThat(flushResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); assertThat(flushResponse.getFailedShards(), equalTo(0)); - client().index(new IndexRequest("test").id("2").source(source("2", "test"), XContentType.JSON)).actionGet(); + indexRequest = new IndexRequest("test").id("2").source(source("2", "test"), XContentType.JSON); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); BroadcastResponse refreshResponse = indicesAdmin().refresh(new RefreshRequest("test")).actionGet(); assertThat(refreshResponse.getTotalShards(), equalTo(numShards.totalNumShards)); assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java index 28c56e0cdc916..427b7ddae2d9b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java @@ -85,12 +85,15 @@ public void testCancelRecoveryAndResume() throws Exception { // index some docs and check if they are coming back int numDocs = randomIntBetween(100, 200); - List builder = new ArrayList<>(); + List builders = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); - builder.add(prepareIndex("test").setId(id).setSource("field1", English.intToEnglish(i), "the_id", id)); + builders.add(prepareIndex("test").setId(id).setSource("field1", English.intToEnglish(i), "the_id", id)); + } + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); } - indexRandom(true, builder); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); assertHitCount(prepareSearch().setQuery(QueryBuilders.termQuery("the_id", id)), 1); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasResolveRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasResolveRoutingIT.java index 53001e30763a0..b51a6a1e23885 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasResolveRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasResolveRoutingIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.routing; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -16,6 +17,7 @@ import org.elasticsearch.test.ESIntegTestCase; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -35,12 +37,15 @@ public void testSearchClosedWildcardIndex() throws ExecutionException, Interrupt ensureGreen(); indicesAdmin().prepareAliases().addAlias("test-0", "alias-0").addAlias("test-1", "alias-1").get(); indicesAdmin().prepareClose("test-1").get(); - indexRandom( - true, + List builders = List.of( prepareIndex("test-0").setId("1").setSource("field1", "the quick brown fox jumps"), prepareIndex("test-0").setId("2").setSource("field1", "quick brown"), prepareIndex("test-0").setId("3").setSource("field1", "quick") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } refresh("test-*"); assertHitCount( prepareSearch().setIndices("alias-*").setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(queryStringQuery("quick")), diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java index 8fb56d17b93ff..f0e8c47a59ce7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java @@ -8,9 +8,12 @@ package org.elasticsearch.routing; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.internal.Requests; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -18,6 +21,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentFactory; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; @@ -39,7 +43,7 @@ public void testAliasCrudRouting() throws Exception { assertAcked(indicesAdmin().prepareAliases().addAliasAction(AliasActions.add().index("test").alias("alias0").routing("0"))); logger.info("--> indexing with id [1], and routing [0] using alias"); - prepareIndex("alias0").setId("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("alias0", "1", "field", "value1"); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "1").get().isExists(), equalTo(false)); @@ -55,10 +59,11 @@ public void testAliasCrudRouting() throws Exception { } logger.info("--> updating with id [1] and routing through alias"); - client().prepareUpdate("alias0", "1") + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("alias0", "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject()) - .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2") - .get(); + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2"); + updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("alias0", "1").get().isExists(), equalTo(true)); assertThat(client().prepareGet("alias0", "1").get().getSourceAsMap().get("field").toString(), equalTo("value2")); @@ -81,7 +86,7 @@ public void testAliasCrudRouting() throws Exception { } logger.info("--> indexing with id [1], and routing [0] using alias"); - prepareIndex("alias0").setId("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("alias0", "1", "field", "value1"); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "1").get().isExists(), equalTo(false)); @@ -105,7 +110,7 @@ public void testAliasSearchRouting() throws Exception { ); logger.info("--> indexing with id [1], and routing [0] using alias"); - prepareIndex("alias0").setId("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("alias0", "1", "field", "value1"); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "1").get().isExists(), equalTo(false)); @@ -138,7 +143,7 @@ public void testAliasSearchRouting() throws Exception { } logger.info("--> indexing with id [2], and routing [1] using alias"); - prepareIndex("alias1").setId("2").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("alias1", "2", "field", "value1"); logger.info("--> search with no routing, should fine two"); for (int i = 0; i < 5; i++) { @@ -212,7 +217,7 @@ public void testAliasSearchRoutingWithTwoIndices() throws Exception { ); ensureGreen(); // wait for events again to make sure we got the aliases on all nodes logger.info("--> indexing with id [1], and routing [0] using alias to test-a"); - prepareIndex("alias-a0").setId("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("alias-a0", "1", "field", "value1"); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test-a", "1").get().isExists(), equalTo(false)); @@ -223,7 +228,7 @@ public void testAliasSearchRoutingWithTwoIndices() throws Exception { } logger.info("--> indexing with id [0], and routing [1] using alias to test-b"); - prepareIndex("alias-b1").setId("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("alias-b1", "1", "field", "value1"); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test-a", "1").get().isExists(), equalTo(false)); @@ -264,9 +269,9 @@ public void testAliasSearchRoutingWithConcreteAndAliasedIndices_issue2682() thro assertAcked(indicesAdmin().prepareAliases().addAliasAction(AliasActions.add().index("index").alias("index_1").routing("1"))); logger.info("--> indexing on index_1 which is an alias for index with routing [1]"); - prepareIndex("index_1").setId("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("index_1", "1", "field", "value1"); logger.info("--> indexing on index_2 which is a concrete index"); - prepareIndex("index_2").setId("2").setSource("field", "value2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("index_2", "2", "field", "value2"); logger.info("--> search all on index_* should find two"); for (int i = 0; i < 5; i++) { @@ -287,9 +292,9 @@ public void testAliasSearchRoutingWithConcreteAndAliasedIndices_issue3268() thro assertAcked(indicesAdmin().prepareAliases().addAliasAction(AliasActions.add().index("index").alias("index_1").routing("1"))); logger.info("--> indexing on index_1 which is an alias for index with routing [1]"); - prepareIndex("index_1").setId("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("index_1", "1", "field", "value1"); logger.info("--> indexing on index_2 which is a concrete index"); - prepareIndex("index_2").setId("2").setSource("field", "value2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("index_2", "2", "field", "value2"); assertResponse( prepareSearch("index_*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1).setQuery(QueryBuilders.matchAllQuery()), @@ -310,7 +315,7 @@ public void testIndexingAliasesOverTime() throws Exception { assertAcked(indicesAdmin().prepareAliases().addAliasAction(AliasActions.add().index("test").alias("alias").routing("3"))); logger.info("--> indexing with id [0], and routing [3]"); - prepareIndex("alias").setId("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("alias", "0", "field", "value1"); logger.info("--> verifying get with no routing, should not find anything"); logger.info("--> verifying get and search with routing, should find"); @@ -336,7 +341,7 @@ public void testIndexingAliasesOverTime() throws Exception { ); logger.info("--> indexing with id [1], and routing [4]"); - prepareIndex("alias").setId("1").setSource("field", "value2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("alias", "1", "field", "value2"); logger.info("--> verifying get with no routing, should not find anything"); logger.info("--> verifying get and search with routing, should find"); @@ -348,4 +353,13 @@ public void testIndexingAliasesOverTime() throws Exception { } } + private DocWriteResponse indexDocImmediate(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java index e25da54d7b214..2ead871085f0e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.routing; import org.apache.lucene.util.Constants; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -216,7 +217,12 @@ private Map> generateRoutedDocumentIds(String index) { String id = routingValue + "_" + String.valueOf(k); routingToDocumentIds.get(routingValue).add(id); - prepareIndex(index).setId(id).setRouting(routingValue).setSource("foo", "bar").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id).setRouting(routingValue).setSource("foo", "bar"); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java index d5405bc2cfb2b..68bbc91373caa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -21,11 +22,13 @@ import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Requests; import org.elasticsearch.cluster.ClusterState; @@ -36,6 +39,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentFactory; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -70,11 +74,7 @@ public void testSimpleCrudRouting() throws Exception { ensureGreen(); String routingValue = findNonMatchingRoutingValue("test", "1"); logger.info("--> indexing with id [1], and routing [{}]", routingValue); - prepareIndex("test").setId("1") - .setRouting(routingValue) - .setSource("field", "value1") - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); + indexDocImmediateWithRouting("test", "1", routingValue, "field", "value1"); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "1").get().isExists(), equalTo(false)); @@ -99,11 +99,7 @@ public void testSimpleCrudRouting() throws Exception { } logger.info("--> indexing with id [1], and routing [0]"); - prepareIndex("test").setId("1") - .setRouting(routingValue) - .setSource("field", "value1") - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); + indexDocImmediateWithRouting("test", "1", routingValue, "field", "value1"); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "1").get().isExists(), equalTo(false)); @@ -120,11 +116,7 @@ public void testSimpleSearchRouting() { String routingValue = findNonMatchingRoutingValue("test", "1"); logger.info("--> indexing with id [1], and routing [{}]", routingValue); - prepareIndex("test").setId("1") - .setRouting(routingValue) - .setSource("field", "value1") - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); + indexDocImmediateWithRouting("test", "1", routingValue, "field", "value1"); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "1").get().isExists(), equalTo(false)); @@ -153,11 +145,7 @@ public void testSimpleSearchRouting() { String secondRoutingValue = "1"; logger.info("--> indexing with id [{}], and routing [{}]", routingValue, secondRoutingValue); - prepareIndex("test").setId(routingValue) - .setRouting(secondRoutingValue) - .setSource("field", "value1") - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); + indexDocImmediateWithRouting("test", routingValue, secondRoutingValue, "field", "value1"); logger.info("--> search with no routing, should fine two"); for (int i = 0; i < 5; i++) { @@ -219,16 +207,12 @@ public void testRequiredRoutingCrudApis() throws Exception { String routingValue = findNonMatchingRoutingValue("test", "1"); logger.info("--> indexing with id [1], and routing [{}]", routingValue); - prepareIndex(indexOrAlias()).setId("1") - .setRouting(routingValue) - .setSource("field", "value1") - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); + indexDocImmediateWithRouting(indexOrAlias(), "1", routingValue, "field", "value1"); logger.info("--> verifying get with no routing, should fail"); logger.info("--> indexing with id [1], with no routing, should fail"); try { - prepareIndex(indexOrAlias()).setId("1").setSource("field", "value1").get(); + indexDoc(indexOrAlias(), "1", "field", "value1"); fail("index with missing routing when routing is required should fail"); } catch (ElasticsearchException e) { assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class)); @@ -258,14 +242,22 @@ public void testRequiredRoutingCrudApis() throws Exception { assertThat(client().prepareGet(indexOrAlias(), "1").setRouting(routingValue).get().isExists(), equalTo(true)); } + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2"); try { - client().prepareUpdate(indexOrAlias(), "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get(); + updateRequestBuilder.get(); fail("update with missing routing when routing is required should fail"); } catch (ElasticsearchException e) { assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class)); + } finally { + updateRequestBuilder.request().decRef(); } - client().prepareUpdate(indexOrAlias(), "1").setRouting(routingValue).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get(); + updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") + .setRouting(routingValue) + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2"); + updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); indicesAdmin().prepareRefresh().get(); for (int i = 0; i < 5; i++) { @@ -312,9 +304,9 @@ public void testRequiredRoutingBulk() throws Exception { ensureGreen(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { String index = indexOrAlias(); - BulkResponse bulkResponse = bulkRequestBuilder.add( - new IndexRequest(index).id("1").source(Requests.INDEX_CONTENT_TYPE, "field", "value") - ).get(); + IndexRequest indexRequest = new IndexRequest(index).id("1").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + BulkResponse bulkResponse = bulkRequestBuilder.add(indexRequest).get(); + indexRequest.decRef(); assertThat(bulkResponse.getItems().length, equalTo(1)); assertThat(bulkResponse.hasFailures(), equalTo(true)); @@ -329,16 +321,16 @@ public void testRequiredRoutingBulk() throws Exception { try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { String index = indexOrAlias(); - BulkResponse bulkResponse = bulkRequestBuilder.add( - new IndexRequest(index).id("1").routing("0").source(Requests.INDEX_CONTENT_TYPE, "field", "value") - ).get(); + IndexRequest indexRequest = new IndexRequest(index).id("1").routing("0").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + BulkResponse bulkResponse = bulkRequestBuilder.add(indexRequest).get(); + indexRequest.decRef(); assertThat(bulkResponse.hasFailures(), equalTo(false)); } try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add( - new UpdateRequest(indexOrAlias(), "1").doc(Requests.INDEX_CONTENT_TYPE, "field", "value2") - ).get(); + UpdateRequest updateRequest = new UpdateRequest(indexOrAlias(), "1").doc(Requests.INDEX_CONTENT_TYPE, "field", "value2"); + BulkResponse bulkResponse = bulkRequestBuilder.add(updateRequest).get(); + updateRequest.decRef(); assertThat(bulkResponse.getItems().length, equalTo(1)); assertThat(bulkResponse.hasFailures(), equalTo(true)); @@ -352,9 +344,10 @@ public void testRequiredRoutingBulk() throws Exception { } try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add( - new UpdateRequest(indexOrAlias(), "1").doc(Requests.INDEX_CONTENT_TYPE, "field", "value2").routing("0") - ).get(); + UpdateRequest updateRequest = new UpdateRequest(indexOrAlias(), "1").doc(Requests.INDEX_CONTENT_TYPE, "field", "value2") + .routing("0"); + BulkResponse bulkResponse = bulkRequestBuilder.add(updateRequest).get(); + updateRequest.decRef(); assertThat(bulkResponse.hasFailures(), equalTo(false)); } @@ -399,13 +392,13 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { ensureGreen(); String routingValue = findNonMatchingRoutingValue("test", "1"); logger.info("--> indexing with id [1], and routing [{}]", routingValue); - prepareIndex(indexOrAlias()).setId("1").setRouting(routingValue).setSource("field", "value1").get(); - logger.info("--> indexing with id [2], and routing [{}]", routingValue); - prepareIndex(indexOrAlias()).setId("2") + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexOrAlias()).setId("1") .setRouting(routingValue) - .setSource("field", "value2") - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); + .setSource("field", "value1"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + logger.info("--> indexing with id [2], and routing [{}]", routingValue); + indexDocImmediateWithRouting(indexOrAlias(), "2", routingValue, "field", "value2"); logger.info("--> verifying get with id [1] with routing [0], should succeed"); assertThat(client().prepareGet(indexOrAlias(), "1").setRouting(routingValue).get().isExists(), equalTo(true)); @@ -446,18 +439,22 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { assertThat(e.getMessage(), equalTo("routing is required for [test]/[1]")); } - UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "1") + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setRouting(routingValue) - .setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1") - .get(); + .setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertThat(updateResponse.getId(), equalTo("1")); assertThat(updateResponse.getVersion(), equalTo(2L)); + updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); try { - client().prepareUpdate(indexOrAlias(), "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get(); + updateRequestBuilder.get(); fail(); } catch (RoutingMissingException e) { assertThat(e.getMessage(), equalTo("routing is required for [test]/[1]")); + } finally { + updateRequestBuilder.request().decRef(); } logger.info("--> verifying mget with ids [1,2], with routing [0], should succeed"); @@ -522,4 +519,13 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { private static String indexOrAlias() { return randomBoolean() ? "test" : "alias"; } + + private DocWriteResponse indexDocImmediateWithRouting(String index, String id, String routing, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setRouting(routing).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java index 702d4a99df2f1..fa5a89d93beb7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.search; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; @@ -47,7 +48,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { private void indexDocs() { for (int i = 0; i < 32; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get(); + indexDoc("test", Integer.toString(i), "field", "value"); } refresh("test"); } @@ -90,7 +91,9 @@ public void testAggsTimeout() { } public void testPartialResultsIntolerantTimeout() throws Exception { - prepareIndex("test").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); ElasticsearchException ex = expectThrows( ElasticsearchException.class, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithRejectionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithRejectionsIT.java index 5030438f829fc..c41f696e233bf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithRejectionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithRejectionsIT.java @@ -36,7 +36,7 @@ public void testOpenContextsAfterRejections() throws Exception { ensureGreen("test"); final int docs = scaledRandomIntBetween(20, 50); for (int i = 0; i < docs; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get(); + indexDoc("test", Integer.toString(i), "field", "value"); } IndicesStatsResponse indicesStats = indicesAdmin().prepareStats().get(); assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0L)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/StressSearchServiceReaperIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/StressSearchServiceReaperIT.java index c7aa4b3179288..bd418f65493d6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/StressSearchServiceReaperIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/StressSearchServiceReaperIT.java @@ -41,6 +41,9 @@ public void testStressReaper() throws ExecutionException, InterruptedException { } createIndex("test"); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } final int iterations = scaledRandomIntBetween(500, 1000); for (int i = 0; i < iterations; i++) { assertHitCountAndNoFailures(prepareSearch("test").setQuery(matchAllQuery()).setSize(num), num); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java index a856ee36aadc2..888db2868e77a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java @@ -35,6 +35,9 @@ public void setupSuiteScopeCluster() throws Exception { docs.add(prepareIndex("index").setSource("f", Integer.toString(i / 3))); } indexRandom(true, docs); + for (IndexRequestBuilder builder : docs) { + builder.request().decRef(); + } } public void testScroll() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java index fc0a93ad3d290..356936ca8498f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java @@ -58,6 +58,9 @@ public void testMultipleAggsOnSameField_WithDifferentRequiredValueSourceType() t } } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); final long finalMissingValues = missingValues; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java index f2aa79d115c4c..54262ef197546 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java @@ -126,7 +126,7 @@ public void testRandomRanges() throws Exception { source = source.value(docs[i][j]); } source = source.endArray().endObject(); - prepareIndex("idx").setSource(source).get(); + index("idx", null, source); } assertNoFailures(indicesAdmin().prepareRefresh("idx").setIndicesOptions(IndicesOptions.lenientExpandOpen()).get()); @@ -252,7 +252,7 @@ public void testDuelTerms() throws Exception { source = source.endArray().endObject(); indexingRequests.add(prepareIndex("idx").setSource(source)); } - indexRandom(true, indexingRequests); + indexRandomAndDecrefRequests(true, indexingRequests); assertNoFailures(indicesAdmin().prepareRefresh("idx").setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get()); @@ -353,7 +353,7 @@ public void testDuelTermsHistogram() throws Exception { source = source.value(randomFrom(values)); } source = source.endArray().endObject(); - prepareIndex("idx").setSource(source).get(); + index("idx", null, source); } assertNoFailures(indicesAdmin().prepareRefresh("idx").setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get()); @@ -402,7 +402,7 @@ public void testLargeNumbersOfPercentileBuckets() throws Exception { for (int i = 0; i < numDocs; ++i) { indexingRequests.add(prepareIndex("idx").setId(Integer.toString(i)).setSource("double_value", randomDouble())); } - indexRandom(true, indexingRequests); + indexRandomAndDecrefRequests(true, indexingRequests); assertResponse( prepareSearch("idx").addAggregation( @@ -421,7 +421,7 @@ public void testLargeNumbersOfPercentileBuckets() throws Exception { public void testReduce() throws Exception { createIndex("idx"); final int value = randomIntBetween(0, 10); - indexRandom(true, prepareIndex("idx").setSource("f", value)); + indexRandomAndDecrefRequests(true, Collections.singletonList(prepareIndex("idx").setSource("f", value))); assertNoFailuresAndResponse( prepareSearch("idx").addAggregation( filter("filter", QueryBuilders.matchAllQuery()).subAggregation( @@ -482,7 +482,7 @@ public void testDuelDepthBreadthFirst() throws Exception { final int v3 = randomInt(1 << randomInt(7)); reqs.add(prepareIndex("idx").setSource("f1", v1, "f2", v2, "f3", v3)); } - indexRandom(true, reqs); + indexRandomAndDecrefRequests(true, reqs); assertNoFailuresAndResponse( prepareSearch("idx").addAggregation( @@ -525,4 +525,13 @@ public void testDuelDepthBreadthFirst() throws Exception { ); } + private void indexRandomAndDecrefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java index 3568391279a7a..233dda359498b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java @@ -8,6 +8,8 @@ package org.elasticsearch.search.aggregations; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.WrapperQueryBuilder; @@ -29,9 +31,9 @@ public class FiltersAggsRewriteIT extends ESSingleNodeTestCase { public void testWrapperQueryIsRewritten() throws IOException { createIndex("test", Settings.EMPTY, "test", "title", "type=text"); - prepareIndex("test").setId("1").setSource("title", "foo bar baz").get(); - prepareIndex("test").setId("2").setSource("title", "foo foo foo").get(); - prepareIndex("test").setId("3").setSource("title", "bar baz bax").get(); + indexDoc("test", "1", "title", "foo bar baz"); + indexDoc("test", "2", "title", "foo foo foo"); + indexDoc("test", "3", "title", "bar baz bax"); client().admin().indices().prepareRefresh("test").get(); XContentType xContentType = randomFrom(XContentType.values()); @@ -63,4 +65,13 @@ public void testWrapperQueryIsRewritten() throws IOException { assertEquals(metadata, filters.getMetadata()); }); } + + protected final DocWriteResponse indexDoc(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java index f22e0a2931634..3d02510f00f45 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java @@ -33,6 +33,9 @@ public void testMetadataSetOnAggregationResult() throws Exception { builders[i] = prepareIndex("idx").setSource("name", name, "value", randomInt()); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); final var nestedMetadata = Map.of("nested", "value"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MissingValueIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MissingValueIT.java index 8110bc124132a..fd981134106a7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MissingValueIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MissingValueIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.aggregations; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; @@ -44,11 +45,12 @@ protected int maximumNumberOfShards() { @Override protected void setupSuiteScopeCluster() throws Exception { assertAcked(prepareCreate("idx").setMapping("date", "type=date", "location", "type=geo_point", "str", "type=keyword").get()); - indexRandom( - true, - prepareIndex("idx").setId("1").setSource(), - prepareIndex("idx").setId("2").setSource("str", "foo", "long", 3L, "double", 5.5, "date", "2015-05-07", "location", "1,2") - ); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("idx").setId("1").setSource(); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("idx").setId("2") + .setSource("str", "foo", "long", 3L, "double", 5.5, "date", "2015-05-07", "location", "1,2"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); } public void testUnmappedTerms() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java index 4a6859620563c..4b631536fb240 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java @@ -68,6 +68,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } } public void testSingleValueField() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index a9ff9f15a7e92..f446e560ed0c3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -135,7 +135,7 @@ public void setupSuiteScopeCluster() throws Exception { indexDoc(3, 23, 6) ) ); // date: Mar 23, dates: Mar 23, Apr 24 - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); ensureSearchable(); } @@ -999,7 +999,7 @@ public void testSingleValueWithTimeZone() throws Exception { reqs[i] = prepareIndex("idx2").setId("" + i).setSource(jsonBuilder().startObject().timeField("date", date).endObject()); date = date.plusHours(1); } - indexRandom(true, reqs); + indexRandomAndDecRefRequests(true, reqs); assertNoFailuresAndResponse( prepareSearch("idx2").setQuery(matchAllQuery()) @@ -1058,7 +1058,7 @@ public void testSingleValueFieldWithExtendedBounds() throws Exception { docCounts[i] = docCount; } } - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); ensureSearchable("idx2"); ZonedDateTime lastDataBucketKey = baseKey.plusDays((numOfBuckets - 1) * interval); @@ -1155,7 +1155,7 @@ public void testSingleValueFieldWithExtendedBoundsTimezone() throws Exception { ZonedDateTime timeZoneNoonToday = parser.parse("now/d+12h", System::currentTimeMillis, false, timezone).atZone(ZoneOffset.UTC); builders.add(indexDoc(index, timeZoneStartToday, 1)); builders.add(indexDoc(index, timeZoneNoonToday, 2)); - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); ensureSearchable(index); // retrieve those docs with the same time zone and extended bounds @@ -1217,7 +1217,7 @@ public void testSingleValueFieldWithExtendedBoundsOffset() throws Exception { builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-03T08:00:00.000Z")), 2)); builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-06T08:00:00.000Z")), 3)); builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-06T08:00:00.000Z")), 4)); - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); ensureSearchable(index); // retrieve those docs with the same time zone and extended bounds @@ -1273,7 +1273,7 @@ public void testSingleValueWithMultipleDateFormatsFromMapping() throws Exception for (int i = 0; i < reqs.length; i++) { reqs[i] = prepareIndex("idx2").setId("" + i).setSource(jsonBuilder().startObject().field("date", "10-03-2014").endObject()); } - indexRandom(true, reqs); + indexRandomAndDecRefRequests(true, reqs); assertNoFailuresAndResponse( prepareSearch("idx2").setQuery(matchAllQuery()) @@ -1340,7 +1340,7 @@ public void testIssue6965() { public void testDSTBoundaryIssue9491() throws InterruptedException, ExecutionException { assertAcked(indicesAdmin().prepareCreate("test9491").setMapping("d", "type=date").get()); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test9491").setSource("d", "2014-10-08T13:00:00Z"), prepareIndex("test9491").setSource("d", "2014-11-08T13:00:00Z") @@ -1364,7 +1364,7 @@ public void testDSTBoundaryIssue9491() throws InterruptedException, ExecutionExc public void testIssue8209() throws InterruptedException, ExecutionException { assertAcked(indicesAdmin().prepareCreate("test8209").setMapping("d", "type=date").get()); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test8209").setSource("d", "2014-01-01T00:00:00Z"), prepareIndex("test8209").setSource("d", "2014-04-01T00:00:00Z"), @@ -1406,7 +1406,7 @@ public void testIssue8209() throws InterruptedException, ExecutionException { */ public void testFormatIndexUnmapped() throws InterruptedException, ExecutionException { String indexDateUnmapped = "test31760"; - indexRandom(true, prepareIndex(indexDateUnmapped).setSource("foo", "bar")); + indexRandomAndDecRefRequests(true, prepareIndex(indexDateUnmapped).setSource("foo", "bar")); ensureSearchable(indexDateUnmapped); assertNoFailuresAndResponse( @@ -1435,7 +1435,7 @@ public void testFormatIndexUnmapped() throws InterruptedException, ExecutionExce public void testRewriteTimeZone_EpochMillisFormat() throws InterruptedException, ExecutionException { String index = "test31392"; assertAcked(indicesAdmin().prepareCreate(index).setMapping("d", "type=date,format=epoch_millis").get()); - indexRandom(true, prepareIndex(index).setSource("d", "1477954800000")); + indexRandomAndDecRefRequests(true, prepareIndex(index).setSource("d", "1477954800000")); ensureSearchable(index); assertNoFailuresAndResponse( prepareSearch(index).addAggregation( @@ -1554,7 +1554,7 @@ public void testScriptCaching() throws Exception { ); String date = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(date(1, 1)); String date2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(date(2, 1)); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("cache_test_idx").setId("1").setSource("d", date), prepareIndex("cache_test_idx").setId("2").setSource("d", date2) @@ -1709,8 +1709,8 @@ private ZonedDateTime key(Histogram.Bucket bucket) { */ public void testDateNanosHistogram() throws Exception { assertAcked(prepareCreate("nanos").setMapping("date", "type=date_nanos").get()); - indexRandom(true, prepareIndex("nanos").setId("1").setSource("date", "2000-01-01")); - indexRandom(true, prepareIndex("nanos").setId("2").setSource("date", "2000-01-02")); + indexRandomAndDecRefRequests(true, prepareIndex("nanos").setId("1").setSource("date", "2000-01-01")); + indexRandomAndDecRefRequests(true, prepareIndex("nanos").setId("2").setSource("date", "2000-01-02")); // Search interval 24 hours assertNoFailuresAndResponse( @@ -1777,4 +1777,24 @@ public void testHardBoundsOnDates() { } ); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java index 5abf52cf37f88..ba7e516f18d38 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java @@ -66,6 +66,9 @@ private void prepareIndex(ZonedDateTime date, int numHours, int stepSizeHours, i date = date.plusHours(stepSizeHours); } indexRandom(true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } } public void testSingleValueWithPositiveOffset() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java index 0b92372652597..2fd27d5f51b5f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java @@ -107,6 +107,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, docs); + for (IndexRequestBuilder indexRequestBuilder : docs) { + indexRequestBuilder.request().decRef(); + } ensureSearchable(); } @@ -617,12 +620,16 @@ public void testScriptCaching() throws Exception { assertAcked( prepareCreate("cache_test_idx").setMapping("date", "type=date") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource(jsonBuilder().startObject().timeField("date", date(1, 1)).endObject()), prepareIndex("cache_test_idx").setId("2").setSource(jsonBuilder().startObject().timeField("date", date(2, 1)).endObject()) ); + indexRandom(true, builders); + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( @@ -709,12 +716,15 @@ public void testScriptCaching() throws Exception { public void testRangeWithFormatStringValue() throws Exception { String indexName = "dateformat_test_idx"; assertAcked(prepareCreate(indexName).setMapping("date", "type=date,format=strict_hour_minute_second")); - indexRandom( - true, + List builders = List.of( prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("date", "00:16:40").endObject()), prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("date", "00:33:20").endObject()), prepareIndex(indexName).setId("3").setSource(jsonBuilder().startObject().field("date", "00:50:00").endObject()) ); + indexRandom(true, builders); + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } // using no format should work when to/from is compatible with format in // mapping @@ -775,12 +785,15 @@ public void testRangeWithFormatStringValue() throws Exception { public void testRangeWithFormatNumericValue() throws Exception { String indexName = "dateformat_numeric_test_idx"; assertAcked(prepareCreate(indexName).setMapping("date", "type=date,format=epoch_second")); - indexRandom( - true, + List builders = List.of( prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("date", 1002).endObject()), prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("date", 2000).endObject()), prepareIndex(indexName).setId("3").setSource(jsonBuilder().startObject().field("date", 3008).endObject()) ); + indexRandom(true, builders); + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } // using no format should work when to/from is compatible with format in // mapping diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java index 5a58780a24817..e69a3e940ea8c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java @@ -77,12 +77,8 @@ public void setupSuiteScopeCluster() throws Exception { for (int i = 0; i < data.length; i++) { String[] parts = data[i].split(","); - prepareIndex("test").setId("" + i) - .setSource("author", parts[5], "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])) - .get(); - prepareIndex("idx_unmapped_author").setId("" + i) - .setSource("name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])) - .get(); + indexDoc("test", "" + i, "author", parts[5], "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])); + indexDoc("idx_unmapped_author", "" + i, "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])); // frequent refresh makes it more likely that more segments are created, hence we may parallelize the search across slices indicesAdmin().refresh(new RefreshRequest()).get(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java index 668b9d79c49a8..a98f2d6428b5d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java @@ -155,6 +155,9 @@ public void setupSuiteScopeCluster() throws Exception { getMultiSortDocs(builders); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -944,11 +947,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=float") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1.5), prepareIndex("cache_test_idx").setId("2").setSource("s", 2.5) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java index b5dea9cbbba49..430c7564ca7fd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java @@ -71,6 +71,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java index b04cb5325a82d..8149f1eee21e5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java @@ -91,6 +91,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java index 0ed83f73e418d..a29e592d241e4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java @@ -98,6 +98,9 @@ public void setupSuiteScopeCluster() throws Exception { } } indexRandom(true, cities); + for (IndexRequestBuilder builder : cities) { + builder.request().decRef(); + } cities.clear(); cities.addAll( @@ -116,6 +119,9 @@ public void setupSuiteScopeCluster() throws Exception { cities.add(indexCity("idx-multi", cityName)); } indexRandom(true, cities); + for (IndexRequestBuilder builder : cities) { + builder.request().decRef(); + } prepareCreate("empty_bucket_idx").setMapping("value", "type=integer", "location", "type=geo_point").get(); List builders = new ArrayList<>(); for (int i = 0; i < 2; i++) { @@ -125,6 +131,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()])); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java index 57b11df3b7d31..38b88801b3801 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java @@ -98,6 +98,9 @@ public void setupSuiteScopeCluster() throws Exception { } } indexRandom(true, cities); + for (IndexRequestBuilder builder : cities) { + builder.request().decRef(); + } assertAcked( prepareCreate("multi_valued_idx").setSettings(settings).setMapping("location", "type=geo_point", "city", "type=keyword") @@ -125,6 +128,9 @@ public void setupSuiteScopeCluster() throws Exception { } } indexRandom(true, cities); + for (IndexRequestBuilder builder : cities) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java index abe93597f02b9..d0ad2fe7b1f3d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java @@ -53,6 +53,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java index 421c1475eb5bc..4ccbf9753fab2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java @@ -164,6 +164,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -1088,11 +1091,14 @@ public void testExeptionOnNegativerInterval() { public void testDecimalIntervalAndOffset() throws Exception { assertAcked(prepareCreate("decimal_values").setMapping("d", "type=float").get()); - indexRandom( - true, + List builders = List.of( prepareIndex("decimal_values").setId("1").setSource("d", -0.6), prepareIndex("decimal_values").setId("2").setSource("d", 0.1) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("decimal_values").addAggregation(histogram("histo").field("d").interval(0.7).offset(0.05)), @@ -1117,11 +1123,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=float") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("d", -0.6), prepareIndex("cache_test_idx").setId("2").setSource("d", 0.1) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( @@ -1248,12 +1257,15 @@ public void testInvalidBounds() { public void testHardBounds() throws Exception { assertAcked(prepareCreate("test").setMapping("d", "type=double").get()); - indexRandom( - true, + List builders = List.of( prepareIndex("test").setId("1").setSource("d", -0.6), prepareIndex("test").setId("2").setSource("d", 0.5), prepareIndex("test").setId("3").setSource("d", 0.1) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("test").addAggregation(histogram("histo").field("d").interval(0.1).hardBounds(new DoubleBounds(0.0, null))), diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java index 887afdb578fdb..683a838876801 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.search.aggregations.bucket; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.plugins.Plugin; @@ -20,6 +21,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -48,13 +50,16 @@ public void setupSuiteScopeCluster() throws Exception { assertAcked(prepareCreate("idx").setMapping("ip", "type=ip", "ips", "type=ip")); waitForRelocation(ClusterHealthStatus.GREEN); - indexRandom( - true, + List builders = List.of( prepareIndex("idx").setId("1").setSource("ip", "192.168.1.7", "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), prepareIndex("idx").setId("2").setSource("ip", "192.168.1.10", "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), prepareIndex("idx").setId("3") .setSource("ip", "2001:db8::ff00:42:8329", "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380")) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertAcked(prepareCreate("idx_unmapped")); waitForRelocation(ClusterHealthStatus.GREEN); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java index 7f9cb01599a00..f283f893b061a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.search.aggregations.bucket; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; @@ -17,6 +18,7 @@ import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -52,12 +54,15 @@ protected Map, Object>> pluginScripts() { public void testScriptValue() throws Exception { assertAcked(prepareCreate("index").setMapping("ip", "type=ip")); - indexRandom( - true, + List builders = List.of( prepareIndex("index").setId("1").setSource("ip", "192.168.1.7"), prepareIndex("index").setId("2").setSource("ip", "192.168.1.7"), prepareIndex("index").setId("3").setSource("ip", "2001:db8::2:1") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['ip'].value", Collections.emptyMap()); assertNoFailuresAndResponse( @@ -83,12 +88,15 @@ public void testScriptValue() throws Exception { public void testScriptValues() throws Exception { assertAcked(prepareCreate("index").setMapping("ip", "type=ip")); - indexRandom( - true, + List builders = List.of( prepareIndex("index").setId("1").setSource("ip", "192.168.1.7"), prepareIndex("index").setId("2").setSource("ip", "192.168.1.7"), prepareIndex("index").setId("3").setSource("ip", "2001:db8::2:1") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['ip']", Collections.emptyMap()); assertNoFailuresAndResponse( @@ -114,13 +122,16 @@ public void testScriptValues() throws Exception { public void testMissingValue() throws Exception { assertAcked(prepareCreate("index").setMapping("ip", "type=ip")); - indexRandom( - true, + List builders = List.of( prepareIndex("index").setId("1").setSource("ip", "192.168.1.7"), prepareIndex("index").setId("2").setSource("ip", "192.168.1.7"), prepareIndex("index").setId("3").setSource("ip", "127.0.0.1"), prepareIndex("index").setId("4").setSource("not_ip", "something") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("index").addAggregation( new TermsAggregationBuilder("my_terms").field("ip").missing("127.0.0.1").executionHint(randomExecutionHint()) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java index f0c5cbf9c76bb..9db1223020691 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java @@ -115,6 +115,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, lowCardBuilders); + for (IndexRequestBuilder builder : lowCardBuilders) { + builder.request().decRef(); + } IndexRequestBuilder[] highCardBuilders = new IndexRequestBuilder[100]; // TODO randomize the size? for (int i = 0; i < highCardBuilders.length; i++) { highCardBuilders[i] = prepareIndex("high_card_idx").setSource( @@ -129,6 +132,9 @@ public void setupSuiteScopeCluster() throws Exception { } indexRandom(true, highCardBuilders); + for (IndexRequestBuilder builder : highCardBuilders) { + builder.request().decRef(); + } createIndex("idx_unmapped"); assertAcked(prepareCreate("empty_bucket_idx").setMapping(SINGLE_VALUED_FIELD_NAME, "type=integer")); @@ -142,7 +148,10 @@ public void setupSuiteScopeCluster() throws Exception { getMultiSortDocs(builders); - indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()])); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -902,11 +911,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java index 4e93bf578cc87..849cbd9fac2b9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java @@ -128,6 +128,9 @@ public void setupSuiteScopeCluster() throws Exception { cardinality = stringTerms.size(); indexRandom(true, indexRequests); + for (IndexRequestBuilder indexRequestBuilder : indexRequests) { + indexRequestBuilder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java index e6d0b6a1f9f1c..b7ce915c6c0ca 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java @@ -113,7 +113,7 @@ public void setupSuiteScopeCluster() throws Exception { if (randomBoolean()) { source.field("numeric_value", randomDouble()); } - prepareIndex("idx").setSource(source.endObject()).get(); + index("idx", source.endObject()); } refresh(); ensureSearchable(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java index 9a27b0d8f75a3..e7c9b0dc95f00 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java @@ -170,6 +170,9 @@ public void setupSuiteScopeCluster() throws Exception { ) ); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -474,6 +477,9 @@ public void testParentFilterResolvedCorrectly() throws Exception { ] }""", XContentType.JSON)); indexRandom(true, indexRequests); + for (IndexRequestBuilder builder : indexRequests) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("idx2").addAggregation( @@ -543,44 +549,44 @@ public void testNestedSameDocIdProcessedMultipleTime() throws Exception { ); ensureGreen("idx4"); - prepareIndex("idx4").setId("1") - .setSource( - jsonBuilder().startObject() - .field("name", "product1") - .array("categories", "1", "2", "3", "4") - .startArray("property") - .startObject() - .field("id", 1) - .endObject() - .startObject() - .field("id", 2) - .endObject() - .startObject() - .field("id", 3) - .endObject() - .endArray() - .endObject() - ) - .get(); - prepareIndex("idx4").setId("2") - .setSource( - jsonBuilder().startObject() - .field("name", "product2") - .array("categories", "1", "2") - .startArray("property") - .startObject() - .field("id", 1) - .endObject() - .startObject() - .field("id", 5) - .endObject() - .startObject() - .field("id", 4) - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "idx4", + "1", + jsonBuilder().startObject() + .field("name", "product1") + .array("categories", "1", "2", "3", "4") + .startArray("property") + .startObject() + .field("id", 1) + .endObject() + .startObject() + .field("id", 2) + .endObject() + .startObject() + .field("id", 3) + .endObject() + .endArray() + .endObject() + ); + index( + "idx4", + "2", + jsonBuilder().startObject() + .field("name", "product2") + .array("categories", "1", "2") + .startArray("property") + .startObject() + .field("id", 1) + .endObject() + .startObject() + .field("id", 5) + .endObject() + .startObject() + .field("id", 4) + .endObject() + .endArray() + .endObject() + ); refresh(); assertNoFailuresAndResponse( @@ -678,84 +684,84 @@ public void testFilterAggInsideNestedAgg() throws Exception { ) ); - prepareIndex("classes").setId("1") - .setSource( - jsonBuilder().startObject() - .field("name", "QueryBuilder") - .startArray("methods") - .startObject() - .field("name", "toQuery") - .field("return_type", "Query") - .startArray("parameters") - .startObject() - .field("name", "context") - .field("type", "SearchExecutionContext") - .endObject() - .endArray() - .endObject() - .startObject() - .field("name", "queryName") - .field("return_type", "QueryBuilder") - .startArray("parameters") - .startObject() - .field("name", "queryName") - .field("type", "String") - .endObject() - .endArray() - .endObject() - .startObject() - .field("name", "boost") - .field("return_type", "QueryBuilder") - .startArray("parameters") - .startObject() - .field("name", "boost") - .field("type", "float") - .endObject() - .endArray() - .endObject() - .endArray() - .endObject() - ) - .get(); - prepareIndex("classes").setId("2") - .setSource( - jsonBuilder().startObject() - .field("name", "Document") - .startArray("methods") - .startObject() - .field("name", "add") - .field("return_type", "void") - .startArray("parameters") - .startObject() - .field("name", "field") - .field("type", "IndexableField") - .endObject() - .endArray() - .endObject() - .startObject() - .field("name", "removeField") - .field("return_type", "void") - .startArray("parameters") - .startObject() - .field("name", "name") - .field("type", "String") - .endObject() - .endArray() - .endObject() - .startObject() - .field("name", "removeFields") - .field("return_type", "void") - .startArray("parameters") - .startObject() - .field("name", "name") - .field("type", "String") - .endObject() - .endArray() - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "classes", + "1", + jsonBuilder().startObject() + .field("name", "QueryBuilder") + .startArray("methods") + .startObject() + .field("name", "toQuery") + .field("return_type", "Query") + .startArray("parameters") + .startObject() + .field("name", "context") + .field("type", "SearchExecutionContext") + .endObject() + .endArray() + .endObject() + .startObject() + .field("name", "queryName") + .field("return_type", "QueryBuilder") + .startArray("parameters") + .startObject() + .field("name", "queryName") + .field("type", "String") + .endObject() + .endArray() + .endObject() + .startObject() + .field("name", "boost") + .field("return_type", "QueryBuilder") + .startArray("parameters") + .startObject() + .field("name", "boost") + .field("type", "float") + .endObject() + .endArray() + .endObject() + .endArray() + .endObject() + ); + index( + "classes", + "2", + jsonBuilder().startObject() + .field("name", "Document") + .startArray("methods") + .startObject() + .field("name", "add") + .field("return_type", "void") + .startArray("parameters") + .startObject() + .field("name", "field") + .field("type", "IndexableField") + .endObject() + .endArray() + .endObject() + .startObject() + .field("name", "removeField") + .field("return_type", "void") + .startArray("parameters") + .startObject() + .field("name", "name") + .field("type", "String") + .endObject() + .endArray() + .endObject() + .startObject() + .field("name", "removeFields") + .field("return_type", "void") + .startArray("parameters") + .startObject() + .field("name", "name") + .field("type", "String") + .endObject() + .endArray() + .endObject() + .endArray() + .endObject() + ); refresh(); assertNoFailuresAndResponse( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java index 28c186c559dff..31944183a9e64 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java @@ -81,6 +81,9 @@ public void setupSuiteScopeCluster() throws Exception { varNumeric = (i * (varNumeric + Math.pow(avgNumericDiff, 2.0)) + Math.pow(resNumeric, 2.0)) / (i + 1); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java index 10e3649e9f161..f9d68e52d508f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java @@ -127,6 +127,9 @@ public void setupSuiteScopeCluster() throws Exception { builders.add(prepareIndex("new_index").setSource(Collections.emptyMap())); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -898,11 +901,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("i", "type=integer") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource(jsonBuilder().startObject().field("i", 1).endObject()), prepareIndex("cache_test_idx").setId("2").setSource(jsonBuilder().startObject().field("i", 2).endObject()) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java index 7585ed42da830..d5ac2e8ee84ee 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.search.aggregations.bucket; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.BucketOrder; @@ -125,7 +126,9 @@ private void insertIdx1(List values1, List values2) throws Excep source.startObject().field("field2", value1).endObject(); } source.endArray().endObject(); - indexRandom(false, prepareIndex("idx1").setRouting("1").setSource(source)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("idx1").setRouting("1").setSource(source); + indexRandom(false, indexRequestBuilder); + indexRequestBuilder.request().decRef(); } private void insertIdx2(String[][] values) throws Exception { @@ -138,7 +141,9 @@ private void insertIdx2(String[][] values) throws Exception { source.endArray().endObject(); } source.endArray().endObject(); - indexRandom(false, prepareIndex("idx2").setRouting("1").setSource(source)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("idx2").setRouting("1").setSource(source); + indexRandom(false, indexRequestBuilder); + indexRequestBuilder.request().decRef(); } public void testSimpleReverseNestedToRoot() throws Exception { @@ -535,7 +540,7 @@ public void testSameParentDocHavingMultipleBuckets() throws Exception { .endObject(); assertAcked(prepareCreate("idx3").setSettings(indexSettings(1, 0)).setMapping(mapping)); - prepareIndex("idx3").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("idx3").setId("1") .setRefreshPolicy(IMMEDIATE) .setSource( jsonBuilder().startObject() @@ -608,8 +613,9 @@ public void testSameParentDocHavingMultipleBuckets() throws Exception { .endObject() .endArray() .endObject() - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNoFailuresAndResponse( prepareSearch("idx3").addAggregation( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java index 00779ba9b256e..ba57bf51124f1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java @@ -75,12 +75,8 @@ public void setupSuiteScopeCluster() throws Exception { for (int i = 0; i < data.length; i++) { String[] parts = data[i].split(","); - prepareIndex("test").setId("" + i) - .setSource("author", parts[5], "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])) - .get(); - prepareIndex("idx_unmapped_author").setId("" + i) - .setSource("name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])) - .get(); + indexDoc("test", "" + i, "author", parts[5], "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])); + indexDoc("idx_unmapped_author", "" + i, "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])); // frequent refresh makes it more likely that more segments are created, hence we may parallelize the search across slices indicesAdmin().refresh(new RefreshRequest()).get(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java index 94db8926f59e7..5534d6e1fdddb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java @@ -22,6 +22,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; +import java.util.List; + import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateRange; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; @@ -80,7 +82,11 @@ public void setupSuiteScopeCluster() throws Exception { ) ); - indexRandom(true, indexDoc("2014-01-01", 1), indexDoc("2014-01-02", 2), indexDoc("2014-01-04", 3)); + List builders = List.of(indexDoc("2014-01-01", 1), indexDoc("2014-01-02", 2), indexDoc("2014-01-04", 3)); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index da1376a300728..ba808bf7232a2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -213,6 +213,9 @@ public void testPopularTermManyDeletedDocs() throws Exception { indexRequestBuilderList.add(prepareIndex(INDEX_NAME).setId("3").setSource(TEXT_FIELD, cat2v1, CLASS_FIELD, "2")); indexRequestBuilderList.add(prepareIndex(INDEX_NAME).setId("4").setSource(TEXT_FIELD, cat2v2, CLASS_FIELD, "2")); indexRandom(true, false, indexRequestBuilderList); + for (IndexRequestBuilder builder : indexRequestBuilderList) { + builder.request().decRef(); + } // Now create some holes in the index with selective deletes caused by updates. // This is the scenario that caused this issue https://github.com/elastic/elasticsearch/issues/7951 @@ -225,6 +228,9 @@ public void testPopularTermManyDeletedDocs() throws Exception { indexRequestBuilderList.add(prepareIndex(INDEX_NAME).setId("1").setSource(TEXT_FIELD, text, CLASS_FIELD, "1")); } indexRandom(true, false, indexRequestBuilderList); + for (IndexRequestBuilder builder : indexRequestBuilderList) { + builder.request().decRef(); + } SearchRequestBuilder request; if (randomBoolean()) { @@ -459,6 +465,9 @@ private void indexEqualTestData() throws ExecutionException, InterruptedExceptio indexRequestBuilders.add(prepareIndex("test").setId("" + i).setSource("class", parts[0], "text", parts[1])); } indexRandom(true, false, indexRequestBuilders); + for (IndexRequestBuilder builder : indexRequestBuilders) { + builder.request().decRef(); + } } public void testScriptScore() throws ExecutionException, InterruptedException, IOException { @@ -536,6 +545,9 @@ private void indexRandomFrequencies01(String type) throws ExecutionException, In ); } indexRandom(true, indexRequestBuilderList); + for (IndexRequestBuilder builder : indexRequestBuilderList) { + builder.request().decRef(); + } } public void testReduceFromSeveralShards() throws IOException, ExecutionException, InterruptedException { @@ -551,11 +563,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("s", "type=long", "t", "type=text") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1, "t", "foo"), prepareIndex("cache_test_idx").setId("2").setSource("s", 2, "t", "bar") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java index 25d6dfb850bbc..ad8390a54b825 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java @@ -146,6 +146,9 @@ public void setupSuiteScopeCluster() throws Exception { buildIndex(shard5DocsPerTerm, "idx_fixed_docs_5", 5, builders); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java index c8e23d65b4e37..e8faa54281945 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java @@ -59,6 +59,9 @@ public void testShardMinDocCountSignificantTermsTest() throws Exception { addTermsDocs("6", 3, 1, indexBuilders); addTermsDocs("7", 0, 3, indexBuilders);// make sure the terms all get score > 0 except for this one indexRandom(true, false, indexBuilders); + for (IndexRequestBuilder builder : indexBuilders) { + builder.request().decRef(); + } // first, check that indeed when not setting the shardMinDocCount parameter 0 terms are returned assertNoFailuresAndResponse( @@ -127,6 +130,9 @@ public void testShardMinDocCountTermsTest() throws Exception { addTermsDocs("5", 3, indexBuilders);// low score but high doc freq addTermsDocs("6", 3, indexBuilders); indexRandom(true, false, indexBuilders); + for (IndexRequestBuilder builder : indexBuilders) { + builder.request().decRef(); + } // first, check that indeed when not setting the shardMinDocCount parameter 0 terms are returned assertNoFailuresAndResponse( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java index ba9f4974b436d..b35051790d814 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java @@ -30,7 +30,9 @@ public class RareTermsIT extends ESSingleNodeTestCase { private void indexDocs(int numDocs) { try (BulkRequestBuilder bulk = client().prepareBulk()) { for (int i = 0; i < numDocs; ++i) { - bulk.add(new IndexRequest(index).source("{\"str_value\" : \"s" + i + "\"}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index).source("{\"str_value\" : \"s" + i + "\"}", XContentType.JSON); + bulk.add(indexRequest); + indexRequest.decRef(); } assertNoFailures(bulk.get()); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java index 2277f4415d4db..c2348dfd5b0b7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java @@ -169,6 +169,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } createIndex("idx_unmapped"); ensureSearchable(); } @@ -1190,11 +1193,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=keyword") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", "foo"), prepareIndex("cache_test_idx").setId("2").setSource("s", "bar") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java index 5e086766dc711..d9afd172caaa4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java @@ -33,15 +33,16 @@ public void testRequestBreaker() throws Exception { final String requestBreaker = randomIntBetween(1, 10000) + "kb"; logger.info("--> Using request breaker setting: {}", requestBreaker); - indexRandom( - true, - IntStream.range(0, randomIntBetween(10, 1000)) - .mapToObj( - i -> prepareIndex("test").setId("id_" + i) - .setSource(Map.of("field0", randomAlphaOfLength(5), "field1", randomAlphaOfLength(5))) - ) - .toArray(IndexRequestBuilder[]::new) - ); + IndexRequestBuilder[] builders = IntStream.range(0, randomIntBetween(10, 1000)) + .mapToObj( + i -> prepareIndex("test").setId("id_" + i) + .setSource(Map.of("field0", randomAlphaOfLength(5), "field1", randomAlphaOfLength(5))) + ) + .toArray(IndexRequestBuilder[]::new); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } updateClusterSettings( Settings.builder().put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), requestBreaker) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java index e15ad15bb4e3a..a1e1f2e8cd83a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; @@ -908,11 +909,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java index 3f5d8e441dc44..91a5fbdee9689 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.apache.logging.log4j.LogManager; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; @@ -571,11 +572,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java index 0dbc811a7debc..b77745f5218a6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.apache.logging.log4j.LogManager; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.plugins.Plugin; @@ -544,11 +545,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java index 06f43416eb03a..ddf5dfc48083f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java @@ -107,6 +107,9 @@ public void setupSuiteScopeCluster() throws Exception { multiValueExactMAD = calculateMAD(multiValueSample); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } prepareCreate("empty_bucket_idx").setMapping("value", "type=integer").get(); @@ -118,6 +121,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -498,11 +504,14 @@ public void testScriptCaching() throws Exception { .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java index d40264d9facf0..03a3007bc1673 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java @@ -280,6 +280,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // creating an index to test the empty buckets functionality. The way it // works is by indexing @@ -317,6 +320,9 @@ public void setupSuiteScopeCluster() throws Exception { """, MockScriptPlugin.NAME)), XContentType.JSON)); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -1142,11 +1148,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java index f97d886ae8df6..96f76151883b5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.settings.Settings; @@ -237,11 +238,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java index 37524dabe7f09..cad645c1c2a4d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java @@ -72,6 +72,9 @@ public void setupSuiteScopeCluster() throws Exception { builders.add(prepareIndex("new_index").setSource("transit_mode", "train", "route_length_miles", 100.2)); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -211,11 +214,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java index 47c443a58eeda..7bfde5665e1f8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.apache.logging.log4j.LogManager; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.plugins.Plugin; @@ -488,11 +489,11 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, - prepareIndex("cache_test_idx").setId("1").setSource("s", 1), - prepareIndex("cache_test_idx").setId("2").setSource("s", 2) - ); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("cache_test_idx").setId("1").setSource("s", 1); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("cache_test_idx").setId("2").setSource("s", 2); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java index 98086451c3456..1b06e16de0e34 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.apache.logging.log4j.LogManager; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.plugins.Plugin; @@ -460,11 +461,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index 6cf274cb69fb3..8ee33188a978b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -291,6 +291,9 @@ public void setupSuiteScopeCluster() throws Exception { ); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } @@ -1083,11 +1086,14 @@ public void testScriptCaching() throws Exception { Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1) ) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java index 7c5ab6600e365..7b78569141233 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; @@ -49,11 +50,11 @@ public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); createIndex("idx_unmapped"); for (int i = 0; i < 10; i++) { - prepareIndex("idx").setId("" + i) - .setSource( - jsonBuilder().startObject().field("value", i + 1).startArray("values").value(i + 2).value(i + 3).endArray().endObject() - ) - .get(); + index( + "idx", + "" + i, + jsonBuilder().startObject().field("value", i + 1).startArray("values").value(i + 2).value(i + 3).endArray().endObject() + ); } indicesAdmin().prepareFlush().get(); indicesAdmin().prepareRefresh().get(); @@ -218,11 +219,14 @@ public void testScriptCaching() throws Exception { prepareCreate("cache_test_idx").setMapping("d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) ); - indexRandom( - true, + List builders = List.of( prepareIndex("cache_test_idx").setId("1").setSource("s", 1), prepareIndex("cache_test_idx").setId("2").setSource("s", 2) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Make sure we are starting with a clear cache assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipeLineAggregationTestCase.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipeLineAggregationTestCase.java index 7509cf3815085..bec9d908be287 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipeLineAggregationTestCase.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipeLineAggregationTestCase.java @@ -110,6 +110,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } ensureSearchable(); histoName = randomName(); termsName = randomName(); @@ -470,7 +473,10 @@ public void testFieldIsntWrittenOutTwice() throws Exception { .field("@timestamp", "2018-07-08T08:07:00.599Z") .endObject(); // end::noformat - prepareIndex("foo_2").setSource(docBuilder).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("foo_2").setSource(docBuilder) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); indicesAdmin().prepareRefresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java index dc612d6bad5ce..649b548828dd6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java @@ -138,6 +138,9 @@ public void setupSuiteScopeCluster() throws Exception { } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java index 6562c485b9204..69526abfe60f1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java @@ -91,6 +91,9 @@ public void setupSuiteScopeCluster() throws Exception { } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java index 1525496176418..289de8ae42d74 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java @@ -109,7 +109,7 @@ private void buildRedIndex(int numShards) throws Exception { assertAcked(prepareCreate("test").setSettings(indexSettings(numShards, 0))); ensureGreen(); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId("" + i).setSource("field1", "value1").get(); + indexDoc("test", "" + i, "field1", "value1"); } refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java index 68d00321848eb..700e20f7484c5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java @@ -54,7 +54,7 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas) if (createIndex) { createIndex("test"); } - prepareIndex("test").setId(id).setSource("field", "test").get(); + indexDoc("test", id, "field", "test"); BroadcastResponse refreshResponse = indicesAdmin().prepareRefresh("test").get(); // at least one shard should be successful when refreshing assertThat(refreshResponse.getSuccessfulShards(), greaterThanOrEqualTo(1)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java index 26d81f672d650..21437cdbcdd4c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java @@ -59,7 +59,10 @@ private void testSearchAndRelocateConcurrently(final int numberOfReplicas) throw ) ); } - indexRandom(true, indexBuilders.toArray(new IndexRequestBuilder[indexBuilders.size()])); + indexRandom(true, indexBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexBuilders) { + indexRequestBuilder.request().decRef(); + } assertHitCount(prepareSearch(), (numDocs)); final int numIters = scaledRandomIntBetween(5, 20); for (int i = 0; i < numIters; i++) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java index 6985ebb17386c..e74ddc7a923d4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java @@ -14,6 +14,7 @@ import org.apache.lucene.tests.util.English; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.common.Strings; @@ -98,8 +99,9 @@ public void testRandomExceptions() throws IOException, InterruptedException, Exe int numCreated = 0; boolean[] added = new boolean[numDocs]; for (int i = 0; i < numDocs; i++) { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); try { - DocWriteResponse indexResponse = prepareIndex("test").setId("" + i) + DocWriteResponse indexResponse = indexRequestBuilder.setId("" + i) .setTimeout(TimeValue.timeValueSeconds(1)) .setSource("test", English.intToEnglish(i)) .get(); @@ -107,7 +109,9 @@ public void testRandomExceptions() throws IOException, InterruptedException, Exe numCreated++; added[i] = true; } - } catch (ElasticsearchException ex) {} + } catch (ElasticsearchException ex) {} finally { + indexRequestBuilder.request().decRef(); + } } logger.info("Start Refresh"); // don't assert on failures here diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java index 07d976437c24c..c77777bccdbc1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.common.Strings; @@ -82,7 +83,7 @@ public void testRandomDirectoryIOExceptions() throws IOException, InterruptedExc numInitialDocs = between(10, 100); ensureGreen(); for (int i = 0; i < numInitialDocs; i++) { - prepareIndex("test").setId("init" + i).setSource("test", "init").get(); + indexDoc("test", "init" + i, "test", "init"); } indicesAdmin().prepareRefresh("test").execute().get(); indicesAdmin().prepareFlush("test").execute().get(); @@ -120,8 +121,9 @@ public void testRandomDirectoryIOExceptions() throws IOException, InterruptedExc boolean[] added = new boolean[numDocs]; for (int i = 0; i < numDocs; i++) { added[i] = false; + IndexRequestBuilder indexRequestBuilder = prepareIndex("test"); try { - DocWriteResponse indexResponse = prepareIndex("test").setId(Integer.toString(i)) + DocWriteResponse indexResponse = indexRequestBuilder.setId(Integer.toString(i)) .setTimeout(TimeValue.timeValueSeconds(1)) .setSource("test", English.intToEnglish(i)) .get(); @@ -129,7 +131,9 @@ public void testRandomDirectoryIOExceptions() throws IOException, InterruptedExc numCreated++; added[i] = true; } - } catch (ElasticsearchException ex) {} + } catch (ElasticsearchException ex) {} finally { + indexRequestBuilder.request().decRef(); + } } ESIntegTestCase.NumShards numShards = getNumShards("test"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java index d4a4debbd61d6..0a6aaad662347 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java @@ -83,7 +83,12 @@ private Set prepareData(int numShards) throws Exception { } private void index(String id, String nameValue, int age) throws IOException { - client().index(new IndexRequest("test").id(id).source(source(id, nameValue, age))).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id(id).source(source(id, nameValue, age)); + try { + client().index(indexRequest).actionGet(); + } finally { + indexRequest.decRef(); + } } private XContentBuilder source(String id, String nameValue, int age) throws IOException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java index 582df3a5bb396..d257bc3955887 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java @@ -11,6 +11,7 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.PointValues; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.CanMatchNodeRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchTransportService; @@ -117,7 +118,9 @@ int createIndexAndIndexDocs(String cluster, String index, int numberOfShards, lo ); int numDocs = between(100, 500); for (int i = 0; i < numDocs; i++) { - client.prepareIndex(index).setSource("position", i, "@timestamp", timestamp + i).get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index).setSource("position", i, "@timestamp", timestamp + i); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } if (exposeTimestamp) { client.admin().indices().prepareClose(index).get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java index 1229141c9bdb5..dc9349a1c096c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchShardsGroup; @@ -95,7 +96,9 @@ protected boolean reuseClusters() { private int indexDocs(Client client, String index) { int numDocs = between(1, 10); for (int i = 0; i < numDocs; i++) { - client.prepareIndex(index).setSource("f", "v").get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index).setSource("f", "v"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } client.admin().indices().prepareRefresh(index).get(); return numDocs; @@ -322,11 +325,17 @@ public void testLookupFields() throws Exception { .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5))) .get(); try (BulkRequestBuilder bulkRequestBuilder = cluster("cluster_a").client().prepareBulk("users")) { - bulkRequestBuilder.add(new IndexRequest().id("a").source("name", "Remote A")) - .add(new IndexRequest().id("b").source("name", "Remote B")) - .add(new IndexRequest().id("c").source("name", "Remote C")) + IndexRequest indexRequest1 = new IndexRequest().id("a").source("name", "Remote A"); + IndexRequest indexRequest2 = new IndexRequest().id("b").source("name", "Remote B"); + IndexRequest indexRequest3 = new IndexRequest().id("c").source("name", "Remote C"); + bulkRequestBuilder.add(indexRequest1) + .add(indexRequest2) + .add(indexRequest3) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); + indexRequest1.decRef(); + indexRequest2.decRef(); + indexRequest3.decRef(); } client().admin() @@ -335,11 +344,17 @@ public void testLookupFields() throws Exception { .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5))) .get(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk("users")) { - bulkRequestBuilder.add(new IndexRequest().id("a").source("name", "Local A")) - .add(new IndexRequest().id("b").source("name", "Local B")) - .add(new IndexRequest().id("c").source("name", "Local C")) + IndexRequest indexRequest1 = new IndexRequest().id("a").source("name", "Local A"); + IndexRequest indexRequest2 = new IndexRequest().id("b").source("name", "Local B"); + IndexRequest indexRequest3 = new IndexRequest().id("c").source("name", "Local C"); + bulkRequestBuilder.add(indexRequest1) + .add(indexRequest2) + .add(indexRequest3) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); + indexRequest1.decRef(); + indexRequest2.decRef(); + indexRequest3.decRef(); } // Setup calls on the local cluster @@ -350,10 +365,11 @@ public void testLookupFields() throws Exception { .setMapping("from_user", "type=keyword", "to_user", "type=keyword") .get(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk("local_calls")) { - bulkRequestBuilder.add(new IndexRequest().source("from_user", "a", "to_user", List.of("b", "c"), "duration", 95)) - .add(new IndexRequest().source("from_user", "a", "to_user", "b", "duration", 25)) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + IndexRequest indexRequest1 = new IndexRequest().source("from_user", "a", "to_user", List.of("b", "c"), "duration", 95); + IndexRequest indexRequest2 = new IndexRequest().source("from_user", "a", "to_user", "b", "duration", 25); + bulkRequestBuilder.add(indexRequest1).add(indexRequest2).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + indexRequest1.decRef(); + indexRequest2.decRef(); } // Setup calls on the remote cluster @@ -365,11 +381,17 @@ public void testLookupFields() throws Exception { .setMapping("from_user", "type=keyword", "to_user", "type=keyword") .get(); try (BulkRequestBuilder bulkRequestBuilder = cluster("cluster_a").client().prepareBulk("remote_calls")) { - bulkRequestBuilder.add(new IndexRequest().source("from_user", "a", "to_user", "b", "duration", 45)) - .add(new IndexRequest().source("from_user", "unknown_caller", "to_user", "c", "duration", 50)) - .add(new IndexRequest().source("from_user", List.of("a", "b"), "to_user", "c", "duration", 60)) + IndexRequest indexRequest1 = new IndexRequest().source("from_user", "a", "to_user", "b", "duration", 45); + IndexRequest indexRequest2 = new IndexRequest().source("from_user", "unknown_caller", "to_user", "c", "duration", 50); + IndexRequest indexRequest3 = new IndexRequest().source("from_user", List.of("a", "b"), "to_user", "c", "duration", 60); + bulkRequestBuilder.add(indexRequest1) + .add(indexRequest2) + .add(indexRequest3) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); + indexRequest1.decRef(); + indexRequest2.decRef(); + indexRequest3.decRef(); } final String runtimeMappingSource = """ @@ -523,7 +545,9 @@ public void testSearchShardsWithIndexNameQuery() { .get(); int numDocs = randomIntBetween(100, 500); for (int i = 0; i < numDocs; i++) { - remoteClient.prepareIndex("my_index").setSource("f", "v").get(); + IndexRequestBuilder indexRequestBuilder = remoteClient.prepareIndex("my_index").setSource("f", "v"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } remoteClient.admin().indices().prepareRefresh("my_index").get(); String[] indices = new String[] { "my_index" }; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java index d21619f4e6f89..ace5dd6418ab3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.ccs; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse.Cluster; @@ -731,7 +732,12 @@ private int indexDocs(Client client, String index) { if (i == numDocs - 1) { ts = LATEST_TIMESTAMP; } - client.prepareIndex(index).setSource("f", "v", "@timestamp", ts).get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); + try { + indexRequestBuilder.setSource("f", "v", "@timestamp", ts).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } client.admin().indices().prepareRefresh(index).get(); return numDocs; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java index 8b6f4112cfc17..f2d25e5756c19 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.ccs; import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -49,7 +50,12 @@ protected boolean reuseClusters() { private int indexDocs(Client client, String field, String index) { int numDocs = between(1, 200); for (int i = 0; i < numDocs; i++) { - client.prepareIndex(index).setSource(field, "v" + i).get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); + try { + indexRequestBuilder.setSource(field, "v" + i).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } client.admin().indices().prepareRefresh(index).get(); return numDocs; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index 15afd6897a40e..565d739f58ce5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -66,8 +66,10 @@ public void testPlugin() throws Exception { ) .get(); - client().index(new IndexRequest("test").id("1").source(jsonBuilder().startObject().field("test", "I am sam i am").endObject())) - .actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("test", "I am sam i am").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); indicesAdmin().prepareRefresh().get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index b536db040e39f..92c8e174f44fa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -140,6 +140,9 @@ public void testSimpleNested() throws Exception { ) ); indexRandom(true, requests); + for (IndexRequestBuilder builder : requests) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("articles").setQuery( @@ -243,6 +246,9 @@ public void testRandomNested() throws Exception { requestBuilders.add(prepareIndex("idx").setId(Integer.toString(i)).setSource(source)); } indexRandom(true, requestBuilders); + for (IndexRequestBuilder builder : requestBuilders) { + builder.request().decRef(); + } int size = randomIntBetween(0, numDocs); BoolQueryBuilder boolQuery = new BoolQueryBuilder(); @@ -360,6 +366,9 @@ public void testNestedMultipleLayers() throws Exception { ) ); indexRandom(true, requests); + for (IndexRequestBuilder builder : requests) { + builder.request().decRef(); + } // Check we can load the first doubly-nested document. assertNoFailuresAndResponse( @@ -529,6 +538,9 @@ public void testNestedDefinedAsObject() throws Exception { ) ); indexRandom(true, requests); + for (IndexRequestBuilder builder : requests) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("articles").setQuery( @@ -600,6 +612,9 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception { ) ); indexRandom(true, requests); + for (IndexRequestBuilder builder : requests) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("articles").setQuery( @@ -673,6 +688,9 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception { ) ); indexRandom(true, requests); + for (IndexRequestBuilder builder : requests) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("articles").setQuery( nestedQuery("comments.messages", matchQuery("comments.messages.message", "fox"), ScoreMode.Avg).innerHit( @@ -775,6 +793,9 @@ public void testMatchesQueriesNestedInnerHits() throws Exception { } indexRandom(true, requests); + for (IndexRequestBuilder indexRequestBuilder : requests) { + indexRequestBuilder.request().decRef(); + } waitForRelocation(ClusterHealthStatus.GREEN); QueryBuilder query = boolQuery().should(termQuery("nested1.n_field1", "n_value1_1").queryName("test1")) @@ -809,27 +830,27 @@ public void testMatchesQueriesNestedInnerHits() throws Exception { public void testNestedSource() throws Exception { assertAcked(prepareCreate("index1").setMapping("comments", "type=nested")); - prepareIndex("index1").setId("1") - .setSource( - jsonBuilder().startObject() - .field("message", "quick brown fox") - .startArray("comments") - .startObject() - .field("message", "fox eat quick") - .field("x", "y") - .endObject() - .startObject() - .field("message", "fox ate rabbit x y z") - .field("x", "y") - .endObject() - .startObject() - .field("message", "rabbit got away") - .field("x", "y") - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "index1", + "1", + jsonBuilder().startObject() + .field("message", "quick brown fox") + .startArray("comments") + .startObject() + .field("message", "fox eat quick") + .field("x", "y") + .endObject() + .startObject() + .field("message", "fox ate rabbit x y z") + .field("x", "y") + .endObject() + .startObject() + .field("message", "rabbit got away") + .field("x", "y") + .endObject() + .endArray() + .endObject() + ); refresh(); // the field name (comments.message) used for source filtering should be the same as when using that field for @@ -909,8 +930,8 @@ public void testNestedSource() throws Exception { public void testInnerHitsWithIgnoreUnmapped() throws Exception { assertAcked(prepareCreate("index1").setMapping("nested_type", "type=nested")); createIndex("index2"); - prepareIndex("index1").setId("1").setSource("nested_type", Collections.singletonMap("key", "value")).get(); - prepareIndex("index2").setId("3").setSource("key", "value").get(); + indexDoc("index1", "1", "nested_type", Collections.singletonMap("key", "value")); + indexDoc("index2", "3", "key", "value"); refresh(); assertSearchHitsWithoutFailures( @@ -931,12 +952,13 @@ public void testUseMaxDocInsteadOfSize() throws Exception { Settings.builder().put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), ArrayUtil.MAX_ARRAY_LENGTH), "index2" ); - prepareIndex("index2").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("index2").setId("1") .setSource( jsonBuilder().startObject().startArray("nested").startObject().field("field", "value1").endObject().endArray().endObject() ) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); QueryBuilder query = nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit( new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1) @@ -946,12 +968,13 @@ public void testUseMaxDocInsteadOfSize() throws Exception { public void testTooHighResultWindow() throws Exception { assertAcked(prepareCreate("index2").setMapping("nested", "type=nested")); - prepareIndex("index2").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("index2").setId("1") .setSource( jsonBuilder().startObject().startArray("nested").startObject().field("field", "value1").endObject().endArray().endObject() ) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCountAndNoFailures( prepareSearch("index2").setQuery( nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java index c996725e6285e..37532095c720b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java @@ -41,9 +41,9 @@ public void testSimpleMatchedQueryFromFilteredQuery() throws Exception { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("name", "test1", "number", 1).get(); - prepareIndex("test").setId("2").setSource("name", "test2", "number", 2).get(); - prepareIndex("test").setId("3").setSource("name", "test3", "number", 3).get(); + indexDoc("test", "1", "name", "test1", "number", 1); + indexDoc("test", "2", "name", "test2", "number", 2); + indexDoc("test", "3", "name", "test3", "number", 3); refresh(); assertResponse( @@ -99,9 +99,9 @@ public void testSimpleMatchedQueryFromTopLevelFilter() throws Exception { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("name", "test", "title", "title1").get(); - prepareIndex("test").setId("2").setSource("name", "test").get(); - prepareIndex("test").setId("3").setSource("name", "test").get(); + indexDoc("test", "1", "name", "test", "title", "title1"); + indexDoc("test", "2", "name", "test"); + indexDoc("test", "3", "name", "test"); refresh(); assertResponse( @@ -159,9 +159,9 @@ public void testSimpleMatchedQueryFromTopLevelFilterAndFilteredQuery() throws Ex createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("name", "test", "title", "title1").get(); - prepareIndex("test").setId("2").setSource("name", "test", "title", "title2").get(); - prepareIndex("test").setId("3").setSource("name", "test", "title", "title3").get(); + indexDoc("test", "1", "name", "test", "title", "title1"); + indexDoc("test", "2", "name", "test", "title", "title2"); + indexDoc("test", "3", "name", "test", "title", "title3"); refresh(); assertResponse( @@ -208,7 +208,7 @@ public void testRegExpQuerySupportsName() { createIndex("test1"); ensureGreen(); - prepareIndex("test1").setId("1").setSource("title", "title1").get(); + indexDoc("test1", "1", "title", "title1"); refresh(); assertResponse(prepareSearch().setQuery(QueryBuilders.regexpQuery("title", "title1").queryName("regex")), response -> { @@ -230,7 +230,7 @@ public void testPrefixQuerySupportsName() { createIndex("test1"); ensureGreen(); - prepareIndex("test1").setId("1").setSource("title", "title1").get(); + indexDoc("test1", "1", "title", "title1"); refresh(); assertResponse(prepareSearch().setQuery(QueryBuilders.prefixQuery("title", "title").queryName("prefix")), response -> { @@ -252,7 +252,7 @@ public void testFuzzyQuerySupportsName() { createIndex("test1"); ensureGreen(); - prepareIndex("test1").setId("1").setSource("title", "title1").get(); + indexDoc("test1", "1", "title", "title1"); refresh(); assertResponse(prepareSearch().setQuery(QueryBuilders.fuzzyQuery("title", "titel1").queryName("fuzzy")), response -> { @@ -274,7 +274,7 @@ public void testWildcardQuerySupportsName() { createIndex("test1"); ensureGreen(); - prepareIndex("test1").setId("1").setSource("title", "title1").get(); + indexDoc("test1", "1", "title", "title1"); refresh(); assertResponse(prepareSearch().setQuery(QueryBuilders.wildcardQuery("title", "titl*").queryName("wildcard")), response -> { @@ -296,7 +296,7 @@ public void testSpanFirstQuerySupportsName() { createIndex("test1"); ensureGreen(); - prepareIndex("test1").setId("1").setSource("title", "title1 title2").get(); + indexDoc("test1", "1", "title", "title1 title2"); refresh(); assertResponse( @@ -324,8 +324,8 @@ public void testMatchedWithShould() throws Exception { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("content", "Lorem ipsum dolor sit amet").get(); - prepareIndex("test").setId("2").setSource("content", "consectetur adipisicing elit").get(); + indexDoc("test", "1", "content", "Lorem ipsum dolor sit amet"); + indexDoc("test", "2", "content", "consectetur adipisicing elit"); refresh(); // Execute search at least two times to load it in cache @@ -361,7 +361,7 @@ public void testMatchedWithWrapperQuery() throws Exception { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("content", "Lorem ipsum dolor sit amet").get(); + indexDoc("test", "1", "content", "Lorem ipsum dolor sit amet"); refresh(); MatchQueryBuilder matchQueryBuilder = matchQuery("content", "amet").queryName("abc"); @@ -384,8 +384,8 @@ public void testMatchedWithRescoreQuery() throws Exception { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("content", "hello world").get(); - prepareIndex("test").setId("2").setSource("content", "hello you").get(); + indexDoc("test", "1", "content", "hello world"); + indexDoc("test", "2", "content", "hello you"); refresh(); assertResponse( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java index 8aaa1bce252d7..63e144d399bde 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.search.fetch.subphase.highlight; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -18,6 +19,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHighlight; @@ -37,11 +39,14 @@ protected Collection> nodePlugins() { @Before protected void setup() throws Exception { - indexRandom( - true, + List builders = List.of( prepareIndex("test").setId("1").setSource("name", "arbitrary content", "other_name", "foo", "other_other_name", "bar"), prepareIndex("test").setId("2").setSource("other_name", "foo", "other_other_name", "bar") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } } public void testThatCustomHighlightersAreSupported() throws IOException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index ab72dbd4db707..90611f385ba00 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -16,10 +16,10 @@ import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.analysis.MockTokenizer; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; @@ -68,6 +68,7 @@ import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.boostingQuery; import static org.elasticsearch.index.query.QueryBuilders.combinedFieldsQuery; @@ -129,12 +130,16 @@ public void testHighlightingWithKeywordIgnoreBoundaryScanner() throws IOExceptio .endObject(); mappings.endObject(); assertAcked(prepareCreate("test").setMapping(mappings)); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().array("tags", "foo bar", "foo bar", "foo bar", "foo baz").field("sort", 1).endObject()) - .get(); - prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().array("tags", "foo baz", "foo baz", "foo baz", "foo bar").field("sort", 2).endObject()) - .get(); + index( + "test", + "1", + jsonBuilder().startObject().array("tags", "foo bar", "foo bar", "foo bar", "foo baz").field("sort", 1).endObject() + ); + index( + "test", + "2", + jsonBuilder().startObject().array("tags", "foo baz", "foo baz", "foo baz", "foo bar").field("sort", 2).endObject() + ); refresh(); for (BoundaryScannerType scanner : BoundaryScannerType.values()) { @@ -164,7 +169,7 @@ public void testHighlightingWithStoredKeyword() throws IOException { .endObject(); mappings.endObject(); assertAcked(prepareCreate("test").setMapping(mappings)); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("text", "foo").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("text", "foo").endObject()); refresh(); assertResponse( prepareSearch().setQuery(matchQuery("text", "foo")).highlighter(new HighlightBuilder().field(new Field("text"))), @@ -188,7 +193,7 @@ public void testHighlightingWithWildcardName() throws IOException { .endObject(); mappings.endObject(); assertAcked(prepareCreate("test").setMapping(mappings)); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("text", "text").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("text", "text").endObject()); refresh(); for (String type : ALL_TYPES) { assertResponse( @@ -217,7 +222,7 @@ public void testFieldAlias() throws IOException { .endObject(); assertAcked(prepareCreate("test").setMapping(mappings)); - prepareIndex("test").setId("1").setSource("text", "foo").get(); + indexDoc("test", "1", "text", "foo"); refresh(); for (String type : ALL_TYPES) { @@ -249,7 +254,7 @@ public void testFieldAliasWithSourceLookup() throws IOException { .endObject(); assertAcked(prepareCreate("test").setMapping(mappings)); - prepareIndex("test").setId("1").setSource("text", "foo bar").get(); + indexDoc("test", "1", "text", "foo bar"); refresh(); for (String type : ALL_TYPES) { @@ -278,7 +283,7 @@ public void testFieldAliasWithWildcardField() throws IOException { .endObject(); assertAcked(prepareCreate("test").setMapping(mappings)); - prepareIndex("test").setId("1").setSource("keyword", "foo").get(); + indexDoc("test", "1", "keyword", "foo"); refresh(); HighlightBuilder builder = new HighlightBuilder().field(new Field("al*")).requireFieldMatch(false); @@ -312,9 +317,7 @@ public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOExc .endObject(); mappings.endObject(); assertAcked(prepareCreate("test").setMapping(mappings)); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("unstored_text", "text").field("text", "text").endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().field("unstored_text", "text").field("text", "text").endObject()); refresh(); for (String type : ALL_TYPES) { assertResponse( @@ -337,7 +340,7 @@ public void testHighTermFrequencyDoc() throws IOException { for (int i = 0; i < 6000; i++) { builder.append("abc").append(" "); } - prepareIndex("test").setId("1").setSource("name", builder.toString()).get(); + indexDoc("test", "1", "name", builder.toString()); refresh(); assertResponse( prepareSearch().setQuery(constantScoreQuery(matchQuery("name", "abc"))).highlighter(new HighlightBuilder().field("name")), @@ -357,14 +360,14 @@ public void testEnsureNoNegativeOffsets() throws Exception { ) ); - prepareIndex("test").setId("1") - .setSource( - "no_long_term", - "This is a test where foo is highlighed and should be highlighted", - "long_term", - "This is a test thisisaverylongwordandmakessurethisfails where foo is highlighed and should be highlighted" - ) - .get(); + indexDoc( + "test", + "1", + "no_long_term", + "This is a test where foo is highlighed and should be highlighted", + "long_term", + "This is a test thisisaverylongwordandmakessurethisfails where foo is highlighed and should be highlighted" + ); refresh(); assertHighlight( prepareSearch().setQuery(matchQuery("long_term", "thisisaverylongwordandmakessurethisfails foo highlighed")) @@ -439,7 +442,7 @@ public void testSourceLookupHighlightingUsingPlainHighlighter() throws Exception .endObject() ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertResponse( prepareSearch().setQuery(matchQuery("title", "bug")).highlighter(new HighlightBuilder().field("title", -1, 0)), @@ -514,7 +517,7 @@ public void testSourceLookupHighlightingUsingFastVectorHighlighter() throws Exce .endObject() ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertResponse( prepareSearch().setQuery(matchQuery("title", "bug")).highlighter(new HighlightBuilder().field("title", -1, 0)), @@ -593,7 +596,7 @@ public void testSourceLookupHighlightingUsingPostingsHighlighter() throws Except .endObject() ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertResponse( prepareSearch().setQuery(matchQuery("title", "bug")) @@ -654,9 +657,9 @@ public void testHighlightIssue1994() throws Exception { ); String[] titles = new String[] { "This is a test on the highlighting bug present in elasticsearch", "The bug is bugging us" }; - indexRandom(false, prepareIndex("test").setId("1").setSource("title", titles, "titleTV", titles)); + indexRandomAndDecrefRequests(false, prepareIndex("test").setId("1").setSource("title", titles, "titleTV", titles)); - indexRandom( + indexRandomAndDecrefRequests( true, prepareIndex("test").setId("2").setSource("titleTV", new String[] { "some text to highlight", "highlight other text" }) ); @@ -696,12 +699,14 @@ public void testGlobalHighlightingSettingsOverriddenAtFieldLevel() { createIndex("test"); ensureGreen(); - prepareIndex("test").setSource( + indexDoc( + "test", + null, "field1", new String[] { "this is a test", "this is the second test" }, "field2", new String[] { "this is another test", "yet another test" } - ).get(); + ); refresh(); logger.info("--> highlighting and searching on field1 and field2 produces different tags"); @@ -742,14 +747,16 @@ public void testHighlightingOnWildcardFields() throws Exception { ); ensureGreen(); - prepareIndex("test").setSource( + indexDoc( + "test", + null, "field-postings", "This is the first test sentence. Here is the second one.", "field-fvh", "This is the test with term_vectors", "field-plain", "This is the test for the plain highlighter" - ).get(); + ); refresh(); logger.info("--> highlighting and searching on field*"); @@ -775,7 +782,7 @@ public void testHighlightingOnWildcardFields() throws Exception { public void testPlainHighlighter() throws Exception { ensureGreen(); - prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog"); refresh(); SearchSourceBuilder source = searchSource().query(termQuery("field1", "test")) @@ -786,8 +793,7 @@ public void testPlainHighlighter() throws Exception { public void testPlainHighlighterOrder() throws Exception { ensureGreen(); - prepareIndex("test").setSource("field1", "The quick brown fox jumps over the lazy brown dog but to no suprise the dog doesn't care") - .get(); + indexDoc("test", null, "field1", "The quick brown fox jumps over the lazy brown dog but to no suprise the dog doesn't care"); refresh(); { @@ -832,7 +838,7 @@ public void testFastVectorHighlighter() throws Exception { assertAcked(prepareCreate("test").setMapping(type1TermVectorMapping())); ensureGreen(); - indexRandom( + indexRandomAndDecrefRequests( true, prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog") ); @@ -860,7 +866,10 @@ public void testHighlighterWithSentenceBoundaryScanner() throws Exception { assertAcked(prepareCreate("test").setMapping(type1TermVectorMapping())); ensureGreen(); - indexRandom(true, prepareIndex("test").setSource("field1", "A sentence with few words. Another sentence with even more words.")); + indexRandomAndDecrefRequests( + true, + prepareIndex("test").setSource("field1", "A sentence with few words. Another sentence with even more words.") + ); for (String type : new String[] { "unified", "fvh" }) { logger.info("--> highlighting and searching on 'field' with sentence boundary_scanner"); @@ -898,7 +907,10 @@ public void testHighlighterWithSentenceBoundaryScannerAndLocale() throws Excepti assertAcked(prepareCreate("test").setMapping(type1TermVectorMapping())); ensureGreen(); - indexRandom(true, prepareIndex("test").setSource("field1", "A sentence with few words. Another sentence with even more words.")); + indexRandomAndDecrefRequests( + true, + prepareIndex("test").setSource("field1", "A sentence with few words. Another sentence with even more words.") + ); for (String type : new String[] { "fvh", "unified" }) { logger.info("--> highlighting and searching on 'field' with sentence boundary_scanner"); @@ -938,7 +950,10 @@ public void testHighlighterWithWordBoundaryScanner() throws Exception { assertAcked(prepareCreate("test").setMapping(type1TermVectorMapping())); ensureGreen(); - indexRandom(true, prepareIndex("test").setSource("field1", "some quick and hairy brown:fox jumped over the lazy dog")); + indexRandomAndDecrefRequests( + true, + prepareIndex("test").setSource("field1", "some quick and hairy brown:fox jumped over the lazy dog") + ); logger.info("--> highlighting and searching on 'field' with word boundary_scanner"); for (String type : new String[] { "unified", "fvh" }) { @@ -966,7 +981,10 @@ public void testHighlighterWithWordBoundaryScannerAndLocale() throws Exception { assertAcked(prepareCreate("test").setMapping(type1TermVectorMapping())); ensureGreen(); - indexRandom(true, prepareIndex("test").setSource("field1", "some quick and hairy brown:fox jumped over the lazy dog")); + indexRandomAndDecrefRequests( + true, + prepareIndex("test").setSource("field1", "some quick and hairy brown:fox jumped over the lazy dog") + ); for (String type : new String[] { "unified", "fvh" }) { SearchSourceBuilder source = searchSource().query(termQuery("field1", "some")) @@ -1001,7 +1019,7 @@ public void testFVHManyMatches() throws Exception { // Index one megabyte of "t " over and over and over again String pattern = "t "; String value = new String(new char[1024 * 256 / pattern.length()]).replace("\0", pattern); - prepareIndex("test").setSource("field1", value).get(); + indexDoc("test", null, "field1", value); refresh(); final long[] tookDefaultPhrase = new long[1]; @@ -1278,7 +1296,7 @@ public void testFastVectorHighlighterManyDocs() throws Exception { indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)).setSource("field1", "test " + i); } logger.info("--> indexing docs"); - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); logger.info("--> searching explicitly on field1 and highlighting on it"); assertResponse( @@ -1321,7 +1339,7 @@ public void testSameContent() throws Exception { indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)) .setSource("title", "This is a test on the highlighting bug present in elasticsearch"); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertResponse( prepareSearch().setQuery(matchQuery("title", "bug")).highlighter(new HighlightBuilder().field("title", -1, 0)), @@ -1348,7 +1366,7 @@ public void testFastVectorHighlighterOffsetParameter() throws Exception { indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)) .setSource("title", "This is a test on the highlighting bug present in elasticsearch"); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertResponse( prepareSearch().setQuery(matchQuery("title", "bug")) @@ -1370,7 +1388,7 @@ public void testEscapeHtml() throws Exception { indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)) .setSource("title", "This is a html escaping highlighting test for *&? elasticsearch"); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertResponse( prepareSearch().setQuery(matchQuery("title", "test")) @@ -1398,7 +1416,7 @@ public void testEscapeHtmlVector() throws Exception { indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)) .setSource("title", "This is a html escaping highlighting test for *&? elasticsearch"); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertResponse( prepareSearch().setQuery(matchQuery("title", "test")) @@ -1437,7 +1455,7 @@ public void testMultiMapperVectorWithStore() throws Exception { ) ); ensureGreen(); - prepareIndex("test").setId("1").setSource("title", "this is a test").get(); + indexDoc("test", "1", "title", "this is a test"); refresh(); // simple search on body with standard analyzer with a simple field query @@ -1490,7 +1508,7 @@ public void testMultiMapperVectorFromSource() throws Exception { ); ensureGreen(); - prepareIndex("test").setId("1").setSource("title", "this is a test").get(); + indexDoc("test", "1", "title", "this is a test"); refresh(); // simple search on body with standard analyzer with a simple field query @@ -1543,7 +1561,7 @@ public void testMultiMapperNoVectorWithStore() throws Exception { ); ensureGreen(); - prepareIndex("test").setId("1").setSource("title", "this is a test").get(); + indexDoc("test", "1", "title", "this is a test"); refresh(); // simple search on body with standard analyzer with a simple field query @@ -1595,7 +1613,7 @@ public void testMultiMapperNoVectorFromSource() throws Exception { ) ); ensureGreen(); - prepareIndex("test").setId("1").setSource("title", "this is a test").get(); + indexDoc("test", "1", "title", "this is a test"); refresh(); // simple search on body with standard analyzer with a simple field query @@ -1630,7 +1648,7 @@ public void testFastVectorHighlighterShouldFailIfNoTermVectors() throws Exceptio indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)) .setSource("title", "This is a test for the enabling fast vector highlighter"); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertNoFailures( prepareSearch().setQuery(matchPhraseQuery("title", "this is a test")) @@ -1662,7 +1680,7 @@ public void testDisableFastVectorHighlighter() throws Exception { indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)) .setSource("title", "This is a test for the workaround for the fast vector highlighting SOLR-3724"); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertResponse( prepareSearch().setQuery(matchPhraseQuery("title", "test for the workaround")) @@ -1718,14 +1736,14 @@ public void testDisableFastVectorHighlighter() throws Exception { public void testFSHHighlightAllMvFragments() throws Exception { assertAcked(prepareCreate("test").setMapping("tags", "type=text,term_vector=with_positions_offsets")); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - "tags", - new String[] { - "this is a really long tag i would like to highlight", - "here is another one that is very long and has the tag token near the end" } - ) - .get(); + indexDoc( + "test", + "1", + "tags", + new String[] { + "this is a really long tag i would like to highlight", + "here is another one that is very long and has the tag token near the end" } + ); refresh(); assertResponse( @@ -1748,7 +1766,7 @@ public void testFSHHighlightAllMvFragments() throws Exception { public void testBoostingQuery() { createIndex("test"); ensureGreen(); - prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog"); refresh(); logger.info("--> highlighting and searching on field1"); @@ -1768,7 +1786,7 @@ public void testBoostingQuery() { public void testBoostingQueryTermVector() throws IOException { assertAcked(prepareCreate("test").setMapping(type1TermVectorMapping())); ensureGreen(); - prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog"); refresh(); logger.info("--> highlighting and searching on field1"); @@ -1788,17 +1806,17 @@ public void testBoostingQueryTermVector() throws IOException { public void testPlainHighlightDifferentFragmenter() throws Exception { assertAcked(prepareCreate("test").setMapping("tags", "type=text")); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .array( - "tags", - "this is a really long tag i would like to highlight", - "here is another one that is very long tag and has the tag token near the end" - ) - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .array( + "tags", + "this is a really long tag i would like to highlight", + "here is another one that is very long tag and has the tag token near the end" + ) + .endObject() + ); refresh(); assertResponse( @@ -1905,7 +1923,7 @@ public void testFastVectorHighlighterMultipleFields() { public void testMissingStoredField() throws Exception { assertAcked(prepareCreate("test").setMapping("highlight_field", "type=text,store=true")); ensureGreen(); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("field", "highlight").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("field", "highlight").endObject()); refresh(); // This query used to fail when the field to highlight was absent @@ -1942,9 +1960,24 @@ public void testNumericHighlighting() throws Exception { ); ensureGreen(); - prepareIndex("test").setId("1") - .setSource("text", "elasticsearch test", "byte", 25, "short", 42, "int", 100, "long", -1, "float", 3.2f, "double", 42.42) - .get(); + indexDoc( + "test", + "1", + "text", + "elasticsearch test", + "byte", + 25, + "short", + 42, + "int", + 100, + "long", + -1, + "float", + 3.2f, + "double", + 42.42 + ); refresh(); // Highlighting of numeric fields is not supported, but it should not raise errors @@ -1972,7 +2005,7 @@ public void testResetTwice() throws Exception { ).setMapping("text", "type=text,analyzer=my_analyzer") ); ensureGreen(); - prepareIndex("test").setId("1").setSource("text", "elasticsearch test").get(); + indexDoc("test", "1", "text", "elasticsearch test"); refresh(); // Mock tokenizer will throw an exception if it is resetted twice @@ -2340,7 +2373,7 @@ public void testPostingsHighlighter() throws Exception { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy quick dog").get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy quick dog"); refresh(); logger.info("--> highlighting and searching on field1"); @@ -2419,16 +2452,16 @@ public void testPostingsHighlighterNumberOfFragments() throws Exception { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - "field1", - "The quick brown fox jumps over the lazy dog. The lazy red fox jumps over the quick dog. " - + "The quick brown dog jumps over the lazy fox.", - "field2", - "The quick brown fox jumps over the lazy dog. The lazy red fox jumps over the quick dog. " - + "The quick brown dog jumps over the lazy fox." - ) - .get(); + indexDoc( + "test", + "1", + "field1", + "The quick brown fox jumps over the lazy dog. The lazy red fox jumps over the quick dog. " + + "The quick brown dog jumps over the lazy fox.", + "field2", + "The quick brown fox jumps over the lazy dog. The lazy red fox jumps over the quick dog. " + + "The quick brown dog jumps over the lazy fox." + ); refresh(); logger.info("--> highlighting and searching on field1"); @@ -2450,15 +2483,15 @@ public void testPostingsHighlighterNumberOfFragments() throws Exception { ); assertHighlight(response, 0, "field1", 1, 2, equalTo("The quick brown dog jumps over the lazy fox.")); }); - prepareIndex("test").setId("2") - .setSource( - "field1", - new String[] { - "The quick brown fox jumps over the lazy dog. Second sentence not finished", - "The lazy red fox jumps over the quick dog.", - "The quick brown dog jumps over the lazy fox." } - ) - .get(); + indexDoc( + "test", + "2", + "field1", + new String[] { + "The quick brown fox jumps over the lazy dog. Second sentence not finished", + "The lazy red fox jumps over the quick dog.", + "The quick brown dog jumps over the lazy fox." } + ); refresh(); source = searchSource().query(termQuery("field1", "fox")) @@ -2517,7 +2550,7 @@ public void testMultiMatchQueryHighlight() throws Exception { .endObject(); assertAcked(prepareCreate("test").setMapping(mapping)); ensureGreen(); - prepareIndex("test").setSource("field1", "The quick brown fox jumps over", "field2", "The quick brown fox jumps over").get(); + indexDoc("test", null, "field1", "The quick brown fox jumps over", "field2", "The quick brown fox jumps over"); refresh(); final int iters = scaledRandomIntBetween(20, 30); for (int i = 0; i < iters; i++) { @@ -2569,7 +2602,7 @@ public void testCombinedFieldsQueryHighlight() throws Exception { assertAcked(prepareCreate("test").setMapping(mapping)); ensureGreen(); - prepareIndex("test").setSource("field1", "The quick brown fox jumps over", "field2", "The quick brown fox jumps over").get(); + indexDoc("test", null, "field1", "The quick brown fox jumps over", "field2", "The quick brown fox jumps over"); refresh(); for (String highlighterType : ALL_TYPES) { @@ -2597,7 +2630,9 @@ public void testPostingsHighlighterOrderByScore() throws Exception { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource( + indexDoc( + "test", + null, "field1", new String[] { "This sentence contains one match, not that short. This sentence contains two sentence matches. " @@ -2606,7 +2641,7 @@ public void testPostingsHighlighterOrderByScore() throws Exception { + "This sentence contains three sentence occurrences (sentence).", "One sentence match here and scored lower since the text is quite long, not that appealing. " + "This one contains no matches." } - ).get(); + ); refresh(); logger.info("--> highlighting and searching on field1"); @@ -2648,7 +2683,7 @@ public void testPostingsHighlighterEscapeHtml() throws Exception { indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)) .setSource("title", "This is a html escaping highlighting test for *&? elasticsearch"); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertResponse( prepareSearch().setQuery(matchQuery("title", "test")).highlighter(new HighlightBuilder().field("title").encoder("html")), @@ -2693,7 +2728,7 @@ public void testPostingsHighlighterMultiMapperWithStore() throws Exception { ) ); ensureGreen(); - prepareIndex("test").setId("1").setSource("title", "this is a test . Second sentence.").get(); + indexDoc("test", "1", "title", "this is a test . Second sentence."); refresh(); // simple search on body with standard analyzer with a simple field query @@ -2756,7 +2791,7 @@ public void testPostingsHighlighterMultiMapperFromSource() throws Exception { ); ensureGreen(); - prepareIndex("test").setId("1").setSource("title", "this is a test").get(); + indexDoc("test", "1", "title", "this is a test"); refresh(); // simple search on body with standard analyzer with a simple field query @@ -2803,7 +2838,7 @@ public void testPostingsHighlighterShouldFailIfNoOffsets() throws Exception { indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)) .setSource("title", "This is a test for the postings highlighter"); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); assertNoFailures( prepareSearch().setQuery(matchQuery("title", "this is a test")).highlighter(new HighlightBuilder().field("title")) @@ -2813,12 +2848,7 @@ public void testPostingsHighlighterShouldFailIfNoOffsets() throws Exception { public void testPostingsHighlighterBoostingQuery() throws IOException, ExecutionException, InterruptedException { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource( - "field1", - "this is a test", - "field2", - "The quick brown fox jumps over the lazy dog! Second sentence." - ).get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence."); refresh(); logger.info("--> highlighting and searching on field1"); @@ -2857,12 +2887,7 @@ public void testPostingsHighlighterPrefixQuery() throws Exception { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource( - "field1", - "this is a test", - "field2", - "The quick brown fox jumps over the lazy dog! Second sentence." - ).get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence."); refresh(); logger.info("--> highlighting and searching on field2"); @@ -2881,12 +2906,7 @@ public void testPostingsHighlighterFuzzyQuery() throws Exception { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource( - "field1", - "this is a test", - "field2", - "The quick brown fox jumps over the lazy dog! Second sentence." - ).get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence."); refresh(); logger.info("--> highlighting and searching on field2"); @@ -2905,12 +2925,7 @@ public void testPostingsHighlighterRegexpQuery() throws Exception { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource( - "field1", - "this is a test", - "field2", - "The quick brown fox jumps over the lazy dog! Second sentence." - ).get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence."); refresh(); logger.info("--> highlighting and searching on field2"); @@ -2929,12 +2944,7 @@ public void testPostingsHighlighterWildcardQuery() throws Exception { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource( - "field1", - "this is a test", - "field2", - "The quick brown fox jumps over the lazy dog! Second sentence." - ).get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence."); refresh(); logger.info("--> highlighting and searching on field2"); @@ -2960,7 +2970,7 @@ public void testPostingsHighlighterTermRangeQuery() throws Exception { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource("field1", "this is a test", "field2", "aaab").get(); + indexDoc("test", null, "field1", "this is a test", "field2", "aaab"); refresh(); logger.info("--> highlighting and searching on field2"); @@ -2973,12 +2983,7 @@ public void testPostingsHighlighterQueryString() throws Exception { assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource( - "field1", - "this is a test", - "field2", - "The quick brown fox jumps over the lazy dog! Second sentence." - ).get(); + indexDoc("test", null, "field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence."); refresh(); logger.info("--> highlighting and searching on field2"); @@ -2998,7 +3003,7 @@ public void testPostingsHighlighterRegexpQueryWithinConstantScoreQuery() throws assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource("field1", "The photography word will get highlighted").get(); + indexDoc("test", null, "field1", "The photography word will get highlighted"); refresh(); logger.info("--> highlighting and searching on field1"); @@ -3018,7 +3023,7 @@ public void testPostingsHighlighterMultiTermQueryMultipleLevels() throws Excepti assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource("field1", "The photography word will get highlighted").get(); + indexDoc("test", null, "field1", "The photography word will get highlighted"); refresh(); logger.info("--> highlighting and searching on field1"); @@ -3041,7 +3046,7 @@ public void testPostingsHighlighterPrefixQueryWithinBooleanQuery() throws Except assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource("field1", "The photography word will get highlighted").get(); + indexDoc("test", null, "field1", "The photography word will get highlighted"); refresh(); logger.info("--> highlighting and searching on field1"); @@ -3062,7 +3067,7 @@ public void testPostingsHighlighterQueryStringWithinFilteredQuery() throws Excep assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping())); ensureGreen(); - prepareIndex("test").setSource("field1", "The photography word will get highlighted").get(); + indexDoc("test", null, "field1", "The photography word will get highlighted"); refresh(); logger.info("--> highlighting and searching on field1"); @@ -3096,7 +3101,7 @@ public void testPostingsHighlighterManyDocs() throws Exception { .setSource("field1", "Sentence " + prefix + " test. Sentence two."); } logger.info("--> indexing docs"); - indexRandom(true, indexRequestBuilders); + indexRandomAndDecrefRequests(true, indexRequestBuilders); logger.info("--> searching explicitly on field1 and highlighting on it"); SearchRequestBuilder searchRequestBuilder = prepareSearch().setSize(COUNT) @@ -3128,7 +3133,7 @@ public void testDoesNotHighlightTypeName() throws Exception { assertAcked(prepareCreate("test").setMapping(mapping)); ensureGreen(); - indexRandom(true, prepareIndex("test").setSource("foo", "test typename")); + indexRandomAndDecrefRequests(true, prepareIndex("test").setSource("foo", "test typename")); for (String highlighter : ALL_TYPES) { assertHighlight( @@ -3160,7 +3165,7 @@ public void testDoesNotHighlightAliasFilters() throws Exception { assertAcked(indicesAdmin().prepareAliases().addAlias("test", "filtered_alias", matchQuery("foo", "japanese"))); ensureGreen(); - indexRandom(true, prepareIndex("test").setSource("foo", "test japanese")); + indexRandomAndDecrefRequests(true, prepareIndex("test").setSource("foo", "test japanese")); for (String highlighter : ALL_TYPES) { assertHighlight( @@ -3290,9 +3295,11 @@ public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOExcepti mappings.endObject(); assertAcked(prepareCreate("test").setMapping(mappings)); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("text", "Arbitrary text field which will should not cause a failure").endObject()) - .get(); + index( + "test", + "1", + jsonBuilder().startObject().field("text", "Arbitrary text field which will should not cause a failure").endObject() + ); refresh(); String highlighterType = randomFrom(ALL_TYPES); QueryBuilder query = QueryBuilders.boolQuery() @@ -3331,14 +3338,14 @@ public void testGeoFieldHighlightingWhenQueryGetsRewritten() throws IOException assertAcked(prepareCreate("test").setMapping(mappings)); ensureYellow(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("jd", "some आवश्यकता है- आर्य समाज अनाथालय, 68 सिविल लाइन्स, बरेली को एक पुरूष" + " रस text") - .field("loc", "12.934059,77.610741") - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("jd", "some आवश्यकता है- आर्य समाज अनाथालय, 68 सिविल लाइन्स, बरेली को एक पुरूष" + " रस text") + .field("loc", "12.934059,77.610741") + .endObject() + ); refresh(); QueryBuilder query = QueryBuilders.functionScoreQuery( @@ -3370,7 +3377,7 @@ public void testKeywordFieldHighlighting() throws IOException { mappings.endObject(); assertAcked(prepareCreate("test").setMapping(mappings)); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("keyword_field", "some text").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("keyword_field", "some text").endObject()); refresh(); assertNoFailuresAndResponse( prepareSearch().setSource( @@ -3397,10 +3404,7 @@ public void testCopyToFields() throws Exception { b.endObject().endObject(); prepareCreate("test").setMapping(b).get(); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("foo", "how now brown cow").endObject()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + indexImmediate("test", "1", jsonBuilder().startObject().field("foo", "how now brown cow").endObject()); assertResponse( prepareSearch().setQuery(matchQuery("foo_copy", "brown")).highlighter(new HighlightBuilder().field(new Field("foo_copy"))), @@ -3436,21 +3440,20 @@ public void testACopyFieldWithNestedQuery() throws Exception { ); prepareCreate("test").setMapping(mapping).get(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .startArray("foo") - .startObject() - .field("text", "brown") - .endObject() - .startObject() - .field("text", "cow") - .endObject() - .endArray() - .endObject() - ) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + indexImmediate( + "test", + "1", + jsonBuilder().startObject() + .startArray("foo") + .startObject() + .field("text", "brown") + .endObject() + .startObject() + .field("text", "cow") + .endObject() + .endArray() + .endObject() + ); assertResponse( prepareSearch().setQuery(nestedQuery("foo", matchQuery("foo.text", "brown cow"), ScoreMode.None)) @@ -3466,10 +3469,7 @@ public void testACopyFieldWithNestedQuery() throws Exception { } public void testFunctionScoreQueryHighlight() throws Exception { - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("text", "brown").endObject()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + indexImmediate("test", "1", jsonBuilder().startObject().field("text", "brown").endObject()); assertResponse( prepareSearch().setQuery(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro"))) @@ -3484,10 +3484,7 @@ public void testFunctionScoreQueryHighlight() throws Exception { } public void testFiltersFunctionScoreQueryHighlight() throws Exception { - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("text", "brown").field("enable", "yes").endObject()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + indexImmediate("test", "1", jsonBuilder().startObject().field("text", "brown").field("enable", "yes").endObject()); FunctionScoreQueryBuilder.FilterFunctionBuilder filterBuilder = new FunctionScoreQueryBuilder.FilterFunctionBuilder( QueryBuilders.termQuery("enable", "yes"), new RandomScoreFunctionBuilder() @@ -3517,7 +3514,7 @@ public void testHighlightQueryRewriteDatesWithNow() throws Exception { ); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); DateFormatter formatter = DateFormatter.forPattern("strict_date_optional_time"); - indexRandom( + indexRandomAndDecrefRequests( true, prepareIndex("index-1").setId("1").setSource("d", formatter.format(now), "field", "hello world"), prepareIndex("index-1").setId("2").setSource("d", formatter.format(now.minusDays(1)), "field", "hello"), @@ -3564,22 +3561,21 @@ public void testWithNestedQuery() throws Exception { ); prepareCreate("test").setMapping(mapping).get(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .startArray("foo") - .startObject() - .field("text", "brown shoes") - .endObject() - .startObject() - .field("text", "cow") - .endObject() - .endArray() - .field("text", "brown") - .endObject() - ) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + indexImmediate( + "test", + "1", + jsonBuilder().startObject() + .startArray("foo") + .startObject() + .field("text", "brown shoes") + .endObject() + .startObject() + .field("text", "cow") + .endObject() + .endArray() + .field("text", "brown") + .endObject() + ); for (String type : new String[] { "unified", "plain" }) { assertResponse( @@ -3648,7 +3644,7 @@ public void testWithNormalizer() throws Exception { assertAcked(prepareCreate("test").setSettings(builder.build()).setMapping("keyword", "type=keyword,normalizer=my_normalizer")); ensureGreen(); - prepareIndex("test").setId("0").setSource("keyword", "Hello World").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + indexDocImmediate("test", "0", "keyword", "Hello World"); for (String highlighterType : new String[] { "unified", "plain" }) { assertResponse( @@ -3668,10 +3664,7 @@ public void testDisableHighlightIdField() throws Exception { assertAcked(prepareCreate("test").setMapping("keyword", "type=keyword")); ensureGreen(); - prepareIndex("test").setId("d33f85bf1e51e84d9ab38948db9f3a068e1fe5294f1d8603914ac8c7bcc39ca1") - .setSource("keyword", "Hello World") - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + indexDocImmediate("test", "d33f85bf1e51e84d9ab38948db9f3a068e1fe5294f1d8603914ac8c7bcc39ca1", "keyword", "Hello World"); for (String highlighterType : new String[] { "plain", "unified" }) { assertResponse( @@ -3733,4 +3726,32 @@ public Analyzer get() { }); } } + + private void indexRandomAndDecrefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private DocWriteResponse indexDocImmediate(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private DocWriteResponse indexImmediate(String index, String id, XContentBuilder source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java index afc62323ca544..0464b82573909 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; @@ -48,12 +49,16 @@ public void testFailuresFromRemote() { final Client remoteClient = client("remote_cluster"); String localIndex = "local_test"; assertAcked(localClient.admin().indices().prepareCreate(localIndex).setSettings(indexSettings)); - localClient.prepareIndex(localIndex).setId("1").setSource("foo", "bar").get(); + IndexRequestBuilder indexRequestBuilder = localClient.prepareIndex(localIndex).setId("1").setSource("foo", "bar"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); localClient.admin().indices().prepareRefresh(localIndex).get(); String remoteErrorIndex = "remote_test_error"; assertAcked(remoteClient.admin().indices().prepareCreate(remoteErrorIndex).setSettings(indexSettings)); - remoteClient.prepareIndex(remoteErrorIndex).setId("2").setSource("foo", "bar").get(); + indexRequestBuilder = remoteClient.prepareIndex(remoteErrorIndex).setId("2").setSource("foo", "bar"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); remoteClient.admin().indices().prepareRefresh(remoteErrorIndex).get(); // regular field_caps across clusters @@ -87,7 +92,9 @@ public void testFailuresFromRemote() { // add an index that doesn't fail to the remote assertAcked(remoteClient.admin().indices().prepareCreate("okay_remote_index")); - remoteClient.prepareIndex("okay_remote_index").setId("2").setSource("foo", "bar").get(); + indexRequestBuilder = remoteClient.prepareIndex("okay_remote_index").setId("2").setSource("foo", "bar"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); remoteClient.admin().indices().prepareRefresh("okay_remote_index").get(); response = client().prepareFieldCaps("*", "remote_cluster:*") diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 282e29866a699..c331611982279 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -321,6 +321,9 @@ public void testWithIndexFilter() throws InterruptedException { reqs.add(prepareIndex("index-2").setSource("timestamp", "2019-10-12")); reqs.add(prepareIndex("index-2").setSource("timestamp", "2020-07-08")); indexRandom(true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } FieldCapabilitiesResponse response = client().prepareFieldCaps("index-*").setFields("*").get(); assertIndices(response, "index-1", "index-2"); @@ -450,6 +453,9 @@ private void populateTimeRangeIndices() throws Exception { reqs.add(prepareIndex("log-index-2").setSource("timestamp", "2020-02-02")); reqs.add(prepareIndex("log-index-2").setSource("timestamp", "2020-10-10")); indexRandom(true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } ensureGreen("log-index-1", "log-index-2"); indicesAdmin().prepareRefresh("log-index-1", "log-index-2").get(); } @@ -633,7 +639,7 @@ public void testManyIndicesWithSameMapping() { ensureGreen(indices); assertAcked(indicesAdmin().preparePutMapping(indicesWithExtraField).setSource("extra_field", "type=integer").get()); for (String index : indicesWithExtraField) { - prepareIndex(index).setSource("extra_field", randomIntBetween(1, 1000)).get(); + indexDoc(index, null, "extra_field", randomIntBetween(1, 1000)); } FieldCapabilitiesResponse resp = client().execute(TransportFieldCapabilitiesAction.TYPE, request).actionGet(); verifyResponse.accept(resp); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 9ad6363d0e57d..7a05d5d279760 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -181,11 +181,11 @@ public void testStoredFields() throws Exception { indicesAdmin().preparePutMapping().setSource(mapping, XContentType.JSON).get(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject().field("field1", "value1").field("field2", "value2").field("field3", "value3").endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject().field("field1", "value1").field("field2", "value2").field("field3", "value3").endObject() + ); indicesAdmin().prepareRefresh().get(); @@ -274,23 +274,23 @@ public void testScriptDocAndFields() throws Exception { indicesAdmin().preparePutMapping().setSource(mapping, XContentType.JSON).get(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).field("date", "1970-01-01T00:00:00").endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).field("date", "1970-01-01T00:00:00").endObject() + ); indicesAdmin().prepareFlush().get(); - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).field("date", "1970-01-01T00:00:25").endObject() - ) - .get(); + index( + "test", + "2", + jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).field("date", "1970-01-01T00:00:25").endObject() + ); indicesAdmin().prepareFlush().get(); - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).field("date", "1970-01-01T00:02:00").endObject() - ) - .get(); + index( + "test", + "3", + jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).field("date", "1970-01-01T00:02:00").endObject() + ); indicesAdmin().refresh(new RefreshRequest()).actionGet(); logger.info("running doc['num1'].value"); @@ -376,7 +376,7 @@ public void testScriptFieldWithNanos() throws Exception { indicesAdmin().preparePutMapping().setSource(mapping, XContentType.JSON).get(); String date = "2019-01-31T10:00:00.123456789Z"; - indexRandom( + indexRandomAndDecRefRequests( true, false, prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("date", "1970-01-01T00:00:00.000Z").endObject()), @@ -420,7 +420,7 @@ public void testIdBasedScriptFields() throws Exception { indexRequestBuilders[i] = prepareIndex("test").setId(Integer.toString(i)) .setSource(jsonBuilder().startObject().field("num1", i).endObject()); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); assertNoFailuresAndResponse( prepareSearch().setQuery(matchAllQuery()) @@ -442,26 +442,26 @@ public void testIdBasedScriptFields() throws Exception { public void testScriptFieldUsingSource() throws Exception { createIndex("test"); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .startObject("obj1") - .field("test", "something") - .endObject() - .startObject("obj2") - .startArray("arr2") - .value("arr_value1") - .value("arr_value2") - .endArray() - .endObject() - .startArray("arr3") - .startObject() - .field("arr3_field1", "arr3_value1") - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .startObject("obj1") + .field("test", "something") + .endObject() + .startObject("obj2") + .startArray("arr2") + .value("arr_value1") + .value("arr_value2") + .endArray() + .endObject() + .startArray("arr3") + .startObject() + .field("arr3_field1", "arr3_value1") + .endObject() + .endArray() + .endObject() + ); indicesAdmin().refresh(new RefreshRequest()).actionGet(); assertResponse( @@ -505,7 +505,9 @@ public void testScriptFieldUsingSource() throws Exception { } public void testScriptFieldsForNullReturn() throws Exception { - prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy("true").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy("true"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNoFailuresAndResponse( prepareSearch().setQuery(matchAllQuery()) @@ -526,28 +528,28 @@ public void testScriptFieldsForNullReturn() throws Exception { public void testPartialFields() throws Exception { createIndex("test"); - prepareIndex("test").setId("1") - .setSource( - XContentFactory.jsonBuilder() - .startObject() - .field("field1", "value1") - .startObject("obj1") - .startArray("arr1") - .startObject() - .startObject("obj2") - .field("field2", "value21") - .endObject() - .endObject() - .startObject() - .startObject("obj2") - .field("field2", "value22") - .endObject() - .endObject() - .endArray() - .endObject() - .endObject() - ) - .get(); + index( + "test", + "1", + XContentFactory.jsonBuilder() + .startObject() + .field("field1", "value1") + .startObject("obj1") + .startArray("arr1") + .startObject() + .startObject("obj2") + .field("field2", "value21") + .endObject() + .endObject() + .startObject() + .startObject("obj2") + .field("field2", "value22") + .endObject() + .endObject() + .endArray() + .endObject() + .endObject() + ); indicesAdmin().prepareRefresh().get(); } @@ -607,21 +609,21 @@ public void testStoredFieldsWithoutSource() throws Exception { indicesAdmin().preparePutMapping().setSource(mapping, XContentType.JSON).get(); ZonedDateTime date = ZonedDateTime.of(2012, 3, 22, 0, 0, 0, 0, ZoneOffset.UTC); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("byte_field", (byte) 1) - .field("short_field", (short) 2) - .field("integer_field", 3) - .field("long_field", 4L) - .field("float_field", 5.0f) - .field("double_field", 6.0d) - .field("date_field", DateFormatter.forPattern("date_optional_time").format(date)) - .field("boolean_field", true) - .field("binary_field", Base64.getEncoder().encodeToString("testing text".getBytes("UTF-8"))) - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("byte_field", (byte) 1) + .field("short_field", (short) 2) + .field("integer_field", 3) + .field("long_field", 4L) + .field("float_field", 5.0f) + .field("double_field", 6.0d) + .field("date_field", DateFormatter.forPattern("date_optional_time").format(date)) + .field("boolean_field", true) + .field("binary_field", Base64.getEncoder().encodeToString("testing text".getBytes("UTF-8"))) + .endObject() + ); indicesAdmin().prepareRefresh().get(); @@ -673,11 +675,12 @@ public void testStoredFieldsWithoutSource() throws Exception { } public void testSearchFieldsMetadata() throws Exception { - prepareIndex("my-index").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("my-index").setId("1") .setRouting("1") .setSource(jsonBuilder().startObject().field("field1", "value").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertResponse(prepareSearch("my-index").addStoredField("field1").addStoredField("_routing"), response -> { assertThat(response.getHits().getTotalHits().value, equalTo(1L)); @@ -743,7 +746,11 @@ public void testGetFieldsComplexField() throws Exception { .endObject() ); - prepareIndex("my-index").setId("1").setRefreshPolicy(IMMEDIATE).setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("my-index").setId("1") + .setRefreshPolicy(IMMEDIATE) + .setSource(source, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); String field = "field1.field2.field3.field4"; @@ -758,7 +765,7 @@ public void testGetFieldsComplexField() throws Exception { // see #8203 public void testSingleValueFieldDatatField() throws ExecutionException, InterruptedException { assertAcked(indicesAdmin().prepareCreate("test").setMapping("test_field", "type=keyword").get()); - indexRandom(true, prepareIndex("test").setId("1").setSource("test_field", "foobar")); + indexRandomAndDecRefRequests(true, prepareIndex("test").setId("1").setSource("test_field", "foobar")); refresh(); assertResponse( prepareSearch("test").setSource(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).docValueField("test_field")), @@ -827,24 +834,24 @@ public void testDocValueFields() throws Exception { indicesAdmin().preparePutMapping().setSource(mapping, XContentType.JSON).get(); ZonedDateTime date = ZonedDateTime.of(2012, 3, 22, 0, 0, 0, 0, ZoneOffset.UTC); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("text_field", "foo") - .field("keyword_field", "foo") - .field("byte_field", (byte) 1) - .field("short_field", (short) 2) - .field("integer_field", 3) - .field("long_field", 4L) - .field("float_field", 5.0f) - .field("double_field", 6.0d) - .field("date_field", DateFormatter.forPattern("date_optional_time").format(date)) - .field("boolean_field", true) - .field("binary_field", new byte[] { 42, 100 }) - .field("ip_field", "::1") - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("text_field", "foo") + .field("keyword_field", "foo") + .field("byte_field", (byte) 1) + .field("short_field", (short) 2) + .field("integer_field", 3) + .field("long_field", 4L) + .field("float_field", 5.0f) + .field("double_field", 6.0d) + .field("date_field", DateFormatter.forPattern("date_optional_time").format(date)) + .field("boolean_field", true) + .field("binary_field", new byte[] { 42, 100 }) + .field("ip_field", "::1") + .endObject() + ); indicesAdmin().prepareRefresh().get(); @@ -1015,7 +1022,7 @@ public void testScriptFields() throws Exception { ) ); } - indexRandom(true, reqs); + indexRandomAndDecRefRequests(true, reqs); ensureSearchable(); SearchRequestBuilder req = prepareSearch("index"); for (String field : Arrays.asList("s", "ms", "l", "ml", "d", "md")) { @@ -1253,7 +1260,7 @@ public void testWildcardStoredFieldsWithFieldAlias() throws Exception { public void testLoadMetadata() throws Exception { assertAcked(prepareCreate("test")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setRouting("1").setSource(jsonBuilder().startObject().field("field1", "value").endObject()) ); @@ -1268,4 +1275,35 @@ public void testLoadMetadata() throws Exception { assertThat(response.getHits().getAt(0).getDocumentFields().size(), equalTo(0)); }); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, boolean dummyDocuments, IndexRequestBuilder... builders) + throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java index dcbf4996358d7..f2e13bc33175c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java @@ -128,6 +128,9 @@ public void testDistanceScoreGeoLinGaussExp() throws Exception { } indexRandom(true, indexBuilders); + for (IndexRequestBuilder builder : indexBuilders) { + builder.request().decRef(); + } // Test Gauss List lonlat = new ArrayList<>(); @@ -232,6 +235,9 @@ public void testDistanceScoreGeoLinGaussExpWithOffset() throws Exception { } indexRandom(true, indexBuilders); + for (IndexRequestBuilder builder : indexBuilders) { + builder.request().decRef(); + } // Test Gauss @@ -353,6 +359,9 @@ public void testBoostModeSettingWorks() throws Exception { ) ); indexRandom(true, false, indexBuilders); // force no dummy docs + for (IndexRequestBuilder builder : indexBuilders) { + builder.request().decRef(); + } // Test Gauss List lonlat = new ArrayList<>(); @@ -430,7 +439,7 @@ public void testParseGeoPoint() throws Exception { ) ); - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource( jsonBuilder().startObject() .field("test", "value") @@ -440,8 +449,9 @@ public void testParseGeoPoint() throws Exception { .endObject() .endObject() ) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); FunctionScoreQueryBuilder baseQueryBuilder = functionScoreQuery( constantScoreQuery(termQuery("test", "value")), ScoreFunctionBuilders.weightFactorFunction(randomIntBetween(1, 10)) @@ -505,10 +515,11 @@ public void testCombineModes() throws Exception { ) ); - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setRefreshPolicy(IMMEDIATE) - .setSource(jsonBuilder().startObject().field("test", "value value").field("num", 1.0).endObject()) - .get(); + .setSource(jsonBuilder().startObject().field("test", "value value").field("num", 1.0).endObject()); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); FunctionScoreQueryBuilder baseQueryBuilder = functionScoreQuery( constantScoreQuery(termQuery("test", "value")), ScoreFunctionBuilders.weightFactorFunction(2) @@ -647,14 +658,14 @@ public void testExceptionThrownIfScaleLE0() throws Exception { .endObject() ) ); - client().index( - new IndexRequest("test").id("1") - .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-27").endObject()) - ).actionGet(); - client().index( - new IndexRequest("test").id("2") - .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-28").endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-27").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2") + .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-28").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); refresh(); SearchPhaseExecutionException e = expectThrows( @@ -689,19 +700,19 @@ public void testParseDateMath() throws Exception { .endObject() ) ); - client().index( - new IndexRequest("test").id("1") - .source(jsonBuilder().startObject().field("test", "value").field("num1", System.currentTimeMillis()).endObject()) - ).actionGet(); - client().index( - new IndexRequest("test").id("2") - .source( - jsonBuilder().startObject() - .field("test", "value") - .field("num1", System.currentTimeMillis() - (1000 * 60 * 60 * 24)) - .endObject() - ) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("test", "value").field("num1", System.currentTimeMillis()).endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2") + .source( + jsonBuilder().startObject() + .field("test", "value") + .field("num1", System.currentTimeMillis() - (1000 * 60 * 60 * 24)) + .endObject() + ); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); refresh(); assertNoFailuresAndResponse( @@ -744,21 +755,22 @@ public void testValueMissingLin() throws Exception { ) ); - client().index( - new IndexRequest("test").id("1") - .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-27").field("num2", "1.0").endObject()) - ).actionGet(); - client().index( - new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("test", "value").field("num2", "1.0").endObject()) - ).actionGet(); - client().index( - new IndexRequest("test").id("3") - .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-30").field("num2", "1.0").endObject()) - ).actionGet(); - client().index( - new IndexRequest("test").id("4") - .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-30").endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-27").field("num2", "1.0").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2") + .source(jsonBuilder().startObject().field("test", "value").field("num2", "1.0").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("3") + .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-30").field("num2", "1.0").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("4") + .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-30").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); refresh(); @@ -815,30 +827,30 @@ public void testDateWithoutOrigin() throws Exception { + Strings.format("%02d", docDate.getMonthValue()) + "-" + Strings.format("%02d", docDate.getDayOfMonth()); - client().index( - new IndexRequest("test").id("1") - .source(jsonBuilder().startObject().field("test", "value").field("num1", docDateString).endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("test", "value").field("num1", docDateString).endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); docDate = dt.minusDays(2); docDateString = docDate.getYear() + "-" + Strings.format("%02d", docDate.getMonthValue()) + "-" + Strings.format("%02d", docDate.getDayOfMonth()); - client().index( - new IndexRequest("test").id("2") - .source(jsonBuilder().startObject().field("test", "value").field("num1", docDateString).endObject()) - ).actionGet(); + indexRequest = new IndexRequest("test").id("2") + .source(jsonBuilder().startObject().field("test", "value").field("num1", docDateString).endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); docDate = dt.minusDays(3); docDateString = docDate.getYear() + "-" + Strings.format("%02d", docDate.getMonthValue()) + "-" + Strings.format("%02d", docDate.getDayOfMonth()); - client().index( - new IndexRequest("test").id("3") - .source(jsonBuilder().startObject().field("test", "value").field("num1", docDateString).endObject()) - ).actionGet(); + indexRequest = new IndexRequest("test").id("3") + .source(jsonBuilder().startObject().field("test", "value").field("num1", docDateString).endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); refresh(); @@ -918,6 +930,9 @@ public void testManyDocsLin() throws Exception { ); } indexRandom(true, indexBuilders); + for (IndexRequestBuilder builder : indexBuilders) { + builder.request().decRef(); + } List lonlat = new ArrayList<>(); lonlat.add(100f); lonlat.add(110f); @@ -969,17 +984,11 @@ public void testParsingExceptionIfFieldDoesNotExist() throws Exception { ) ); int numDocs = 2; - client().index( - new IndexRequest("test").source( - jsonBuilder().startObject() - .field("test", "value") - .startObject("geo") - .field("lat", 1) - .field("lon", 2) - .endObject() - .endObject() - ) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").source( + jsonBuilder().startObject().field("test", "value").startObject("geo").field("lat", 1).field("lon", 2).endObject().endObject() + ); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); refresh(); List lonlat = new ArrayList<>(); lonlat.add(100f); @@ -1019,11 +1028,11 @@ public void testParsingExceptionIfFieldTypeDoesNotMatch() throws Exception { .endObject() ) ); - client().index( - new IndexRequest("test").source( - jsonBuilder().startObject().field("test", "value").field("num", Integer.toString(1)).endObject() - ) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").source( + jsonBuilder().startObject().field("test", "value").field("num", Integer.toString(1)).endObject() + ); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); refresh(); // so, we indexed a string field, but now we try to score a num field SearchPhaseExecutionException e = expectThrows( @@ -1059,8 +1068,11 @@ public void testNoQueryGiven() throws Exception { .endObject() ) ); - client().index(new IndexRequest("test").source(jsonBuilder().startObject().field("test", "value").field("num", 1.0).endObject())) - .actionGet(); + IndexRequest indexRequest = new IndexRequest("test").source( + jsonBuilder().startObject().field("test", "value").field("num", 1.0).endObject() + ); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); refresh(); // so, we indexed a string field, but now we try to score a num field assertResponse( @@ -1126,6 +1138,8 @@ public void testMultiFieldOptions() throws Exception { ); indexRandom(true, doc1, doc2); + doc1.request().decRef(); + doc2.request().decRef(); assertResponse(client().search(new SearchRequest(new String[] {}).source(searchSource().query(baseQuery))), response -> { assertSearchHits(response, "1", "2"); @@ -1178,6 +1192,8 @@ public void testMultiFieldOptions() throws Exception { doc2 = prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("test", "value").field("num", 1.0).endObject()); indexRandom(true, doc1, doc2); + doc1.request().decRef(); + doc2.request().decRef(); assertResponse( client().search( new SearchRequest(new String[] {}).source( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java index ee60888d7a0a8..3ece1b2837443 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java @@ -121,6 +121,9 @@ public void testExplainScript() throws InterruptedException, IOException, Execut ); } indexRandom(true, true, indexRequests); + for (IndexRequestBuilder builder : indexRequests) { + builder.request().decRef(); + } client().admin().indices().prepareRefresh().get(); ensureYellow(); assertNoFailuresAndResponse( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java index 0a43255967dcd..bdd73531439ef 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java @@ -46,9 +46,9 @@ public void testFieldValueFactor() throws IOException { ) ); - prepareIndex("test").setId("1").setSource("test", 5, "body", "foo").get(); - prepareIndex("test").setId("2").setSource("test", 17, "body", "foo").get(); - prepareIndex("test").setId("3").setSource("body", "bar").get(); + indexDoc("test", "1", "test", 5, "body", "foo"); + indexDoc("test", "2", "test", 17, "body", "foo"); + indexDoc("test", "3", "body", "bar"); refresh(); @@ -123,7 +123,7 @@ public void testFieldValueFactor() throws IOException { response -> assertEquals(response.getHits().getAt(0).getScore(), response.getHits().getAt(2).getScore(), 0) ); - prepareIndex("test").setId("2").setSource("test", -1, "body", "foo").get(); + indexDoc("test", "2", "test", -1, "body", "foo"); refresh(); // -1 divided by 0 is infinity, which should provoke an exception. diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java index bcecc49c2d463..0c1d11ae4dc66 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java @@ -186,6 +186,9 @@ public void testMinScoreFunctionScoreManyDocsAndRandomMinScore() throws IOExcept docs.add(prepareIndex(INDEX).setId(Integer.toString(i)).setSource("num", i + scoreOffset)); } indexRandom(true, docs); + for (IndexRequestBuilder builder : docs) { + builder.request().decRef(); + } Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "return (doc['num'].value)", Collections.emptyMap()); int numMatchingDocs = numDocs + scoreOffset - minScore; if (numMatchingDocs < 0) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java index 396af7e8501cf..ce82a0bd278bd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java @@ -67,14 +67,14 @@ public void testPlugin() throws Exception { .get(); clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().get(); - client().index( - new IndexRequest("test").id("1") - .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-26").endObject()) - ).actionGet(); - client().index( - new IndexRequest("test").id("2") - .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-27").endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-26").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2") + .source(jsonBuilder().startObject().field("test", "value").field("num1", "2013-05-27").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); client().admin().indices().prepareRefresh().get(); DecayFunctionBuilder gfb = new CustomDistanceScoreBuilder("num1", "2013-05-28", "+1d"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index 110ac76849e0b..8ef3d78a4635b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -68,7 +68,7 @@ public void testEnforceWindowSize() { // this int iters = scaledRandomIntBetween(10, 20); for (int i = 0; i < iters; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("f", Integer.toString(i)).get(); + indexDoc("test", Integer.toString(i), "f", Integer.toString(i)); } refresh(); @@ -118,11 +118,9 @@ public void testRescorePhrase() throws Exception { ).setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 1)) ); - prepareIndex("test").setId("1").setSource("field1", "the quick brown fox").get(); - prepareIndex("test").setId("2").setSource("field1", "the quick lazy huge brown fox jumps over the tree ").get(); - prepareIndex("test").setId("3") - .setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree") - .get(); + indexDoc("test", "1", "field1", "the quick brown fox"); + indexDoc("test", "2", "field1", "the quick lazy huge brown fox jumps over the tree "); + indexDoc("test", "3", "field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree"); refresh(); assertResponse( prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR)) @@ -178,21 +176,21 @@ public void testMoreDocs() throws Exception { assertAcked(indicesAdmin().prepareCreate("test").setMapping(mapping).setSettings(builder.put("index.number_of_shards", 1))); - prepareIndex("test").setId("1").setSource("field1", "massachusetts avenue boston massachusetts").get(); - prepareIndex("test").setId("2").setSource("field1", "lexington avenue boston massachusetts").get(); - prepareIndex("test").setId("3").setSource("field1", "boston avenue lexington massachusetts").get(); + indexDoc("test", "1", "field1", "massachusetts avenue boston massachusetts"); + indexDoc("test", "2", "field1", "lexington avenue boston massachusetts"); + indexDoc("test", "3", "field1", "boston avenue lexington massachusetts"); indicesAdmin().prepareRefresh("test").get(); - prepareIndex("test").setId("4").setSource("field1", "boston road lexington massachusetts").get(); - prepareIndex("test").setId("5").setSource("field1", "lexington street lexington massachusetts").get(); - prepareIndex("test").setId("6").setSource("field1", "massachusetts avenue lexington massachusetts").get(); - prepareIndex("test").setId("7").setSource("field1", "bosten street san franciso california").get(); + indexDoc("test", "4", "field1", "boston road lexington massachusetts"); + indexDoc("test", "5", "field1", "lexington street lexington massachusetts"); + indexDoc("test", "6", "field1", "massachusetts avenue lexington massachusetts"); + indexDoc("test", "7", "field1", "bosten street san franciso california"); indicesAdmin().prepareRefresh("test").get(); - prepareIndex("test").setId("8").setSource("field1", "hollywood boulevard los angeles california").get(); - prepareIndex("test").setId("9").setSource("field1", "1st street boston massachussetts").get(); - prepareIndex("test").setId("10").setSource("field1", "1st street boston massachusetts").get(); + indexDoc("test", "8", "field1", "hollywood boulevard los angeles california"); + indexDoc("test", "9", "field1", "1st street boston massachussetts"); + indexDoc("test", "10", "field1", "1st street boston massachusetts"); indicesAdmin().prepareRefresh("test").get(); - prepareIndex("test").setId("11").setSource("field1", "2st street boston massachusetts").get(); - prepareIndex("test").setId("12").setSource("field1", "3st street boston massachusetts").get(); + indexDoc("test", "11", "field1", "2st street boston massachusetts"); + indexDoc("test", "12", "field1", "3st street boston massachusetts"); indicesAdmin().prepareRefresh("test").get(); assertResponse( prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "lexington avenue massachusetts").operator(Operator.OR)) @@ -269,11 +267,11 @@ public void testSmallRescoreWindow() throws Exception { assertAcked(indicesAdmin().prepareCreate("test").setMapping(mapping).setSettings(builder.put("index.number_of_shards", 1))); - prepareIndex("test").setId("3").setSource("field1", "massachusetts").get(); - prepareIndex("test").setId("6").setSource("field1", "massachusetts avenue lexington massachusetts").get(); + indexDoc("test", "3", "field1", "massachusetts"); + indexDoc("test", "6", "field1", "massachusetts avenue lexington massachusetts"); indicesAdmin().prepareRefresh("test").get(); - prepareIndex("test").setId("1").setSource("field1", "lexington massachusetts avenue").get(); - prepareIndex("test").setId("2").setSource("field1", "lexington avenue boston massachusetts road").get(); + indexDoc("test", "1", "field1", "lexington massachusetts avenue"); + indexDoc("test", "2", "field1", "lexington avenue boston massachusetts road"); indicesAdmin().prepareRefresh("test").get(); assertResponse(prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "massachusetts")).setFrom(0).setSize(5), response -> { @@ -349,11 +347,11 @@ public void testRescorerMadeScoresWorse() throws Exception { assertAcked(indicesAdmin().prepareCreate("test").setMapping(mapping).setSettings(builder.put("index.number_of_shards", 1))); - prepareIndex("test").setId("3").setSource("field1", "massachusetts").get(); - prepareIndex("test").setId("6").setSource("field1", "massachusetts avenue lexington massachusetts").get(); + indexDoc("test", "3", "field1", "massachusetts"); + indexDoc("test", "6", "field1", "massachusetts avenue lexington massachusetts"); indicesAdmin().prepareRefresh("test").get(); - prepareIndex("test").setId("1").setSource("field1", "lexington massachusetts avenue").get(); - prepareIndex("test").setId("2").setSource("field1", "lexington avenue boston massachusetts road").get(); + indexDoc("test", "1", "field1", "lexington massachusetts avenue"); + indexDoc("test", "2", "field1", "lexington avenue boston massachusetts road"); indicesAdmin().prepareRefresh("test").get(); assertResponse( @@ -498,11 +496,9 @@ public void testExplain() throws Exception { ) ); ensureGreen(); - prepareIndex("test").setId("1").setSource("field1", "the quick brown fox").get(); - prepareIndex("test").setId("2").setSource("field1", "the quick lazy huge brown fox jumps over the tree").get(); - prepareIndex("test").setId("3") - .setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree") - .get(); + indexDoc("test", "1", "field1", "the quick brown fox"); + indexDoc("test", "2", "field1", "the quick lazy huge brown fox jumps over the tree"); + indexDoc("test", "3", "field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree"); refresh(); { @@ -785,6 +781,9 @@ private int indexRandomNumbers(String analyzer, int shards, boolean dummyDocs) t } indexRandom(true, dummyDocs, docs); + for (IndexRequestBuilder indexRequestBuilder : docs) { + indexRequestBuilder.request().decRef(); + } ensureGreen(); return numDocs; } @@ -793,7 +792,7 @@ private int indexRandomNumbers(String analyzer, int shards, boolean dummyDocs) t public void testFromSize() throws Exception { assertAcked(prepareCreate("test").setSettings(indexSettings(1, 0))); for (int i = 0; i < 5; i++) { - prepareIndex("test").setId("" + i).setSource("text", "hello world").get(); + indexDoc("test", "" + i, "text", "hello world"); } refresh(); @@ -809,7 +808,7 @@ public void testFromSize() throws Exception { public void testRescorePhaseWithInvalidSort() throws Exception { assertAcked(prepareCreate("test")); for (int i = 0; i < 5; i++) { - prepareIndex("test").setId("" + i).setSource("number", 0).get(); + indexDoc("test", "" + i, "number", 0); } refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java index 1fe128da6889c..66bcbad73c75b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java @@ -157,10 +157,8 @@ public void testScoreAccessWithinScript() throws Exception { int docCount = randomIntBetween(100, 200); for (int i = 0; i < docCount; i++) { - prepareIndex("test").setId("" + i) - // we add 1 to the index field to make sure that the scripts below never compute log(0) - .setSource("body", randomFrom(Arrays.asList("foo", "bar", "baz")), "index", i + 1) - .get(); + // we add 1 to the index field to make sure that the scripts below never compute log(0) + indexDoc("test", "" + i, "body", randomFrom(Arrays.asList("foo", "bar", "baz")), "index", i + 1); } refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java index 37c78ec568332..0bc740171708e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java @@ -109,17 +109,17 @@ public void setupTestIndex() throws IOException { } public void testDistanceScript() throws Exception { - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("name", "TestPosition") - .startObject("location") - .field("lat", src_lat) - .field("lon", src_lon) - .endObject() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("name", "TestPosition") + .startObject("location") + .field("lat", src_lat) + .field("lon", src_lon) + .endObject() + .endObject() + ); refresh(); @@ -190,17 +190,17 @@ public void testDistanceScript() throws Exception { } public void testGeoDistanceAggregation() throws IOException { - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("name", "TestPosition") - .startObject("location") - .field("lat", src_lat) - .field("lon", src_lon) - .endObject() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("name", "TestPosition") + .startObject("location") + .field("lat", src_lat) + .field("lon", src_lon) + .endObject() + .endObject() + ); refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPointScriptDocValuesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPointScriptDocValuesIT.java index 3b2d266e77cda..2ef370cd1dd9f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPointScriptDocValuesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPointScriptDocValuesIT.java @@ -9,6 +9,8 @@ package org.elasticsearch.search.geo; import org.apache.lucene.geo.GeoEncodingUtils; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.geo.BoundingBox; import org.elasticsearch.common.geo.GeoPoint; @@ -147,9 +149,11 @@ public void setupTestIndex() throws IOException { public void testRandomPoint() throws Exception { final double lat = GeometryTestUtils.randomLat(); final double lon = GeometryTestUtils.randomLon(); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("name", "TestPosition").field("location", new double[] { lon, lat }).endObject()) - .get(); + index( + "test", + "1", + jsonBuilder().startObject().field("name", "TestPosition").field("location", new double[] { lon, lat }).endObject() + ); client().admin().indices().prepareRefresh("test").get(); assertNoFailuresAndResponse( @@ -193,7 +197,7 @@ public void testRandomMultiPoint() throws Exception { } XContentBuilder builder = jsonBuilder().startObject().field("name", "TestPosition").field("location", values).endObject(); - prepareIndex("test").setId("1").setSource(builder).get(); + index("test", "1", builder); client().admin().indices().prepareRefresh("test").get(); @@ -232,9 +236,7 @@ public void testRandomMultiPoint() throws Exception { } public void testNullPoint() throws Exception { - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("name", "TestPosition").nullField("location").endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().field("name", "TestPosition").nullField("location").endObject()); client().admin().indices().prepareRefresh("test").get(); @@ -259,6 +261,15 @@ private static MultiPointLabelPosition isMultiPointLabelPosition(double[] lats, return new MultiPointLabelPosition(lats, lons); } + protected final DocWriteResponse index(String index, String id, XContentBuilder source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + private static class MultiPointLabelPosition extends BaseMatcher { private final GeoPoint[] points; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java index e929487af9240..835f169c3d699 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.geo; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; @@ -46,7 +47,7 @@ protected void setupSuiteScopeCluster() throws Exception { ); ensureGreen(); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1") .setSource( @@ -180,4 +181,14 @@ public void testFieldAlias() { 4L ); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java index f632bbed134f0..dfadf38adf503 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java @@ -76,10 +76,13 @@ public void testSimpleMoreLikeThis() throws Exception { assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN)); logger.info("Indexing..."); - client().index(new IndexRequest("test").id("1").source(jsonBuilder().startObject().field("text", "lucene").endObject())) - .actionGet(); - client().index(new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject())) - .actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("text", "lucene").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); indicesAdmin().refresh(new RefreshRequest()).actionGet(); logger.info("Running moreLikeThis"); @@ -109,10 +112,13 @@ public void testSimpleMoreLikeThisWithTypes() throws Exception { assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN)); logger.info("Indexing..."); - client().index(new IndexRequest("test").id("1").source(jsonBuilder().startObject().field("text", "lucene").endObject())) - .actionGet(); - client().index(new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject())) - .actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("text", "lucene").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); indicesAdmin().refresh(new RefreshRequest()).actionGet(); logger.info("Running moreLikeThis"); @@ -143,12 +149,14 @@ public void testMoreLikeThisForZeroTokensInOneOfTheAnalyzedFields() throws Excep ensureGreen(); - client().index( - new IndexRequest("test").id("1").source(jsonBuilder().startObject().field("myField", "and_foo").field("empty", "").endObject()) - ).actionGet(); - client().index( - new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("myField", "and_foo").field("empty", "").endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("myField", "and_foo").field("empty", "").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2") + .source(jsonBuilder().startObject().field("myField", "and_foo").field("empty", "").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); indicesAdmin().refresh(new RefreshRequest()).actionGet(); @@ -168,11 +176,16 @@ public void testSimpleMoreLikeOnLongField() throws Exception { assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN)); logger.info("Indexing..."); - client().index(new IndexRequest("test").id("1").source(jsonBuilder().startObject().field("some_long", 1367484649580L).endObject())) - .actionGet(); - client().index(new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("some_long", 0).endObject())).actionGet(); - client().index(new IndexRequest("test").id("3").source(jsonBuilder().startObject().field("some_long", -666).endObject())) - .actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("some_long", 1367484649580L).endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("some_long", 0).endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("3").source(jsonBuilder().startObject().field("some_long", -666).endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); indicesAdmin().refresh(new RefreshRequest()).actionGet(); @@ -208,15 +221,20 @@ public void testMoreLikeThisWithAliases() throws Exception { assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN)); logger.info("Indexing..."); - client().index(new IndexRequest("test").id("1").source(jsonBuilder().startObject().field("text", "lucene beta").endObject())) - .actionGet(); - client().index(new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject())) - .actionGet(); - client().index(new IndexRequest("test").id("3").source(jsonBuilder().startObject().field("text", "elasticsearch beta").endObject())) - .actionGet(); - client().index( - new IndexRequest("test").id("4").source(jsonBuilder().startObject().field("text", "elasticsearch release").endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source(jsonBuilder().startObject().field("text", "lucene beta").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("3").source(jsonBuilder().startObject().field("text", "elasticsearch beta").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("4") + .source(jsonBuilder().startObject().field("text", "elasticsearch release").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); indicesAdmin().refresh(new RefreshRequest()).actionGet(); logger.info("Running moreLikeThis on index"); @@ -268,14 +286,17 @@ public void testMoreLikeThisWithAliasesInLikeDocuments() throws Exception { assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN)); - client().index( - new IndexRequest(indexName).id("1").source(jsonBuilder().startObject().field("text", "elasticsearch index").endObject()) - ).actionGet(); - client().index(new IndexRequest(indexName).id("2").source(jsonBuilder().startObject().field("text", "lucene index").endObject())) - .actionGet(); - client().index( - new IndexRequest(indexName).id("3").source(jsonBuilder().startObject().field("text", "elasticsearch index").endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest(indexName).id("1") + .source(jsonBuilder().startObject().field("text", "elasticsearch " + "index").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest(indexName).id("2").source(jsonBuilder().startObject().field("text", "lucene index").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest(indexName).id("3") + .source(jsonBuilder().startObject().field("text", "elasticsearch index").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); refresh(indexName); assertResponse( @@ -291,9 +312,7 @@ public void testMoreLikeThisWithAliasesInLikeDocuments() throws Exception { public void testMoreLikeThisIssue2197() throws Exception { indicesAdmin().prepareCreate("foo").get(); - prepareIndex("foo").setId("1") - .setSource(jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject().endObject()) - .get(); + index("foo", "1", jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject().endObject()); indicesAdmin().prepareRefresh("foo").get(); assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN)); @@ -306,10 +325,11 @@ public void testMoreLikeWithCustomRouting() throws Exception { indicesAdmin().prepareCreate("foo").get(); ensureGreen(); - prepareIndex("foo").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("foo").setId("1") .setSource(jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject().endObject()) - .setRouting("2") - .get(); + .setRouting("2"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); indicesAdmin().prepareRefresh("foo").get(); assertNoFailures(prepareSearch().setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("2") }))); @@ -320,10 +340,11 @@ public void testMoreLikeThisIssueRoutingNotSerialized() throws Exception { assertAcked(prepareCreate("foo", 2, indexSettings(2, 0))); ensureGreen(); - prepareIndex("foo").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("foo").setId("1") .setSource(jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject().endObject()) - .setRouting("4000") - .get(); + .setRouting("4000"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); indicesAdmin().prepareRefresh("foo").get(); assertNoFailures(prepareSearch().setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("4000") }))); } @@ -346,12 +367,8 @@ public void testNumericField() throws Exception { .endObject() ).get(); ensureGreen(); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("string_value", "lucene index").field("int_value", 1).endObject()) - .get(); - prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("string_value", "elasticsearch index").field("int_value", 42).endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().field("string_value", "lucene index").field("int_value", 1).endObject()); + index("test", "2", jsonBuilder().startObject().field("string_value", "elasticsearch index").field("int_value", 42).endObject()); refresh(); @@ -480,18 +497,18 @@ public void testSimpleMoreLikeInclude() throws Exception { assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN)); logger.info("Indexing..."); - client().index( - new IndexRequest("test").id("1") - .source( - jsonBuilder().startObject() - .field("text", "Apache Lucene is a free/open source information retrieval software library") - .endObject() - ) - ).actionGet(); - client().index( - new IndexRequest("test").id("2") - .source(jsonBuilder().startObject().field("text", "Lucene has been ported to other programming languages").endObject()) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest("test").id("1") + .source( + jsonBuilder().startObject() + .field("text", "Apache Lucene is a free/open source information retrieval software library") + .endObject() + ); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest("test").id("2") + .source(jsonBuilder().startObject().field("text", "Lucene has been ported to other programming languages").endObject()); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); indicesAdmin().refresh(new RefreshRequest()).actionGet(); logger.info("Running More Like This with include true"); @@ -552,7 +569,7 @@ public void testSimpleMoreLikeThisIds() throws Exception { builders.add(prepareIndex("test").setSource("text", "lucene").setId("1")); builders.add(prepareIndex("test").setSource("text", "lucene release").setId("2")); builders.add(prepareIndex("test").setSource("text", "apache lucene").setId("3")); - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); logger.info("Running MoreLikeThis"); Item[] items = new Item[] { new Item(null, "1") }; @@ -580,7 +597,7 @@ public void testMoreLikeThisMultiValueFields() throws Exception { for (int i = 0; i < values.length; i++) { builders.add(prepareIndex("test").setId(String.valueOf(i + 1)).setSource("text", values[i])); } - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); int maxIters = randomIntBetween(10, 20); for (int i = 0; i < maxIters; i++) { @@ -612,7 +629,7 @@ public void testMinimumShouldMatch() throws ExecutionException, InterruptedExcep } builders.add(prepareIndex("test").setId(i + "").setSource("text", text)); } - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); logger.info("Testing each minimum_should_match from 0% - 100% with 10% increment ..."); for (int i = 0; i <= 10; i++) { @@ -645,7 +662,7 @@ public void testMoreLikeThisArtificialDocs() throws Exception { doc.field("field" + i, generateRandomStringArray(5, 10, false) + "a"); // make sure they are not all empty } doc.endObject(); - indexRandom(true, prepareIndex("test").setId("0").setSource(doc)); + indexRandomAndDecRefRequests(true, prepareIndex("test").setId("0").setSource(doc)); logger.info("Checking the document matches ..."); // routing to ensure we hit the shard with the doc @@ -662,7 +679,7 @@ public void testMoreLikeThisMalformedArtificialDocs() throws Exception { ensureGreen("test"); logger.info("Creating an index with a single document ..."); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1") .setSource(jsonBuilder().startObject().field("text", "Hello World!").field("date", "2009-01-01").endObject()) @@ -711,7 +728,7 @@ public void testMoreLikeThisUnlike() throws InterruptedException, IOException { for (int i = 0; i < numFields; i++) { builders.add(prepareIndex("test").setId(i + "").setSource("field" + i, i + "")); } - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); logger.info("First check the document matches all indexed docs."); MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new Item[] { new Item("test", doc) }).minTermFreq(0) @@ -738,7 +755,7 @@ public void testSelectFields() throws IOException, ExecutionException, Interrupt assertAcked(prepareCreate("test").setMapping("text", "type=text,analyzer=whitespace", "text1", "type=text,analyzer=whitespace")); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1") .setSource(jsonBuilder().startObject().field("text", "hello world").field("text1", "elasticsearch").endObject()), @@ -760,9 +777,15 @@ public void testSelectFields() throws IOException, ExecutionException, Interrupt } public void testWithRouting() throws IOException { - prepareIndex("index").setId("1").setRouting("3").setSource("text", "this is a document").get(); - prepareIndex("index").setId("2").setRouting("1").setSource("text", "this is another document").get(); - prepareIndex("index").setId("3").setRouting("4").setSource("text", "this is yet another document").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setRouting("3").setSource("text", "this is a document"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex("index").setId("2").setRouting("1").setSource("text", "this is another document"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex("index").setId("3").setRouting("4").setSource("text", "this is yet another document"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh("index"); Item item = new Item("index", "2").routing("1"); @@ -830,4 +853,24 @@ public void testWithMissingRouting() throws IOException { assertThat(cause.getMessage(), equalTo("routing is required for [test]/[2]")); } } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java index 7072594eab8ec..5c6e345753d38 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse.Item; import org.elasticsearch.common.settings.Settings; @@ -40,8 +41,8 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { public void testSimpleMultiSearch() { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("field", "xxx").get(); - prepareIndex("test").setId("2").setSource("field", "yyy").get(); + indexDoc("test", "1", "field", "xxx"); + indexDoc("test", "2", "field", "yyy"); refresh(); assertResponse( client().prepareMultiSearch() @@ -66,7 +67,9 @@ public void testSimpleMultiSearchMoreRequests() throws Exception { createIndex("test"); int numDocs = randomIntBetween(0, 16); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } refresh(); @@ -95,8 +98,8 @@ public void testCCSCheckCompatibility() throws Exception { TransportVersion transportVersion = TransportVersionUtils.getNextVersion(TransportVersions.MINIMUM_CCS_VERSION, true); createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("field", "xxx").get(); - prepareIndex("test").setId("2").setSource("field", "yyy").get(); + indexDoc("test", "1", "field", "xxx"); + indexDoc("test", "2", "field", "yyy"); refresh(); assertResponse( client().prepareMultiSearch() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java index 245fb1651f4d6..969530e7eb18e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.nested; import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -95,7 +96,11 @@ public void testNestedWithMinScore() throws Exception { doc.endArray(); doc.endObject(); - prepareIndex("test").setId("d1").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).setSource(doc).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("d1") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setSource(doc); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); final BoolQueryBuilder childQuery = new BoolQueryBuilder().filter( new MatchPhraseQueryBuilder("toolTracks.data", "cash dispenser, automated teller machine, automatic teller machine") ).filter(new RangeQueryBuilder("toolTracks.confidence").from(0.8)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java index 29a3e589e7923..b81a4ec2af39f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -56,23 +57,23 @@ public void testSimpleNested() throws Exception { assertHitCount(prepareSearch("test"), 0L); assertHitCount(prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")), 0L); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("field1", "value1") - .startArray("nested1") - .startObject() - .field("n_field1", "n_value1_1") - .field("n_field2", "n_value2_1") - .endObject() - .startObject() - .field("n_field1", "n_value1_2") - .field("n_field2", "n_value2_2") - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("field1", "value1") + .startArray("nested1") + .startObject() + .field("n_field1", "n_value1_1") + .field("n_field2", "n_value2_1") + .endObject() + .startObject() + .field("n_field1", "n_value1_2") + .field("n_field2", "n_value2_2") + .endObject() + .endArray() + .endObject() + ); waitForRelocation(ClusterHealthStatus.GREEN); GetResponse getResponse = client().prepareGet("test", "1").get(); @@ -101,23 +102,23 @@ public void testSimpleNested() throws Exception { // add another doc, one that would match if it was not nested... - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .field("field1", "value1") - .startArray("nested1") - .startObject() - .field("n_field1", "n_value1_1") - .field("n_field2", "n_value2_2") - .endObject() - .startObject() - .field("n_field1", "n_value1_2") - .field("n_field2", "n_value2_1") - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "2", + jsonBuilder().startObject() + .field("field1", "value1") + .startArray("nested1") + .startObject() + .field("n_field1", "n_value1_1") + .field("n_field2", "n_value2_2") + .endObject() + .startObject() + .field("n_field1", "n_value1_2") + .field("n_field2", "n_value2_1") + .endObject() + .endArray() + .endObject() + ); waitForRelocation(ClusterHealthStatus.GREEN); refresh(); assertDocumentCount("test", 6); @@ -195,37 +196,37 @@ public void testMultiNested() throws Exception { ); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("field", "value") - .startArray("nested1") - .startObject() - .field("field1", "1") - .startArray("nested2") - .startObject() - .field("field2", "2") - .endObject() - .startObject() - .field("field2", "3") - .endObject() - .endArray() - .endObject() - .startObject() - .field("field1", "4") - .startArray("nested2") - .startObject() - .field("field2", "5") - .endObject() - .startObject() - .field("field2", "6") - .endObject() - .endArray() - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("field", "value") + .startArray("nested1") + .startObject() + .field("field1", "1") + .startArray("nested2") + .startObject() + .field("field2", "2") + .endObject() + .startObject() + .field("field2", "3") + .endObject() + .endArray() + .endObject() + .startObject() + .field("field1", "4") + .startArray("nested2") + .startObject() + .field("field2", "5") + .endObject() + .startObject() + .field("field2", "6") + .endObject() + .endArray() + .endObject() + .endArray() + .endObject() + ); GetResponse getResponse = client().prepareGet("test", "1").get(); assertThat(getResponse.isExists(), equalTo(true)); @@ -344,41 +345,41 @@ public void testDeleteNestedDocsWithAlias() throws Exception { ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("field1", "value1") - .startArray("nested1") - .startObject() - .field("n_field1", "n_value1_1") - .field("n_field2", "n_value2_1") - .endObject() - .startObject() - .field("n_field1", "n_value1_2") - .field("n_field2", "n_value2_2") - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("field1", "value1") + .startArray("nested1") + .startObject() + .field("n_field1", "n_value1_1") + .field("n_field2", "n_value2_1") + .endObject() + .startObject() + .field("n_field1", "n_value1_2") + .field("n_field2", "n_value2_2") + .endObject() + .endArray() + .endObject() + ); - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .field("field1", "value2") - .startArray("nested1") - .startObject() - .field("n_field1", "n_value1_1") - .field("n_field2", "n_value2_1") - .endObject() - .startObject() - .field("n_field1", "n_value1_2") - .field("n_field2", "n_value2_2") - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "2", + jsonBuilder().startObject() + .field("field1", "value2") + .startArray("nested1") + .startObject() + .field("n_field1", "n_value1_1") + .field("n_field2", "n_value2_1") + .endObject() + .startObject() + .field("n_field1", "n_value1_2") + .field("n_field2", "n_value2_2") + .endObject() + .endArray() + .endObject() + ); flush(); refresh(); @@ -402,7 +403,7 @@ public void testExplain() throws Exception { ensureGreen(); - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource( jsonBuilder().startObject() .field("field1", "value1") @@ -416,8 +417,9 @@ public void testExplain() throws Exception { .endArray() .endObject() ) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNoFailuresAndResponse( prepareSearch("test").setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1"), ScoreMode.Total)) @@ -455,51 +457,51 @@ public void testSimpleNestedSorting() throws Exception { ); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("field1", 1) - .startArray("nested1") - .startObject() - .field("field1", 5) - .endObject() - .startObject() - .field("field1", 4) - .endObject() - .endArray() - .endObject() - ) - .get(); - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .field("field1", 2) - .startArray("nested1") - .startObject() - .field("field1", 1) - .endObject() - .startObject() - .field("field1", 2) - .endObject() - .endArray() - .endObject() - ) - .get(); - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .field("field1", 3) - .startArray("nested1") - .startObject() - .field("field1", 3) - .endObject() - .startObject() - .field("field1", 4) - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("field1", 1) + .startArray("nested1") + .startObject() + .field("field1", 5) + .endObject() + .startObject() + .field("field1", 4) + .endObject() + .endArray() + .endObject() + ); + index( + "test", + "2", + jsonBuilder().startObject() + .field("field1", 2) + .startArray("nested1") + .startObject() + .field("field1", 1) + .endObject() + .startObject() + .field("field1", 2) + .endObject() + .endArray() + .endObject() + ); + index( + "test", + "3", + jsonBuilder().startObject() + .field("field1", 3) + .startArray("nested1") + .startObject() + .field("field1", 3) + .endObject() + .startObject() + .field("field1", 4) + .endObject() + .endArray() + .endObject() + ); refresh(); assertResponse( @@ -555,59 +557,59 @@ public void testSimpleNestedSortingWithNestedFilterMissing() throws Exception { ); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("field1", 1) - .startArray("nested1") - .startObject() - .field("field1", 5) - .field("field2", true) - .endObject() - .startObject() - .field("field1", 4) - .field("field2", true) - .endObject() - .endArray() - .endObject() - ) - .get(); - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .field("field1", 2) - .startArray("nested1") - .startObject() - .field("field1", 1) - .field("field2", true) - .endObject() - .startObject() - .field("field1", 2) - .field("field2", true) - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("field1", 1) + .startArray("nested1") + .startObject() + .field("field1", 5) + .field("field2", true) + .endObject() + .startObject() + .field("field1", 4) + .field("field2", true) + .endObject() + .endArray() + .endObject() + ); + index( + "test", + "2", + jsonBuilder().startObject() + .field("field1", 2) + .startArray("nested1") + .startObject() + .field("field1", 1) + .field("field2", true) + .endObject() + .startObject() + .field("field1", 2) + .field("field2", true) + .endObject() + .endArray() + .endObject() + ); // Doc with missing nested docs if nested filter is used refresh(); - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .field("field1", 3) - .startArray("nested1") - .startObject() - .field("field1", 3) - .field("field2", false) - .endObject() - .startObject() - .field("field1", 4) - .field("field2", false) - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "3", + jsonBuilder().startObject() + .field("field1", 3) + .startArray("nested1") + .startObject() + .field("field1", 3) + .field("field2", false) + .endObject() + .startObject() + .field("field1", 4) + .field("field2", false) + .endObject() + .endArray() + .endObject() + ); refresh(); SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()) @@ -682,7 +684,7 @@ public void testNestedSortWithMultiLevelFiltering() throws Exception { }""")); ensureGreen(); - prepareIndex("test").setId("1").setSource(""" + index("test", "1", """ { "acl": [ { @@ -730,9 +732,9 @@ public void testNestedSortWithMultiLevelFiltering() throws Exception { ] } ] - }""", XContentType.JSON).get(); + }""", XContentType.JSON); - prepareIndex("test").setId("2").setSource(""" + index("test", "2", """ { "acl": [ { @@ -777,7 +779,7 @@ public void testNestedSortWithMultiLevelFiltering() throws Exception { ] } ] - }""", XContentType.JSON).get(); + }""", XContentType.JSON); refresh(); // access id = 1, read, max value, asc, should use matt and shay @@ -912,7 +914,7 @@ public void testLeakingSortValues() throws Exception { """)); ensureGreen(); - prepareIndex("test").setId("1").setSource(""" + index("test", "1", """ { "nested1": [ { @@ -924,9 +926,9 @@ public void testLeakingSortValues() throws Exception { ] } ] - }""", XContentType.JSON).get(); + }""", XContentType.JSON); - prepareIndex("test").setId("2").setSource(""" + index("test", "2", """ { "nested1": [ { @@ -938,7 +940,7 @@ public void testLeakingSortValues() throws Exception { ] } ] - }""", XContentType.JSON).get(); + }""", XContentType.JSON); refresh(); @@ -997,130 +999,130 @@ public void testSortNestedWithNestedFilter() throws Exception { ensureGreen(); // sum: 11 - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("grand_parent_values", 1L) - .startArray("parent") - .startObject() - .field("filter", false) - .field("parent_values", 1L) - .startArray("child") - .startObject() - .field("filter", true) - .field("child_values", 1L) - .startObject("child_obj") - .field("value", 1L) - .endObject() - .endObject() - .startObject() - .field("filter", false) - .field("child_values", 6L) - .endObject() - .endArray() - .endObject() - .startObject() - .field("filter", true) - .field("parent_values", 2L) - .startArray("child") - .startObject() - .field("filter", false) - .field("child_values", -1L) - .endObject() - .startObject() - .field("filter", false) - .field("child_values", 5L) - .endObject() - .endArray() - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("grand_parent_values", 1L) + .startArray("parent") + .startObject() + .field("filter", false) + .field("parent_values", 1L) + .startArray("child") + .startObject() + .field("filter", true) + .field("child_values", 1L) + .startObject("child_obj") + .field("value", 1L) + .endObject() + .endObject() + .startObject() + .field("filter", false) + .field("child_values", 6L) + .endObject() + .endArray() + .endObject() + .startObject() + .field("filter", true) + .field("parent_values", 2L) + .startArray("child") + .startObject() + .field("filter", false) + .field("child_values", -1L) + .endObject() + .startObject() + .field("filter", false) + .field("child_values", 5L) + .endObject() + .endArray() + .endObject() + .endArray() + .endObject() + ); // sum: 7 - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .field("grand_parent_values", 2L) - .startArray("parent") - .startObject() - .field("filter", false) - .field("parent_values", 2L) - .startArray("child") - .startObject() - .field("filter", true) - .field("child_values", 2L) - .startObject("child_obj") - .field("value", 2L) - .endObject() - .endObject() - .startObject() - .field("filter", false) - .field("child_values", 4L) - .endObject() - .endArray() - .endObject() - .startObject() - .field("parent_values", 3L) - .field("filter", true) - .startArray("child") - .startObject() - .field("child_values", -2L) - .field("filter", false) - .endObject() - .startObject() - .field("filter", false) - .field("child_values", 3L) - .endObject() - .endArray() - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "2", + jsonBuilder().startObject() + .field("grand_parent_values", 2L) + .startArray("parent") + .startObject() + .field("filter", false) + .field("parent_values", 2L) + .startArray("child") + .startObject() + .field("filter", true) + .field("child_values", 2L) + .startObject("child_obj") + .field("value", 2L) + .endObject() + .endObject() + .startObject() + .field("filter", false) + .field("child_values", 4L) + .endObject() + .endArray() + .endObject() + .startObject() + .field("parent_values", 3L) + .field("filter", true) + .startArray("child") + .startObject() + .field("child_values", -2L) + .field("filter", false) + .endObject() + .startObject() + .field("filter", false) + .field("child_values", 3L) + .endObject() + .endArray() + .endObject() + .endArray() + .endObject() + ); // sum: 2 - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .field("grand_parent_values", 3L) - .startArray("parent") - .startObject() - .field("parent_values", 3L) - .field("filter", false) - .startArray("child") - .startObject() - .field("filter", true) - .field("child_values", 3L) - .startObject("child_obj") - .field("value", 3L) - .endObject() - .endObject() - .startObject() - .field("filter", false) - .field("child_values", 1L) - .endObject() - .endArray() - .endObject() - .startObject() - .field("parent_values", 4L) - .field("filter", true) - .startArray("child") - .startObject() - .field("filter", false) - .field("child_values", -3L) - .endObject() - .startObject() - .field("filter", false) - .field("child_values", 1L) - .endObject() - .endArray() - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "3", + jsonBuilder().startObject() + .field("grand_parent_values", 3L) + .startArray("parent") + .startObject() + .field("parent_values", 3L) + .field("filter", false) + .startArray("child") + .startObject() + .field("filter", true) + .field("child_values", 3L) + .startObject("child_obj") + .field("value", 3L) + .endObject() + .endObject() + .startObject() + .field("filter", false) + .field("child_values", 1L) + .endObject() + .endArray() + .endObject() + .startObject() + .field("parent_values", 4L) + .field("filter", true) + .startArray("child") + .startObject() + .field("filter", false) + .field("child_values", -3L) + .endObject() + .startObject() + .field("filter", false) + .field("child_values", 1L) + .endObject() + .endArray() + .endObject() + .endArray() + .endObject() + ); refresh(); // Without nested filter @@ -1392,110 +1394,110 @@ public void testNestedSortingWithNestedFilterAsFilter() throws Exception { ) ); - DocWriteResponse indexResponse1 = prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("officelocation", "gendale") - .startArray("users") - .startObject() - .field("first", "fname1") - .field("last", "lname1") - .startArray("workstations") - .startObject() - .field("stationid", "s1") - .field("phoneid", "p1") - .endObject() - .startObject() - .field("stationid", "s2") - .field("phoneid", "p2") - .endObject() - .endArray() - .endObject() - .startObject() - .field("first", "fname2") - .field("last", "lname2") - .startArray("workstations") - .startObject() - .field("stationid", "s3") - .field("phoneid", "p3") - .endObject() - .startObject() - .field("stationid", "s4") - .field("phoneid", "p4") - .endObject() - .endArray() - .endObject() - .startObject() - .field("first", "fname3") - .field("last", "lname3") - .startArray("workstations") - .startObject() - .field("stationid", "s5") - .field("phoneid", "p5") - .endObject() - .startObject() - .field("stationid", "s6") - .field("phoneid", "p6") - .endObject() - .endArray() - .endObject() - .endArray() - .endObject() - ) - .get(); + DocWriteResponse indexResponse1 = index( + "test", + "1", + jsonBuilder().startObject() + .field("officelocation", "gendale") + .startArray("users") + .startObject() + .field("first", "fname1") + .field("last", "lname1") + .startArray("workstations") + .startObject() + .field("stationid", "s1") + .field("phoneid", "p1") + .endObject() + .startObject() + .field("stationid", "s2") + .field("phoneid", "p2") + .endObject() + .endArray() + .endObject() + .startObject() + .field("first", "fname2") + .field("last", "lname2") + .startArray("workstations") + .startObject() + .field("stationid", "s3") + .field("phoneid", "p3") + .endObject() + .startObject() + .field("stationid", "s4") + .field("phoneid", "p4") + .endObject() + .endArray() + .endObject() + .startObject() + .field("first", "fname3") + .field("last", "lname3") + .startArray("workstations") + .startObject() + .field("stationid", "s5") + .field("phoneid", "p5") + .endObject() + .startObject() + .field("stationid", "s6") + .field("phoneid", "p6") + .endObject() + .endArray() + .endObject() + .endArray() + .endObject() + ); assertTrue(indexResponse1.getShardInfo().getSuccessful() > 0); - DocWriteResponse indexResponse2 = prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .field("officelocation", "gendale") - .startArray("users") - .startObject() - .field("first", "fname4") - .field("last", "lname4") - .startArray("workstations") - .startObject() - .field("stationid", "s1") - .field("phoneid", "p1") - .endObject() - .startObject() - .field("stationid", "s2") - .field("phoneid", "p2") - .endObject() - .endArray() - .endObject() - .startObject() - .field("first", "fname5") - .field("last", "lname5") - .startArray("workstations") - .startObject() - .field("stationid", "s3") - .field("phoneid", "p3") - .endObject() - .startObject() - .field("stationid", "s4") - .field("phoneid", "p4") - .endObject() - .endArray() - .endObject() - .startObject() - .field("first", "fname1") - .field("last", "lname1") - .startArray("workstations") - .startObject() - .field("stationid", "s5") - .field("phoneid", "p5") - .endObject() - .startObject() - .field("stationid", "s6") - .field("phoneid", "p6") - .endObject() - .endArray() - .endObject() - .endArray() - .endObject() - ) - .get(); + DocWriteResponse indexResponse2 = index( + "test", + "2", + jsonBuilder().startObject() + .field("officelocation", "gendale") + .startArray("users") + .startObject() + .field("first", "fname4") + .field("last", "lname4") + .startArray("workstations") + .startObject() + .field("stationid", "s1") + .field("phoneid", "p1") + .endObject() + .startObject() + .field("stationid", "s2") + .field("phoneid", "p2") + .endObject() + .endArray() + .endObject() + .startObject() + .field("first", "fname5") + .field("last", "lname5") + .startArray("workstations") + .startObject() + .field("stationid", "s3") + .field("phoneid", "p3") + .endObject() + .startObject() + .field("stationid", "s4") + .field("phoneid", "p4") + .endObject() + .endArray() + .endObject() + .startObject() + .field("first", "fname1") + .field("last", "lname1") + .startArray("workstations") + .startObject() + .field("stationid", "s5") + .field("phoneid", "p5") + .endObject() + .startObject() + .field("stationid", "s6") + .field("phoneid", "p6") + .endObject() + .endArray() + .endObject() + .endArray() + .endObject() + ); assertTrue(indexResponse2.getShardInfo().getSuccessful() > 0); refresh(); @@ -1533,8 +1535,8 @@ public void testCheckFixedBitSetCache() throws Exception { } assertAcked(prepareCreate("test").setSettings(settingsBuilder)); - prepareIndex("test").setId("0").setSource("field", "value").get(); - prepareIndex("test").setId("1").setSource("field", "value").get(); + indexDoc("test", "0", "field", "value"); + indexDoc("test", "1", "field", "value"); refresh(); ensureSearchable("test"); @@ -1553,11 +1555,11 @@ public void testCheckFixedBitSetCache() throws Exception { .endArray() .endObject(); // index simple data - prepareIndex("test").setId("2").setSource(builder).get(); - prepareIndex("test").setId("3").setSource(builder).get(); - prepareIndex("test").setId("4").setSource(builder).get(); - prepareIndex("test").setId("5").setSource(builder).get(); - prepareIndex("test").setId("6").setSource(builder).get(); + index("test", "2", builder); + index("test", "3", builder); + index("test", "4", builder); + index("test", "5", builder); + index("test", "6", builder); refresh(); ensureSearchable("test"); @@ -1585,4 +1587,13 @@ private void assertDocumentCount(String index, long numdocs) { assertThat(stats.getIndex(index).getPrimaries().docs.getCount(), is(numdocs)); } + private void index(String index, String id, String source, XContentType contentType) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source, contentType).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java index f830ca9ac0cb6..27286484b15f2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java @@ -48,17 +48,17 @@ public void testSimpleNested() throws Exception { ); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .startArray("nested") - .startObject() - .field("vector", new float[] { 1, 1, 1 }) - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .startArray("nested") + .startObject() + .field("vector", new float[] { 1, 1, 1 }) + .endObject() + .endArray() + .endObject() + ); waitForRelocation(ClusterHealthStatus.GREEN); GetResponse getResponse = client().prepareGet("test", "1").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java index 0acf9be574ffe..ea91bc810c7f4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java @@ -116,6 +116,9 @@ protected void setupSuiteScopeCluster() throws Exception { } indexRandom(true, false, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } createIndex("idx_unmapped"); } @@ -645,6 +648,9 @@ public void testFilterByFilter() throws InterruptedException, IOException { builders.add(prepareIndex("dateidx").setSource(jsonBuilder().startObject().field("date", date).endObject())); } indexRandom(true, false, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("dateidx").setProfile(true) @@ -724,6 +730,9 @@ public void testDateHistogramFilterByFilterDisabled() throws InterruptedExceptio ); } indexRandom(true, false, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertNoFailuresAndResponse( prepareSearch("date_filter_by_filter_disabled").setProfile(true) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java index 65393f4185ce8..49b7e72db1454 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java @@ -62,6 +62,9 @@ public void testProfileDfs() throws Exception { ); } indexRandom(true, docs); + for (IndexRequestBuilder builder : docs) { + builder.request().decRef(); + } refresh(); int iters = between(5, 10); for (int i = 0; i < iters; i++) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java index e02bed8409bc4..6b8c2a12d3379 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java @@ -56,7 +56,7 @@ public void testProfileQuery() throws Exception { List stringFields = Arrays.asList("field1"); List numericFields = Arrays.asList("field2"); - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); refresh(); int iters = between(20, 100); @@ -105,7 +105,7 @@ public void testProfileMatchesRegular() throws Exception { List stringFields = Arrays.asList("field1"); List numericFields = Arrays.asList("field2"); - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); refresh(); QueryBuilder q = randomQueryBuilder(stringFields, numericFields, numDocs, 3); @@ -182,7 +182,7 @@ public void testSimpleMatch() throws Exception { docs[i] = prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); ensureGreen(); QueryBuilder q = QueryBuilders.matchQuery("field1", "one"); @@ -222,7 +222,7 @@ public void testBool() throws Exception { docs[i] = prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); QueryBuilder q = QueryBuilders.boolQuery() .must(QueryBuilders.matchQuery("field1", "one")) @@ -281,7 +281,7 @@ public void testEmptyBool() throws Exception { docs[i] = prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); refresh(); @@ -324,7 +324,7 @@ public void testCollapsingBool() throws Exception { docs[i] = prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); refresh(); @@ -364,7 +364,7 @@ public void testBoosting() throws Exception { docs[i] = prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); refresh(); @@ -404,7 +404,7 @@ public void testDisMaxRange() throws Exception { docs[i] = prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); refresh(); @@ -444,7 +444,7 @@ public void testRange() throws Exception { docs[i] = prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); refresh(); @@ -484,7 +484,7 @@ public void testPhrase() throws Exception { .setSource("field1", English.intToEnglish(i) + " " + English.intToEnglish(i + 1), "field2", i); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); refresh(); @@ -536,7 +536,7 @@ public void testNoProfile() throws Exception { docs[i] = prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i); } - indexRandom(true, docs); + indexRandomAndDecRefRequests(true, docs); refresh(); QueryBuilder q = QueryBuilders.rangeQuery("field2").from(0).to(5); @@ -547,4 +547,14 @@ public void testNoProfile() throws Exception { response -> assertThat("Profile response element should be an empty map", response.getProfileResults().size(), equalTo(0)) ); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java index 81c612107e44a..0957eacf4a974 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java @@ -101,6 +101,9 @@ public void testExists() throws Exception { // would have _field_names indexed while the current type might not which might // confuse the exists/missing parser at query time indexRandom(true, false, reqs); + for (IndexRequestBuilder builder : reqs) { + builder.request().decRef(); + } final Map expected = new LinkedHashMap<>(); expected.put("foo", 1); @@ -189,6 +192,9 @@ public void testFieldAlias() throws Exception { indexRequests.add(prepareIndex("idx").setSource("foo", singletonMap("bar", 2.718))); indexRequests.add(prepareIndex("idx").setSource("foo", singletonMap("bar", 6.283))); indexRandom(true, false, indexRequests); + for (IndexRequestBuilder builder : indexRequests) { + builder.request().decRef(); + } Map expected = new LinkedHashMap<>(); expected.put("foo.bar", 2); @@ -228,6 +234,9 @@ public void testFieldAliasWithNoDocValues() throws Exception { indexRequests.add(prepareIndex("idx").setSource("foo", 3)); indexRequests.add(prepareIndex("idx").setSource("foo", 43)); indexRandom(true, false, indexRequests); + for (IndexRequestBuilder builder : indexRequests) { + builder.request().decRef(); + } assertHitCountAndNoFailures(prepareSearch("idx").setQuery(QueryBuilders.existsQuery("foo-alias")), 2L); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/IntervalQueriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/IntervalQueriesIT.java index 50a1924843e74..57ec3645eb0fb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/IntervalQueriesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/IntervalQueriesIT.java @@ -12,6 +12,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.query.IntervalQueryBuilder; @@ -25,6 +26,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Map; import static java.util.Collections.singletonMap; @@ -49,12 +51,15 @@ public void testEmptyIntervalsWithNestedMappings() throws InterruptedException { }} """)); - indexRandom( - true, + List builders = List.of( prepareIndex("nested").setId("1").setSource("text", "the quick brown fox jumps"), prepareIndex("nested").setId("2").setSource("text", "quick brown"), prepareIndex("nested").setId("3").setSource("text", "quick") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertNoFailures( prepareSearch("nested").setQuery( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java index 2d6bb8176b091..947681c8bae0d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java @@ -224,6 +224,9 @@ public void init() throws Exception { ); } indexRandom(true, false, builders); + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } } private XContentBuilder createMapping() throws IOException { @@ -920,6 +923,9 @@ public void testFuzzyFieldLevelBoosting() throws InterruptedException, Execution builders.add(prepareIndex(idx).setId("1").setSource("title", "foo", "body", "bar")); builders.add(prepareIndex(idx).setId("2").setSource("title", "bar", "body", "foo")); indexRandom(true, false, builders); + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } assertResponse( prepareSearch(idx).setExplain(true) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java index a7ce84f3cd02d..510cad5c19059 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java @@ -47,7 +47,7 @@ public void testBasicAllQuery() throws Exception { reqs.add(prepareIndex("test").setId("1").setSource("f1", "foo bar baz")); reqs.add(prepareIndex("test").setId("2").setSource("f2", "Bar")); reqs.add(prepareIndex("test").setId("3").setSource("f3", "foo bar baz")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(queryStringQuery("foo")), response -> { assertHitCount(response, 2L); @@ -67,7 +67,7 @@ public void testWithDate() throws Exception { List reqs = new ArrayList<>(); reqs.add(prepareIndex("test").setId("1").setSource("f1", "foo", "f_date", "2015/09/02")); reqs.add(prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(queryStringQuery("foo bar")), response -> { assertHits(response.getHits(), "1", "2"); @@ -91,7 +91,7 @@ public void testWithLotsOfTypes() throws Exception { List reqs = new ArrayList<>(); reqs.add(prepareIndex("test").setId("1").setSource("f1", "foo", "f_date", "2015/09/02", "f_float", "1.7", "f_ip", "127.0.0.1")); reqs.add(prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01", "f_float", "1.8", "f_ip", "127.0.0.2")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(queryStringQuery("foo bar")), response -> { assertHits(response.getHits(), "1", "2"); @@ -115,7 +115,7 @@ public void testDocWithAllTypes() throws Exception { List reqs = new ArrayList<>(); String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json"); reqs.add(prepareIndex("test").setId("1").setSource(docBody, XContentType.JSON)); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(queryStringQuery("foo")), response -> assertHits(response.getHits(), "1")); assertResponse(prepareSearch("test").setQuery(queryStringQuery("Bar")), response -> assertHits(response.getHits(), "1")); @@ -141,7 +141,7 @@ public void testKeywordWithWhitespace() throws Exception { reqs.add(prepareIndex("test").setId("1").setSource("f2", "Foo Bar")); reqs.add(prepareIndex("test").setId("2").setSource("f1", "bar")); reqs.add(prepareIndex("test").setId("3").setSource("f1", "foo bar")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(queryStringQuery("foo")), response -> { assertHits(response.getHits(), "3"); @@ -166,7 +166,7 @@ public void testAllFields() throws Exception { List reqs = new ArrayList<>(); reqs.add(prepareIndex("test_1").setId("1").setSource("f1", "foo", "f2", "eggplant")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertHitCount(prepareSearch("test_1").setQuery(queryStringQuery("foo eggplant").defaultOperator(Operator.AND)), 0L); @@ -180,7 +180,7 @@ public void testPhraseQueryOnFieldWithNoPositions() throws Exception { List reqs = new ArrayList<>(); reqs.add(prepareIndex("test").setId("1").setSource("f1", "foo bar", "f4", "eggplant parmesan")); reqs.add(prepareIndex("test").setId("2").setSource("f1", "foo bar", "f4", "chicken parmesan")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertHitCount(prepareSearch("test").setQuery(queryStringQuery("\"eggplant parmesan\"").lenient(true)), 0L); @@ -214,7 +214,7 @@ public void testFieldAlias() throws Exception { indexRequests.add(prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one")); indexRequests.add(prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two")); indexRequests.add(prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three")); - indexRandom(true, false, indexRequests); + indexRandomAndDecRefRequests(true, false, indexRequests); assertNoFailuresAndResponse(prepareSearch("test").setQuery(queryStringQuery("value").field("f3_alias")), response -> { assertHitCount(response, 2); @@ -227,7 +227,7 @@ public void testFieldAliasWithEmbeddedFieldNames() throws Exception { indexRequests.add(prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one")); indexRequests.add(prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two")); indexRequests.add(prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three")); - indexRandom(true, false, indexRequests); + indexRandomAndDecRefRequests(true, false, indexRequests); assertNoFailuresAndResponse(prepareSearch("test").setQuery(queryStringQuery("f3_alias:value AND f2:three")), response -> { assertHitCount(response, 1); @@ -240,7 +240,7 @@ public void testFieldAliasWithWildcardField() throws Exception { indexRequests.add(prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one")); indexRequests.add(prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two")); indexRequests.add(prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three")); - indexRandom(true, false, indexRequests); + indexRandomAndDecRefRequests(true, false, indexRequests); assertNoFailuresAndResponse(prepareSearch("test").setQuery(queryStringQuery("value").field("f3_*")), response -> { assertHitCount(response, 2); @@ -251,7 +251,7 @@ public void testFieldAliasWithWildcardField() throws Exception { public void testFieldAliasOnDisallowedFieldType() throws Exception { List indexRequests = new ArrayList<>(); indexRequests.add(prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one")); - indexRandom(true, false, indexRequests); + indexRandomAndDecRefRequests(true, false, indexRequests); // The wildcard field matches aliases for both a text and geo_point field. // By default, the geo_point field should be ignored when building the query. @@ -269,4 +269,15 @@ private void assertHits(SearchHits hits, String... ids) { } assertThat(hitIds, containsInAnyOrder(ids)); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, boolean dummyDocuments, List builders) + throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java index 30eeb86ba44d3..06b7ba5cc0975 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java @@ -66,7 +66,7 @@ public void testScriptScore() { assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double")); int docCount = 10; for (int i = 1; i <= docCount; i++) { - prepareIndex("test-index").setId("" + i).setSource("field1", "text" + (i % 2), "field2", i).get(); + indexDoc("test-index", "" + i, "field1", "text" + (i % 2), "field2", i); } refresh(); @@ -94,7 +94,7 @@ public void testScriptScoreBoolQuery() { assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double")); int docCount = 10; for (int i = 1; i <= docCount; i++) { - prepareIndex("test-index").setId("" + i).setSource("field1", "text" + i, "field2", i).get(); + indexDoc("test-index", "" + i, "field1", "text" + i, "field2", i); } refresh(); @@ -115,9 +115,9 @@ public void testRewrittenQuery() { prepareCreate("test-index2").setSettings(Settings.builder().put("index.number_of_shards", 1)) .setMapping("field1", "type=date", "field2", "type=double") ); - prepareIndex("test-index2").setId("1").setSource("field1", "2019-09-01", "field2", 1).get(); - prepareIndex("test-index2").setId("2").setSource("field1", "2019-10-01", "field2", 2).get(); - prepareIndex("test-index2").setId("3").setSource("field1", "2019-11-01", "field2", 3).get(); + indexDoc("test-index2", "1", "field1", "2019-09-01", "field2", 1); + indexDoc("test-index2", "2", "field1", "2019-10-01", "field2", 2); + indexDoc("test-index2", "3", "field1", "2019-11-01", "field2", 3); refresh(); RangeQueryBuilder rangeQB = new RangeQueryBuilder("field1").from("2019-01-01"); // the query should be rewritten to from:null @@ -133,7 +133,7 @@ public void testDisallowExpensiveQueries() { assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double")); int docCount = 10; for (int i = 1; i <= docCount; i++) { - prepareIndex("test-index").setId("" + i).setSource("field1", "text" + (i % 2), "field2", i).get(); + indexDoc("test-index", "" + i, "field1", "text" + (i % 2), "field2", i); } refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java index 384395bcb78e7..c9d1bb8382e23 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -15,6 +15,7 @@ import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.tests.util.English; import org.apache.lucene.util.AttributeSource; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchType; @@ -140,7 +141,7 @@ protected int maximumNumberOfReplicas() { // see #3952 public void testEmptyQueryString() throws ExecutionException, InterruptedException, IOException { createIndex("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("field1", "the quick brown fox jumps"), prepareIndex("test").setId("2").setSource("field1", "quick brown"), @@ -154,9 +155,9 @@ public void testEmptyQueryString() throws ExecutionException, InterruptedExcepti // see https://github.com/elastic/elasticsearch/issues/3177 public void testIssue3177() { createIndex("test"); - prepareIndex("test").setId("1").setSource("field1", "value1").get(); - prepareIndex("test").setId("2").setSource("field1", "value2").get(); - prepareIndex("test").setId("3").setSource("field1", "value3").get(); + indexDoc("test", "1", "field1", "value1"); + indexDoc("test", "2", "field1", "value2"); + indexDoc("test", "3", "field1", "value3"); ensureGreen(); waitForRelocation(); forceMerge(); @@ -184,7 +185,7 @@ public void testIssue3177() { public void testIndexOptions() throws Exception { assertAcked(prepareCreate("test").setMapping("field1", "type=text,index_options=docs")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("field1", "quick brown fox", "field2", "quick brown fox"), prepareIndex("test").setId("2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox") @@ -203,7 +204,7 @@ public void testIndexOptions() throws Exception { public void testConstantScoreQuery() throws Exception { Random random = random(); createIndex("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("field1", "quick brown fox", "field2", "quick brown fox"), prepareIndex("test").setId("2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox") @@ -256,7 +257,7 @@ public void testConstantScoreQuery() throws Exception { builders[i] = prepareIndex("test_1").setId("" + i).setSource("f", English.intToEnglish(i)); } createIndex("test_1"); - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); int queryRounds = scaledRandomIntBetween(10, 20); for (int i = 0; i < queryRounds; i++) { @@ -291,7 +292,11 @@ public void testConstantScoreQuery() throws Exception { // see #3521 public void testAllDocsQueryString() throws InterruptedException, ExecutionException { createIndex("test"); - indexRandom(true, prepareIndex("test").setId("1").setSource("foo", "bar"), prepareIndex("test").setId("2").setSource("foo", "bar")); + indexRandomAndDecRefRequests( + true, + prepareIndex("test").setId("1").setSource("foo", "bar"), + prepareIndex("test").setId("2").setSource("foo", "bar") + ); int iters = scaledRandomIntBetween(100, 200); for (int i = 0; i < iters; i++) { @@ -311,7 +316,7 @@ public void testAllDocsQueryString() throws InterruptedException, ExecutionExcep public void testQueryStringAnalyzedWildcard() throws Exception { createIndex("test"); - prepareIndex("test").setId("1").setSource("field1", "value_1", "field2", "value_2").get(); + indexDoc("test", "1", "field1", "value_1", "field2", "value_2"); refresh(); assertHitCount(prepareSearch().setQuery(queryStringQuery("value*")), 1L); @@ -324,7 +329,7 @@ public void testQueryStringAnalyzedWildcard() throws Exception { public void testLowercaseExpandedTerms() { createIndex("test"); - prepareIndex("test").setId("1").setSource("field1", "value_1", "field2", "value_2").get(); + indexDoc("test", "1", "field1", "value_1", "field2", "value_2"); refresh(); assertHitCount(prepareSearch().setQuery(queryStringQuery("VALUE_3~1")), 1L); @@ -342,7 +347,7 @@ public void testDateRangeInQueryString() { ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); String aMonthAgo = DateTimeFormatter.ISO_LOCAL_DATE.format(now.minusMonths(1)); String aMonthFromNow = DateTimeFormatter.ISO_LOCAL_DATE.format(now.plusMonths(1)); - prepareIndex("test").setId("1").setSource("past", aMonthAgo, "future", aMonthFromNow).get(); + indexDoc("test", "1", "past", aMonthAgo, "future", aMonthFromNow); refresh(); assertHitCount(prepareSearch().setQuery(queryStringQuery("past:[now-2M/d TO now/d]")), 1L); @@ -365,7 +370,7 @@ public void testDateRangeInQueryStringWithTimeZone_7880() { ZoneId timeZone = randomZone(); String now = DateFormatter.forPattern("strict_date_optional_time").format(Instant.now().atZone(timeZone)); logger.info(" --> Using time_zone [{}], now is [{}]", timeZone.getId(), now); - prepareIndex("test").setId("1").setSource("past", now).get(); + indexDoc("test", "1", "past", now); refresh(); assertHitCount(prepareSearch().setQuery(queryStringQuery("past:[now-1m/m TO now+1m/m]").timeZone(timeZone.getId())), 1L); @@ -377,8 +382,8 @@ public void testDateRangeInQueryStringWithTimeZone_10477() { // as with dynamic mappings some shards might be lacking behind and parse a different query assertAcked(prepareCreate("test").setMapping("past", "type=date")); - prepareIndex("test").setId("1").setSource("past", "2015-04-05T23:00:00+0000").get(); - prepareIndex("test").setId("2").setSource("past", "2015-04-06T00:00:00+0000").get(); + indexDoc("test", "1", "past", "2015-04-05T23:00:00+0000"); + indexDoc("test", "2", "past", "2015-04-06T00:00:00+0000"); refresh(); // Timezone set with dates @@ -403,7 +408,7 @@ public void testDateRangeInQueryStringWithTimeZone_10477() { public void testIdsQueryTestsIdIndexed() throws Exception { assertAcked(indicesAdmin().prepareCreate("test")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("field1", "value1"), prepareIndex("test").setId("2").setSource("field1", "value2"), @@ -424,7 +429,7 @@ public void testTermIndexQuery() throws Exception { for (String indexName : indexNames) { assertAcked(indicesAdmin().prepareCreate(indexName)); - indexRandom(true, prepareIndex(indexName).setId(indexName + "1").setSource("field1", "value1")); + indexRandomAndDecRefRequests(true, prepareIndex(indexName).setId(indexName + "1").setSource("field1", "value1")); } @@ -445,7 +450,7 @@ public void testTermIndexQuery() throws Exception { public void testFilterExistsMissing() throws Exception { createIndex("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1") .setSource( @@ -504,7 +509,7 @@ public void testFilterExistsMissing() throws Exception { public void testPassQueryOrFilterAsJSONString() throws Exception { createIndex("test"); - prepareIndex("test").setId("1").setSource("field1", "value1_1", "field2", "value2_1").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1_1", "field2", "value2_1"); WrapperQueryBuilder wrapper = new WrapperQueryBuilder("{ \"term\" : { \"field1\" : \"value1_1\" } }"); assertHitCount(prepareSearch().setQuery(wrapper), 1L); @@ -519,7 +524,7 @@ public void testPassQueryOrFilterAsJSONString() throws Exception { public void testFiltersWithCustomCacheKey() throws Exception { createIndex("test"); - prepareIndex("test").setId("1").setSource("field1", "value1").get(); + indexDoc("test", "1", "field1", "value1"); refresh(); assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))), 1L); assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))), 1L); @@ -530,7 +535,7 @@ public void testFiltersWithCustomCacheKey() throws Exception { public void testMatchQueryNumeric() throws Exception { assertAcked(prepareCreate("test").setMapping("long", "type=long", "double", "type=double")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("long", 1L, "double", 1.0d), prepareIndex("test").setId("2").setSource("long", 2L, "double", 2.0d), @@ -551,7 +556,7 @@ public void testMatchQueryNumeric() throws Exception { public void testMatchQueryFuzzy() throws Exception { assertAcked(prepareCreate("test").setMapping("text", "type=text")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("text", "Unit"), prepareIndex("test").setId("2").setSource("text", "Unity") @@ -575,7 +580,7 @@ public void testMatchQueryFuzzy() throws Exception { public void testMultiMatchQuery() throws Exception { createIndex("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value4", "field3", "value3"), prepareIndex("test").setId("2").setSource("field1", "value2", "field2", "value5", "field3", "value2"), @@ -612,7 +617,7 @@ public void testMultiMatchQuery() throws Exception { assertSearchHits(response, "3", "1"); }); // Test lenient - prepareIndex("test").setId("3").setSource("field1", "value7", "field2", "value8", "field4", 5).get(); + indexDoc("test", "3", "field1", "value7", "field2", "value8", "field4", 5); refresh(); builder = multiMatchQuery("value1", "field1", "field2", "field4"); @@ -645,8 +650,8 @@ public void testMultiMatchQuery() throws Exception { public void testMatchQueryZeroTermsQuery() { assertAcked(prepareCreate("test").setMapping("field1", "type=text,analyzer=classic", "field2", "type=text,analyzer=classic")); - prepareIndex("test").setId("1").setSource("field1", "value1").get(); - prepareIndex("test").setId("2").setSource("field1", "value2").get(); + indexDoc("test", "1", "field1", "value1"); + indexDoc("test", "2", "field1", "value2"); refresh(); BoolQueryBuilder boolQuery = boolQuery().must(matchQuery("field1", "a").zeroTermsQuery(ZeroTermsQueryOption.NONE)) @@ -663,8 +668,8 @@ public void testMatchQueryZeroTermsQuery() { public void testMultiMatchQueryZeroTermsQuery() { assertAcked(prepareCreate("test").setMapping("field1", "type=text,analyzer=classic", "field2", "type=text,analyzer=classic")); - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").get(); - prepareIndex("test").setId("2").setSource("field1", "value3", "field2", "value4").get(); + indexDoc("test", "1", "field1", "value1", "field2", "value2"); + indexDoc("test", "2", "field1", "value3", "field2", "value4"); refresh(); BoolQueryBuilder boolQuery = boolQuery().must(multiMatchQuery("a", "field1", "field2").zeroTermsQuery(ZeroTermsQueryOption.NONE)) @@ -682,8 +687,8 @@ public void testMultiMatchQueryZeroTermsQuery() { public void testMultiMatchQueryMinShouldMatch() { createIndex("test"); - prepareIndex("test").setId("1").setSource("field1", new String[] { "value1", "value2", "value3" }).get(); - prepareIndex("test").setId("2").setSource("field2", "value1").get(); + indexDoc("test", "1", "field1", new String[] { "value1", "value2", "value3" }); + indexDoc("test", "2", "field2", "value1"); refresh(); MultiMatchQueryBuilder multiMatchQuery = multiMatchQuery("value1 value2 foo", "field1", "field2"); @@ -727,8 +732,8 @@ public void testMultiMatchQueryMinShouldMatch() { public void testBoolQueryMinShouldMatchBiggerThanNumberOfShouldClauses() throws IOException { createIndex("test"); - prepareIndex("test").setId("1").setSource("field1", new String[] { "value1", "value2", "value3" }).get(); - prepareIndex("test").setId("2").setSource("field2", "value1").get(); + indexDoc("test", "1", "field1", new String[] { "value1", "value2", "value3" }); + indexDoc("test", "2", "field2", "value1"); refresh(); BoolQueryBuilder boolQuery = boolQuery().must(termQuery("field1", "value1")) @@ -757,8 +762,8 @@ public void testBoolQueryMinShouldMatchBiggerThanNumberOfShouldClauses() throws public void testFuzzyQueryString() { createIndex("test"); - prepareIndex("test").setId("1").setSource("str", "kimchy", "date", "2012-02-01", "num", 12).get(); - prepareIndex("test").setId("2").setSource("str", "shay", "date", "2012-02-05", "num", 20).get(); + indexDoc("test", "1", "str", "kimchy", "date", "2012-02-01", "num", 12); + indexDoc("test", "2", "str", "shay", "date", "2012-02-05", "num", 20); refresh(); assertNoFailuresAndResponse(prepareSearch().setQuery(queryStringQuery("str:kimcy~1")), response -> { @@ -775,7 +780,7 @@ public void testQuotedQueryStringWithBoost() throws InterruptedException { float boost = 10.0f; assertAcked(prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1))); - indexRandom( + indexRandomAndDecRefRequests( true, false, prepareIndex("test").setId("1").setSource("important", "phrase match", "less_important", "nothing important"), @@ -795,8 +800,8 @@ public void testQuotedQueryStringWithBoost() throws InterruptedException { public void testSpecialRangeSyntaxInQueryString() { createIndex("test"); - prepareIndex("test").setId("1").setSource("str", "kimchy", "date", "2012-02-01", "num", 12).get(); - prepareIndex("test").setId("2").setSource("str", "shay", "date", "2012-02-05", "num", 20).get(); + indexDoc("test", "1", "str", "kimchy", "date", "2012-02-01", "num", 12); + indexDoc("test", "2", "str", "shay", "date", "2012-02-05", "num", 20); refresh(); assertResponse(prepareSearch().setQuery(queryStringQuery("num:>19")), response -> { @@ -818,7 +823,7 @@ public void testSpecialRangeSyntaxInQueryString() { public void testEmptytermsQuery() throws Exception { assertAcked(prepareCreate("test").setMapping("term", "type=text")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("term", "1"), prepareIndex("test").setId("2").setSource("term", "2"), @@ -832,7 +837,7 @@ public void testEmptytermsQuery() throws Exception { public void testTermsQuery() throws Exception { assertAcked(prepareCreate("test").setMapping("str", "type=text", "lng", "type=long", "dbl", "type=double")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("str", "1", "lng", 1L, "dbl", 1.0d), prepareIndex("test").setId("2").setSource("str", "2", "lng", 2L, "dbl", 2.0d), @@ -892,7 +897,7 @@ public void testTermsLookupFilter() throws Exception { assertAcked(prepareCreate("lookup3").setMapping("_source", "enabled=false", "terms", "type=text")); assertAcked(prepareCreate("test").setMapping("term", "type=text")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("lookup").setId("1").setSource("terms", new String[] { "1", "3" }), prepareIndex("lookup").setId("2").setSource("terms", new String[] { "2" }), @@ -1005,9 +1010,9 @@ public void testTermsLookupFilter() throws Exception { public void testBasicQueryById() throws Exception { assertAcked(prepareCreate("test")); - prepareIndex("test").setId("1").setSource("field1", "value1").get(); - prepareIndex("test").setId("2").setSource("field1", "value2").get(); - prepareIndex("test").setId("3").setSource("field1", "value3").get(); + indexDoc("test", "1", "field1", "value1"); + indexDoc("test", "2", "field1", "value2"); + indexDoc("test", "3", "field1", "value3"); refresh(); assertResponse(prepareSearch().setQuery(idsQuery().addIds("1", "2")), response -> { @@ -1050,17 +1055,11 @@ public void testNumericTermsAndRanges() throws Exception { ) ); - prepareIndex("test").setId("1") - .setSource("num_byte", 1, "num_short", 1, "num_integer", 1, "num_long", 1, "num_float", 1, "num_double", 1) - .get(); + indexDoc("test", "1", "num_byte", 1, "num_short", 1, "num_integer", 1, "num_long", 1, "num_float", 1, "num_double", 1); - prepareIndex("test").setId("2") - .setSource("num_byte", 2, "num_short", 2, "num_integer", 2, "num_long", 2, "num_float", 2, "num_double", 2) - .get(); + indexDoc("test", "2", "num_byte", 2, "num_short", 2, "num_integer", 2, "num_long", 2, "num_float", 2, "num_double", 2); - prepareIndex("test").setId("17") - .setSource("num_byte", 17, "num_short", 17, "num_integer", 17, "num_long", 17, "num_float", 17, "num_double", 17) - .get(); + indexDoc("test", "17", "num_byte", 17, "num_short", 17, "num_integer", 17, "num_long", 17, "num_float", 17, "num_double", 17); refresh(); logger.info("--> term query on 1"); @@ -1183,10 +1182,10 @@ public void testNumericRangeFilter_2826() throws Exception { ) ); - prepareIndex("test").setId("1").setSource("field1", "test1", "num_long", 1).get(); - prepareIndex("test").setId("2").setSource("field1", "test1", "num_long", 2).get(); - prepareIndex("test").setId("3").setSource("field1", "test2", "num_long", 3).get(); - prepareIndex("test").setId("4").setSource("field1", "test2", "num_long", 4).get(); + indexDoc("test", "1", "field1", "test1", "num_long", 1); + indexDoc("test", "2", "field1", "test1", "num_long", 2); + indexDoc("test", "3", "field1", "test2", "num_long", 3); + indexDoc("test", "4", "field1", "test2", "num_long", 4); refresh(); assertHitCount( @@ -1223,7 +1222,7 @@ public void testMustNot() throws InterruptedException { .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, between(2, DEFAULT_MAX_NUM_SHARDS))) ); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("description", "foo other anything bar"), prepareIndex("test").setId("2").setSource("description", "foo other anything"), @@ -1242,7 +1241,10 @@ public void testMustNot() throws InterruptedException { public void testIntervals() throws InterruptedException { createIndex("test"); - indexRandom(true, prepareIndex("test").setId("1").setSource("description", "it's cold outside, there's no kind of atmosphere")); + indexRandomAndDecRefRequests( + true, + prepareIndex("test").setId("1").setSource("description", "it's cold outside, there's no kind of atmosphere") + ); String json = """ { @@ -1274,7 +1276,7 @@ public void testIntervals() throws InterruptedException { public void testSimpleSpan() throws IOException, ExecutionException, InterruptedException { createIndex("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("description", "foo other anything bar"), prepareIndex("test").setId("2").setSource("description", "foo other anything"), @@ -1294,10 +1296,10 @@ public void testSimpleSpan() throws IOException, ExecutionException, Interrupted public void testSpanMultiTermQuery() throws IOException { createIndex("test"); - prepareIndex("test").setId("1").setSource("description", "foo other anything bar", "count", 1).get(); - prepareIndex("test").setId("2").setSource("description", "foo other anything", "count", 2).get(); - prepareIndex("test").setId("3").setSource("description", "foo other", "count", 3).get(); - prepareIndex("test").setId("4").setSource("description", "fop", "count", 4).get(); + indexDoc("test", "1", "description", "foo other anything bar", "count", 1); + indexDoc("test", "2", "description", "foo other anything", "count", 2); + indexDoc("test", "3", "description", "foo other", "count", 3); + indexDoc("test", "4", "description", "fop", "count", 4); refresh(); assertHitCount(prepareSearch("test").setQuery(spanOrQuery(spanMultiTermQueryBuilder(fuzzyQuery("description", "fop")))), 4); @@ -1315,8 +1317,8 @@ public void testSpanMultiTermQuery() throws IOException { public void testSpanNot() throws IOException, ExecutionException, InterruptedException { createIndex("test"); - prepareIndex("test").setId("1").setSource("description", "the quick brown fox jumped over the lazy dog").get(); - prepareIndex("test").setId("2").setSource("description", "the quick black fox leaped over the sleeping dog").get(); + indexDoc("test", "1", "description", "the quick brown fox jumped over the lazy dog"); + indexDoc("test", "2", "description", "the quick black fox leaped over the sleeping dog"); refresh(); assertHitCount( @@ -1380,22 +1382,10 @@ public void testSimpleDFSQuery() throws IOException { ) ); - prepareIndex("test").setId("1") - .setRouting("Y") - .setSource("online", false, "bs", "Y", "ts", System.currentTimeMillis() - 100, "type", "s") - .get(); - prepareIndex("test").setId("2") - .setRouting("X") - .setSource("online", true, "bs", "X", "ts", System.currentTimeMillis() - 10000000, "type", "s") - .get(); - prepareIndex("test").setId("3") - .setRouting(randomAlphaOfLength(2)) - .setSource("online", false, "ts", System.currentTimeMillis() - 100, "type", "bs") - .get(); - prepareIndex("test").setId("4") - .setRouting(randomAlphaOfLength(2)) - .setSource("online", true, "ts", System.currentTimeMillis() - 123123, "type", "bs") - .get(); + indexDocWithRouting("test", "1", "Y", "online", false, "bs", "Y", "ts", System.currentTimeMillis() - 100, "type", "s"); + indexDocWithRouting("test", "2", "X", "online", true, "bs", "X", "ts", System.currentTimeMillis() - 10000000, "type", "s"); + indexDocWithRouting("test", "3", randomAlphaOfLength(2), "online", false, "ts", System.currentTimeMillis() - 100, "type", "bs"); + indexDocWithRouting("test", "4", randomAlphaOfLength(2), "online", true, "ts", System.currentTimeMillis() - 123123, "type", "bs"); refresh(); assertNoFailures( @@ -1421,7 +1411,7 @@ public void testSimpleDFSQuery() throws IOException { } public void testMultiFieldQueryString() { - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1", "field2", "value2"); logger.info("regular"); assertHitCount(prepareSearch("test").setQuery(queryStringQuery("value1").field("field1").field("field2")), 1); @@ -1444,7 +1434,7 @@ public void testMultiFieldQueryString() { public void testMultiMatchLenientIssue3797() { createIndex("test"); - prepareIndex("test").setId("1").setSource("field1", 123, "field2", "value2").get(); + indexDoc("test", "1", "field1", 123, "field2", "value2"); refresh(); assertHitCount(prepareSearch("test").setQuery(multiMatchQuery("value2", "field2").field("field1", 2).lenient(true)), 1L); @@ -1455,10 +1445,10 @@ public void testMultiMatchLenientIssue3797() { public void testMinScore() throws ExecutionException, InterruptedException { createIndex("test"); - prepareIndex("test").setId("1").setSource("score", 1.5).get(); - prepareIndex("test").setId("2").setSource("score", 1.0).get(); - prepareIndex("test").setId("3").setSource("score", 2.0).get(); - prepareIndex("test").setId("4").setSource("score", 0.5).get(); + indexDoc("test", "1", "score", 1.5); + indexDoc("test", "2", "score", 1.0); + indexDoc("test", "3", "score", 2.0); + indexDoc("test", "4", "score", 0.5); refresh(); assertResponse( @@ -1476,8 +1466,8 @@ public void testMinScore() throws ExecutionException, InterruptedException { public void testQueryStringWithSlopAndFields() { assertAcked(prepareCreate("test")); - prepareIndex("test").setId("1").setSource("desc", "one two three", "type", "customer").get(); - prepareIndex("test").setId("2").setSource("desc", "one two three", "type", "product").get(); + indexDoc("test", "1", "desc", "one two three", "type", "customer"); + indexDoc("test", "2", "desc", "one two three", "type", "product"); refresh(); assertHitCount(prepareSearch("test").setQuery(QueryBuilders.queryStringQuery("\"one two\"").defaultField("desc")), 2); @@ -1506,7 +1496,7 @@ public void testQueryStringWithSlopAndFields() { public void testDateProvidedAsNumber() throws InterruptedException { createIndex("test"); assertAcked(indicesAdmin().preparePutMapping("test").setSource("field", "type=date,format=epoch_millis").get()); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("field", 1000000000001L), prepareIndex("test").setId("2").setSource("field", 1000000000000L), @@ -1523,7 +1513,7 @@ public void testDateProvidedAsNumber() throws InterruptedException { public void testRangeQueryWithTimeZone() throws Exception { assertAcked(prepareCreate("test").setMapping("date", "type=date", "num", "type=integer")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("date", "2014-01-01", "num", 1), prepareIndex("test").setId("2").setSource("date", "2013-12-31T23:00:00", "num", 2), @@ -1651,7 +1641,7 @@ public void testRangeQueryWithLocaleMapping() throws Exception { ) ); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("date_field", "Mi, 06 Dez 2000 02:55:00 -0800"), prepareIndex("test").setId("2").setSource("date_field", "Do, 07 Dez 2000 02:55:00 -0800") @@ -1672,7 +1662,9 @@ public void testRangeQueryWithLocaleMapping() throws Exception { } public void testSearchEmptyDoc() { - prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); assertHitCount(prepareSearch().setQuery(matchAllQuery()), 1L); @@ -1680,7 +1672,7 @@ public void testSearchEmptyDoc() { public void testMatchPhrasePrefixQuery() throws ExecutionException, InterruptedException { createIndex("test1"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test1").setId("1").setSource("field", "Johnnie Walker Black Label"), prepareIndex("test1").setId("2").setSource("field", "trying out Elasticsearch") @@ -1693,7 +1685,7 @@ public void testMatchPhrasePrefixQuery() throws ExecutionException, InterruptedE public void testQueryStringParserCache() throws Exception { createIndex("test"); - indexRandom(true, false, prepareIndex("test").setId("1").setSource("nameTokens", "xyz")); + indexRandomAndDecRefRequests(true, false, prepareIndex("test").setId("1").setSource("nameTokens", "xyz")); final float[] first = new float[1]; assertResponse( prepareSearch("test").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(QueryBuilders.queryStringQuery("xyz").boost(100)), @@ -1721,9 +1713,7 @@ public void testQueryStringParserCache() throws Exception { public void testRangeQueryRangeFields_24744() throws Exception { assertAcked(prepareCreate("test").setMapping("int_range", "type=integer_range")); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().startObject("int_range").field("gte", 10).field("lte", 20).endObject().endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().startObject("int_range").field("gte", 10).field("lte", 20).endObject().endObject()); refresh(); RangeQueryBuilder range = new RangeQueryBuilder("int_range").relation("intersects").from(Integer.MIN_VALUE).to(Integer.MAX_VALUE); @@ -1789,7 +1779,7 @@ public void testFieldAliasesForMetaFields() throws Exception { assertAcked(prepareCreate("test").setMapping(mapping)); IndexRequestBuilder indexRequest = prepareIndex("test").setId("1").setRouting("custom").setSource("field", "value"); - indexRandom(true, false, indexRequest); + indexRandomAndDecRefRequests(true, false, indexRequest); updateClusterSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true)); try { assertResponse(prepareSearch().setQuery(termQuery("routing-alias", "custom")).addDocValueField("id-alias"), response -> { @@ -1821,7 +1811,7 @@ public void testWildcardQueryNormalizationOnKeywordField() { .build() ).setMapping("field1", "type=keyword,normalizer=lowercase_normalizer") ); - prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get(); + indexDoc("test", "1", "field1", "Bbb Aaa"); refresh(); { @@ -1846,7 +1836,7 @@ public void testWildcardQueryNormalizationOnTextField() { .build() ).setMapping("field1", "type=text,analyzer=lowercase_analyzer") ); - prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get(); + indexDoc("test", "1", "field1", "Bbb Aaa"); refresh(); { @@ -1876,7 +1866,7 @@ public void testWildcardQueryNormalizationKeywordSpecialCharacters() { .build() ).setMapping("field", "type=keyword,normalizer=no_wildcard") ); - prepareIndex("test").setId("1").setSource("field", "label-1").get(); + indexDoc("test", "1", "field", "label-1"); refresh(); WildcardQueryBuilder wildCardQuery = wildcardQuery("field", "la*"); @@ -1928,7 +1918,7 @@ public Map> getTokenizers() { */ public void testIssueFuzzyInsideSpanMulti() { createIndex("test"); - prepareIndex("test").setId("1").setSource("field", "foobarbaz").get(); + indexDoc("test", "1", "field", "foobarbaz"); ensureGreen(); refresh(); @@ -1940,7 +1930,7 @@ public void testFetchIdFieldQuery() { createIndex("test"); int docCount = randomIntBetween(10, 50); for (int i = 0; i < docCount; i++) { - prepareIndex("test").setSource("field", "foobarbaz").get(); + indexDoc("test", null, "field", "foobarbaz"); } ensureGreen(); refresh(); @@ -1953,4 +1943,43 @@ public void testFetchIdFieldQuery() { } }); } + + private DocWriteResponse indexDocWithRouting(String index, String id, String routing, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setRouting(routing).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private DocWriteResponse indexDocImmediate(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, boolean dummyDocuments, IndexRequestBuilder... builders) + throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index 7630ddb000140..c53e6533d8f6c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -74,7 +74,7 @@ public void testSimpleQueryString() throws ExecutionException, InterruptedExcept settings.put("index.analysis.analyzer.mock_snowball.tokenizer", "standard"); settings.put("index.analysis.analyzer.mock_snowball.filter", "mock_snowball"); createIndex("test", settings.build()); - indexRandom( + indexRandomAndDecRefRequests( true, false, prepareIndex("test").setId("1").setSource("body", "foo"), @@ -121,7 +121,7 @@ public void testSimpleQueryString() throws ExecutionException, InterruptedExcept public void testSimpleQueryStringMinimumShouldMatch() throws Exception { createIndex("test"); ensureGreen("test"); - indexRandom( + indexRandomAndDecRefRequests( true, false, prepareIndex("test").setId("1").setSource("body", "foo"), @@ -156,7 +156,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception { "4" ); - indexRandom( + indexRandomAndDecRefRequests( true, false, prepareIndex("test").setId("5").setSource("body2", "foo", "other", "foo"), @@ -212,7 +212,7 @@ public void testNestedFieldSimpleQueryString() throws IOException { .endObject() ) ); - prepareIndex("test").setId("1").setSource("body", "foo bar baz").get(); + indexDoc("test", "1", "body", "foo bar baz"); refresh(); assertSearchHitsWithoutFailures(prepareSearch().setQuery(simpleQueryStringQuery("foo bar baz").field("body")), "1"); @@ -223,7 +223,7 @@ public void testNestedFieldSimpleQueryString() throws IOException { public void testSimpleQueryStringFlags() throws ExecutionException, InterruptedException { createIndex("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("body", "foo"), prepareIndex("test").setId("2").setSource("body", "bar"), @@ -281,7 +281,7 @@ public void testSimpleQueryStringFlags() throws ExecutionException, InterruptedE public void testSimpleQueryStringLenient() throws ExecutionException, InterruptedException { createIndex("test1", "test2"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test1").setId("1").setSource("field", "foo"), prepareIndex("test2").setId("10").setSource("field", 5) @@ -302,7 +302,7 @@ public void testSimpleQueryStringLenient() throws ExecutionException, Interrupte // Issue #7967 public void testLenientFlagBeingTooLenient() throws Exception { - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("num", 1, "body", "foo bar baz"), prepareIndex("test").setId("2").setSource("num", 2, "body", "eggplant spaghetti lasagna") @@ -329,22 +329,22 @@ public void testSimpleQueryStringAnalyzeWildcard() throws ExecutionException, In CreateIndexRequestBuilder mappingRequest = indicesAdmin().prepareCreate("test1").setMapping(mapping); mappingRequest.get(); - indexRandom(true, prepareIndex("test1").setId("1").setSource("location", "Köln")); + indexRandomAndDecRefRequests(true, prepareIndex("test1").setId("1").setSource("location", "Köln")); refresh(); assertSearchHitsWithoutFailures(prepareSearch().setQuery(simpleQueryStringQuery("Köln*").field("location")), "1"); } public void testSimpleQueryStringUsesFieldAnalyzer() throws Exception { - prepareIndex("test").setId("1").setSource("foo", 123, "bar", "abc").get(); - prepareIndex("test").setId("2").setSource("foo", 234, "bar", "bcd").get(); + indexDoc("test", "1", "foo", 123, "bar", "abc"); + indexDoc("test", "2", "foo", 234, "bar", "bcd"); refresh(); assertSearchHitsWithoutFailures(prepareSearch().setQuery(simpleQueryStringQuery("123").field("foo").field("bar")), "1"); } public void testSimpleQueryStringOnIndexMetaField() throws Exception { - prepareIndex("test").setId("1").setSource("foo", 123, "bar", "abc").get(); - prepareIndex("test").setId("2").setSource("foo", 234, "bar", "bcd").get(); + indexDoc("test", "1", "foo", 123, "bar", "abc"); + indexDoc("test", "2", "foo", 234, "bar", "bcd"); refresh(); assertSearchHitsWithoutFailures(prepareSearch().setQuery(simpleQueryStringQuery("test").field("_index")), "1", "2"); } @@ -365,7 +365,7 @@ public void testEmptySimpleQueryStringWithAnalysis() throws Exception { CreateIndexRequestBuilder mappingRequest = indicesAdmin().prepareCreate("test1").setMapping(mapping); mappingRequest.get(); - indexRandom(true, prepareIndex("test1").setId("1").setSource("body", "Some Text")); + indexRandomAndDecRefRequests(true, prepareIndex("test1").setId("1").setSource("body", "Some Text")); refresh(); assertSearchHitsWithoutFailures(prepareSearch().setQuery(simpleQueryStringQuery("the*").field("body"))); @@ -380,7 +380,7 @@ public void testBasicAllQuery() throws Exception { reqs.add(prepareIndex("test").setId("1").setSource("f1", "foo bar baz")); reqs.add(prepareIndex("test").setId("2").setSource("f2", "Bar")); reqs.add(prepareIndex("test").setId("3").setSource("f3", "foo bar baz")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(simpleQueryStringQuery("foo")), response -> { assertHitCount(response, 2L); @@ -404,7 +404,7 @@ public void testWithDate() throws Exception { List reqs = new ArrayList<>(); reqs.add(prepareIndex("test").setId("1").setSource("f1", "foo", "f_date", "2015/09/02")); reqs.add(prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")), response -> { assertHits(response.getHits(), "1", "2"); @@ -432,7 +432,7 @@ public void testWithLotsOfTypes() throws Exception { List reqs = new ArrayList<>(); reqs.add(prepareIndex("test").setId("1").setSource("f1", "foo", "f_date", "2015/09/02", "f_float", "1.7", "f_ip", "127.0.0.1")); reqs.add(prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01", "f_float", "1.8", "f_ip", "127.0.0.2")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")), response -> { assertHits(response.getHits(), "1", "2"); @@ -460,7 +460,7 @@ public void testDocWithAllTypes() throws Exception { List reqs = new ArrayList<>(); String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json"); reqs.add(prepareIndex("test").setId("1").setSource(docBody, XContentType.JSON)); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(simpleQueryStringQuery("foo")), response -> assertHits(response.getHits(), "1")); assertResponse(prepareSearch("test").setQuery(simpleQueryStringQuery("Bar")), response -> assertHits(response.getHits(), "1")); @@ -502,7 +502,7 @@ public void testKeywordWithWhitespace() throws Exception { reqs.add(prepareIndex("test").setId("1").setSource("f2", "Foo Bar")); reqs.add(prepareIndex("test").setId("2").setSource("f1", "bar")); reqs.add(prepareIndex("test").setId("3").setSource("f1", "foo bar")); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); assertResponse(prepareSearch("test").setQuery(simpleQueryStringQuery("foo")), response -> { assertHits(response.getHits(), "3"); @@ -521,7 +521,7 @@ public void testAllFieldsWithSpecifiedLeniency() throws Exception { List reqs = new ArrayList<>(); reqs.add(prepareIndex("test").setId("1").setSource("f_long", 1)); - indexRandom(true, false, reqs); + indexRandomAndDecRefRequests(true, false, reqs); SearchPhaseExecutionException e = expectThrows( SearchPhaseExecutionException.class, @@ -539,7 +539,7 @@ public void testFieldAlias() throws Exception { indexRequests.add(prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one")); indexRequests.add(prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two")); indexRequests.add(prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three")); - indexRandom(true, false, indexRequests); + indexRandomAndDecRefRequests(true, false, indexRequests); assertNoFailuresAndResponse(prepareSearch("test").setQuery(simpleQueryStringQuery("value").field("f3_alias")), response -> { assertHitCount(response, 2); @@ -556,7 +556,7 @@ public void testFieldAliasWithWildcardField() throws Exception { indexRequests.add(prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one")); indexRequests.add(prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two")); indexRequests.add(prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three")); - indexRandom(true, false, indexRequests); + indexRandomAndDecRefRequests(true, false, indexRequests); assertNoFailuresAndResponse(prepareSearch("test").setQuery(simpleQueryStringQuery("value").field("f3_*")), response -> { assertHitCount(response, 2); @@ -571,7 +571,7 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception { List indexRequests = new ArrayList<>(); indexRequests.add(prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one")); - indexRandom(true, false, indexRequests); + indexRandomAndDecRefRequests(true, false, indexRequests); // The wildcard field matches aliases for both a text and boolean field. // By default, the boolean field should be ignored when building the query. @@ -617,4 +617,36 @@ public List getPreConfiguredTokenFilters() { return singletonList(PreConfiguredTokenFilter.singleton("mock_snowball", false, MockSnowBall::new)); } } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, boolean dummyDocuments, IndexRequestBuilder... builders) + throws InterruptedException { + try { + super.indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, boolean dummyDocuments, List builders) + throws InterruptedException { + try { + super.indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + super.indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java index 433f004acdd77..2a6c45c10ce1d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java @@ -55,7 +55,7 @@ public void testStopOneNodePreferenceWithRedState() throws IOException { assertAcked(prepareCreate("test").setSettings(indexSettings(cluster().numDataNodes() + 2, 0))); ensureGreen(); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId("" + i).setSource("field1", "value1").get(); + indexDoc("test", "" + i, "field1", "value1"); } refresh(); internalCluster().stopRandomDataNode(); @@ -97,7 +97,7 @@ public void testNoPreferenceRandom() { ); ensureGreen(); - prepareIndex("test").setSource("field1", "value1").get(); + indexDoc("test", null, "field1", "value1"); refresh(); final Client client = internalCluster().smartClient(); @@ -117,7 +117,7 @@ public void testSimplePreference() { indicesAdmin().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get(); ensureGreen(); - prepareIndex("test").setSource("field1", "value1").get(); + indexDoc("test", null, "field1", "value1"); refresh(); assertResponse( @@ -156,7 +156,7 @@ public void testNodesOnlyRandom() { ) ); ensureGreen(); - prepareIndex("test").setSource("field1", "value1").get(); + indexDoc("test", null, "field1", "value1"); refresh(); final Client client = internalCluster().smartClient(); @@ -227,7 +227,7 @@ public void testCustomPreferenceUnaffectedByOtherShardMovements() { ) ); ensureGreen(); - prepareIndex("test").setSource("field1", "value1").get(); + indexDoc("test", null, "field1", "value1"); refresh(); final String customPreference = randomAlphaOfLength(10); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java index 816fe48e5d97f..cc757204405e2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.OperationRouting; @@ -44,7 +45,9 @@ public void testNodeSelection() { client.admin().indices().prepareCreate("test").setSettings(indexSettings(1, 2)).get(); ensureGreen(); - client.prepareIndex("test").setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex("test").setSource("field", "value"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); // Before we've gathered stats for all nodes, we should try each node once. diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java index 9f6ad69a4eed6..1d30703a584f5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java @@ -102,13 +102,17 @@ public void testCustomScriptBinaryField() throws Exception { final byte[] randomBytesDoc2 = getRandomBytes(16); assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("binary")).setSettings(indexSettings())); - prepareIndex("my-index").setId("1") - .setSource(jsonBuilder().startObject().field("binaryData", Base64.getEncoder().encodeToString(randomBytesDoc1)).endObject()) - .get(); + index( + "my-index", + "1", + jsonBuilder().startObject().field("binaryData", Base64.getEncoder().encodeToString(randomBytesDoc1)).endObject() + ); flush(); - prepareIndex("my-index").setId("2") - .setSource(jsonBuilder().startObject().field("binaryData", Base64.getEncoder().encodeToString(randomBytesDoc2)).endObject()) - .get(); + index( + "my-index", + "2", + jsonBuilder().startObject().field("binaryData", Base64.getEncoder().encodeToString(randomBytesDoc2)).endObject() + ); flush(); refresh(); @@ -150,17 +154,11 @@ private XContentBuilder createMappingSource(String fieldType) throws IOException public void testCustomScriptBoost() throws Exception { createIndex("test"); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); flush(); - prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()) - .get(); + index("test", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); flush(); - prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject()) - .get(); + index("test", "3", jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject()); refresh(); logger.info("running doc['num1'].value > 1"); @@ -229,7 +227,7 @@ public void testDisallowExpensiveQueries() { assertAcked(prepareCreate("test-index").setMapping("num1", "type=double")); int docCount = 10; for (int i = 1; i <= docCount; i++) { - prepareIndex("test-index").setId("" + i).setSource("num1", i).get(); + indexDoc("test-index", "" + i, "num1", i); } refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java index 036467b8d0774..153b844e01923 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java @@ -156,6 +156,7 @@ private TestContext create(SearchType... searchTypes) throws Exception { indexRequestBuilder.setRouting("a"); } indexRandom(false, indexRequestBuilder); + indexRequestBuilder.request().decRef(); } refresh(); @@ -214,6 +215,9 @@ private int createIndex(boolean singleShard) throws Exception { builders[i] = prepareIndex("test").setId(Integer.toString(i)).setSource("foo", random().nextBoolean()); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } return numDocs; } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java index 7dcdb92ec5680..32cd1aee07884 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.scroll; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -75,7 +76,7 @@ public void testSimpleScrollQueryThenFetch() throws Exception { clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().get(); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource(jsonBuilder().startObject().field("field", i).endObject()).get(); + index("test", Integer.toString(i), jsonBuilder().startObject().field("field", i).endObject()); } indicesAdmin().prepareRefresh().get(); @@ -130,7 +131,11 @@ public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws E } else if (i > 60) { routing = "2"; } - prepareIndex("test").setId(Integer.toString(i)).setSource("field", i).setRouting(routing).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)) + .setSource("field", i) + .setRouting(routing); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } indicesAdmin().prepareRefresh().get(); @@ -192,15 +197,15 @@ public void testScrollAndUpdateIndex() throws Exception { clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().get(); for (int i = 0; i < 500; i++) { - prepareIndex("test").setId(Integer.toString(i)) - .setSource( - jsonBuilder().startObject() - .field("user", "kimchy") - .field("postDate", System.currentTimeMillis()) - .field("message", "test") - .endObject() - ) - .get(); + index( + "test", + Integer.toString(i), + jsonBuilder().startObject() + .field("user", "kimchy") + .field("postDate", System.currentTimeMillis()) + .field("message", "test") + .endObject() + ); } indicesAdmin().prepareRefresh().get(); @@ -221,7 +226,7 @@ public void testScrollAndUpdateIndex() throws Exception { for (SearchHit searchHit : searchResponse.getHits().getHits()) { Map map = searchHit.getSourceAsMap(); map.put("message", "update"); - prepareIndex("test").setId(searchHit.getId()).setSource(map).get(); + index("test", searchHit.getId(), map); } searchResponse.decRef(); searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get(); @@ -246,7 +251,7 @@ public void testSimpleScrollQueryThenFetch_clearScrollIds() throws Exception { clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().get(); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource(jsonBuilder().startObject().field("field", i).endObject()).get(); + index("test", Integer.toString(i), jsonBuilder().startObject().field("field", i).endObject()); } indicesAdmin().prepareRefresh().get(); @@ -365,7 +370,7 @@ public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception { clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().get(); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource(jsonBuilder().startObject().field("field", i).endObject()).get(); + index("test", Integer.toString(i), jsonBuilder().startObject().field("field", i).endObject()); } indicesAdmin().prepareRefresh().get(); @@ -447,7 +452,9 @@ public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception { * Tests that we use an optimization shrinking the batch to the size of the shard. Thus the Integer.MAX_VALUE window doesn't OOM us. */ public void testDeepScrollingDoesNotBlowUp() throws Exception { - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).execute().get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); /* * Disable the max result window setting for this test because it'll reject the search's unreasonable batch size. We want * unreasonable batch sizes to just OOM. @@ -474,7 +481,7 @@ public void testDeepScrollingDoesNotBlowUp() throws Exception { } public void testThatNonExistingScrollIdReturnsCorrectException() throws Exception { - prepareIndex("index").setId("1").setSource("field", "value").execute().get(); + indexDoc("index", "1", "field", "value"); refresh(); SearchResponse searchResponse = prepareSearch("index").setSize(1).setScroll("1m").get(); @@ -494,7 +501,7 @@ public void testStringSortMissingAscTerminates() throws Exception { assertAcked( prepareCreate("test").setSettings(indexSettings(1, 0)).setMapping("no_field", "type=keyword", "some_field", "type=keyword") ); - prepareIndex("test").setId("1").setSource("some_field", "test").get(); + indexDoc("test", "1", "some_field", "test"); refresh(); assertResponse( @@ -525,7 +532,7 @@ public void testStringSortMissingAscTerminates() throws Exception { public void testCloseAndReopenOrDeleteWithActiveScroll() { createIndex("test"); for (int i = 0; i < 100; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", i).get(); + indexDoc("test", Integer.toString(i), "field", i); } refresh(); assertResponse( @@ -578,7 +585,7 @@ public void testScrollInvalidDefaultKeepAlive() throws IOException { public void testInvalidScrollKeepAlive() throws IOException { createIndex("test"); for (int i = 0; i < 2; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource(jsonBuilder().startObject().field("field", i).endObject()).get(); + index("test", Integer.toString(i), jsonBuilder().startObject().field("field", i).endObject()); } refresh(); updateClusterSettings(Settings.builder().put("search.default_keep_alive", "5m").put("search.max_keep_alive", "5m")); @@ -621,9 +628,9 @@ public void testScrollRewrittenToMatchNoDocs() { {"properties":{"created_date":{"type": "date", "format": "yyyy-MM-dd"}}} """) ); - prepareIndex("test").setId("1").setSource("created_date", "2020-01-01").get(); - prepareIndex("test").setId("2").setSource("created_date", "2020-01-02").get(); - prepareIndex("test").setId("3").setSource("created_date", "2020-01-03").get(); + indexDoc("test", "1", "created_date", "2020-01-01"); + indexDoc("test", "2", "created_date", "2020-01-02"); + indexDoc("test", "3", "created_date", "2020-01-03"); indicesAdmin().prepareRefresh("test").get(); SearchResponse resp = null; try { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java index 42be70e5ff8b2..7ccd2c0aac6d9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java @@ -56,6 +56,9 @@ public void testScanScrollWithShardExceptions() throws Exception { writes.add(prepareIndex("test").setSource(jsonBuilder().startObject().field("field", i).endObject())); } indexRandom(false, writes); + for (IndexRequestBuilder builder : writes) { + builder.request().decRef(); + } refresh(); SearchResponse searchResponse = prepareSearch().setQuery(matchAllQuery()) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java index 93e3582b645fa..920895d98535a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java @@ -65,7 +65,9 @@ public class SearchAfterIT extends ESIntegTestCase { public void testsShouldFail() throws Exception { assertAcked(indicesAdmin().prepareCreate("test").setMapping("field1", "type=long", "field2", "type=keyword").get()); ensureGreen(); - indexRandom(true, prepareIndex("test").setId("0").setSource("field1", 0, "field2", "toto")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("0").setSource("field1", 0, "field2", "toto"); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); { SearchPhaseExecutionException e = expectThrows( SearchPhaseExecutionException.class, @@ -142,11 +144,14 @@ public void testsShouldFail() throws Exception { public void testWithNullStrings() throws InterruptedException { assertAcked(indicesAdmin().prepareCreate("test").setMapping("field2", "type=keyword").get()); ensureGreen(); - indexRandom( - true, + List builders = List.of( prepareIndex("test").setId("0").setSource("field1", 0), prepareIndex("test").setId("1").setSource("field1", 100, "field2", "toto") ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertResponse( prepareSearch("test").addSort("field1", SortOrder.ASC) .addSort("field2", SortOrder.ASC) @@ -213,13 +218,19 @@ public void testWithCustomFormatSortValueOfDateField() throws Exception { .setMapping(mappings) ); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .add(new IndexRequest("test").id("1").source("start_date", "2019-03-24", "end_date", "2020-01-21")) - .add(new IndexRequest("test").id("2").source("start_date", "2018-04-23", "end_date", "2021-02-22")) - .add(new IndexRequest("test").id("3").source("start_date", "2015-01-22", "end_date", "2022-07-23")) - .add(new IndexRequest("test").id("4").source("start_date", "2016-02-21", "end_date", "2024-03-24")) - .add(new IndexRequest("test").id("5").source("start_date", "2017-01-20", "end_date", "2025-05-28")) - .get(); + List indexRequests = List.of( + new IndexRequest("test").id("1").source("start_date", "2019-03-24", "end_date", "2020-01-21"), + new IndexRequest("test").id("2").source("start_date", "2018-04-23", "end_date", "2021-02-22"), + new IndexRequest("test").id("3").source("start_date", "2015-01-22", "end_date", "2022-07-23"), + new IndexRequest("test").id("4").source("start_date", "2016-02-21", "end_date", "2024-03-24"), + new IndexRequest("test").id("5").source("start_date", "2017-01-20", "end_date", "2025-05-28") + ); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (IndexRequest indexRequest : indexRequests) { + bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); + } + bulkRequestBuilder.get(); } assertNoFailuresAndResponse( @@ -315,6 +326,9 @@ private void assertSearchFromWithSortValues(String indexName, List> requests.add(prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource(builder)); } indexRandom(true, requests); + for (IndexRequestBuilder builder : requests) { + builder.request().decRef(); + } } Collections.sort(documents, LST_COMPARATOR); @@ -426,7 +440,9 @@ public void testScrollAndSearchAfterWithBigIndex() { try (BulkRequestBuilder bulk = client().prepareBulk()) { bulk.setRefreshPolicy(IMMEDIATE); for (long timestamp : timestamps) { - bulk.add(new IndexRequest("test").source("timestamp", timestamp)); + IndexRequest indexRequest = new IndexRequest("test").source("timestamp", timestamp); + bulk.add(indexRequest); + indexRequest.decRef(); } bulk.get(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 543f45b58279e..e9683c5880bc8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.simple; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -70,7 +71,7 @@ public void testSearchNullIndex() { public void testSearchRandomPreference() throws InterruptedException, ExecutionException { createIndex("test"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("field", "value"), prepareIndex("test").setId("2").setSource("field", "value"), @@ -113,7 +114,7 @@ public void testSimpleIp() throws Exception { ) .get(); - prepareIndex("test").setId("1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "from", "192.168.0.5", "to", "192.168.0.10"); assertHitCount( prepareSearch().setQuery(boolQuery().must(rangeQuery("from").lte("192.168.0.7")).must(rangeQuery("to").gte("192.168.0.7"))), 1L @@ -139,11 +140,11 @@ public void testIpCidr() throws Exception { .get(); ensureGreen(); - prepareIndex("test").setId("1").setSource("ip", "192.168.0.1").get(); - prepareIndex("test").setId("2").setSource("ip", "192.168.0.2").get(); - prepareIndex("test").setId("3").setSource("ip", "192.168.0.3").get(); - prepareIndex("test").setId("4").setSource("ip", "192.168.1.4").get(); - prepareIndex("test").setId("5").setSource("ip", "2001:db8::ff00:42:8329").get(); + indexDoc("test", "1", "ip", "192.168.0.1"); + indexDoc("test", "2", "ip", "192.168.0.2"); + indexDoc("test", "3", "ip", "192.168.0.3"); + indexDoc("test", "4", "ip", "192.168.1.4"); + indexDoc("test", "5", "ip", "2001:db8::ff00:42:8329"); refresh(); assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1"))), 1L); @@ -167,7 +168,7 @@ public void testIpCidr() throws Exception { public void testSimpleId() { createIndex("test"); - prepareIndex("test").setId("XXX1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "XXX1", "field", "value"); // id is not indexed, but lets see that we automatically convert to assertHitCount(prepareSearch().setQuery(QueryBuilders.termQuery("_id", "XXX1")), 1L); assertHitCount(prepareSearch().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")), 1L); @@ -175,9 +176,9 @@ public void testSimpleId() { public void testSimpleDateRange() throws Exception { createIndex("test"); - prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); - prepareIndex("test").setId("2").setSource("field", "2010-01-06T02:00").get(); - prepareIndex("test").setId("3").setSource("field", "1967-01-01T00:00").get(); + indexDoc("test", "1", "field", "2010-01-05T02:00"); + indexDoc("test", "2", "field", "2010-01-06T02:00"); + indexDoc("test", "3", "field", "1967-01-01T00:00"); ensureGreen(); refresh(); assertHitCountAndNoFailures( @@ -219,10 +220,10 @@ public void testRangeQueryKeyword() throws Exception { indicesAdmin().preparePutMapping("test").setSource("field", "type=keyword").get(); - prepareIndex("test").setId("0").setSource("field", "").get(); - prepareIndex("test").setId("1").setSource("field", "A").get(); - prepareIndex("test").setId("2").setSource("field", "B").get(); - prepareIndex("test").setId("3").setSource("field", "C").get(); + indexDoc("test", "0", "field", ""); + indexDoc("test", "1", "field", "A"); + indexDoc("test", "2", "field", "B"); + indexDoc("test", "3", "field", "C"); ensureGreen(); refresh(); @@ -247,7 +248,7 @@ public void testSimpleTerminateAfterCount() throws Exception { docbuilders.add(prepareIndex("test").setId(id).setSource("field", i)); } - indexRandom(true, docbuilders); + indexRandomAndDecRefRequests(true, docbuilders); ensureGreen(); refresh(); @@ -281,7 +282,7 @@ public void testSimpleIndexSortEarlyTerminate() throws Exception { docbuilders.add(prepareIndex("test").setId(id).setSource("rank", i)); } - indexRandom(true, docbuilders); + indexRandomAndDecRefRequests(true, docbuilders); ensureGreen(); refresh(); @@ -301,7 +302,7 @@ public void testSimpleIndexSortEarlyTerminate() throws Exception { public void testInsaneFromAndSize() throws Exception { createIndex("idx"); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertWindowFails(prepareSearch("idx").setFrom(Integer.MAX_VALUE)); assertWindowFails(prepareSearch("idx").setSize(Integer.MAX_VALUE)); @@ -309,7 +310,7 @@ public void testInsaneFromAndSize() throws Exception { public void testTooLargeFromAndSize() throws Exception { createIndex("idx"); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertWindowFails(prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))); assertWindowFails(prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1)); @@ -321,7 +322,7 @@ public void testTooLargeFromAndSize() throws Exception { public void testLargeFromAndSizeSucceeds() throws Exception { createIndex("idx"); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertHitCount(prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10), 1); assertHitCount(prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), 1); @@ -337,7 +338,7 @@ public void testTooLargeFromAndSizeOkBySetting() throws Exception { Settings.builder() .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2) ).get(); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertHitCount(prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), 1); assertHitCount(prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1), 1); @@ -355,7 +356,7 @@ public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2), "idx" ); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertHitCount(prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), 1); assertHitCount(prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1), 1); @@ -368,7 +369,7 @@ public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception { prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE)).get(); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertHitCount(prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10), 1); assertHitCount(prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10), 1); @@ -381,7 +382,7 @@ public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws public void testTooLargeRescoreWindow() throws Exception { createIndex("idx"); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertRescoreWindowFails(Integer.MAX_VALUE); assertRescoreWindowFails(IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + 1); @@ -391,7 +392,7 @@ public void testTooLargeRescoreOkBySetting() throws Exception { int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) .get(); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertHitCount(prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)), 1); } @@ -405,7 +406,7 @@ public void testTooLargeRescoreOkByResultWindowSetting() throws Exception { defaultMaxWindow * 2 ) ).get(); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertHitCount(prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)), 1); } @@ -414,7 +415,7 @@ public void testTooLargeRescoreOkByDynamicSetting() throws Exception { int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); createIndex("idx"); updateIndexSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2), "idx"); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertHitCount(prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)), 1); } @@ -427,7 +428,7 @@ public void testTooLargeRescoreOkByDynamicResultWindowSetting() throws Exception Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), defaultMaxWindow * 2), "idx" ); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); assertHitCount(prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)), 1); } @@ -448,10 +449,11 @@ public void testTermQueryBigInt() throws Exception { prepareCreate("idx").setMapping("field", "type=keyword").get(); ensureGreen("idx"); - prepareIndex("idx").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("idx").setId("1") .setSource("{\"field\" : 80315953321748200608 }", XContentType.JSON) - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); String queryJson = "{ \"field\" : { \"value\" : 80315953321748200608 } }"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, queryJson)) { @@ -463,7 +465,7 @@ public void testTermQueryBigInt() throws Exception { public void testTooLongRegexInRegexpQuery() throws Exception { createIndex("idx"); - indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); + indexRandomAndDecRefRequests(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); int defaultMaxRegexLength = IndexSettings.MAX_REGEX_LENGTH_SETTING.get(Settings.EMPTY); StringBuilder regexp = new StringBuilder(defaultMaxRegexLength); @@ -492,7 +494,7 @@ public void testTooLongRegexInRegexpQuery() throws Exception { public void testStrictlyCountRequest() throws Exception { createIndex("test_count_1"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test_count_1").setId("1").setSource("field", "value"), prepareIndex("test_count_1").setId("2").setSource("field", "value"), @@ -503,7 +505,7 @@ public void testStrictlyCountRequest() throws Exception { ); createIndex("test_count_2"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test_count_2").setId("1").setSource("field", "value_2"), prepareIndex("test_count_2").setId("2").setSource("field", "value_2"), @@ -554,6 +556,35 @@ private void assertRescoreWindowFails(int windowSize) { ); } + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private DocWriteResponse indexDocImmediate(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + public static class VerifyReaderContextPlugin extends Plugin { @Override public void onIndexModule(IndexModule indexModule) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java index 53bc8ba439c3e..dc424dfd79318 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java @@ -83,6 +83,9 @@ private void setupIndex(int numDocs, int numberOfShards) throws IOException, Exe requests.add(prepareIndex("test").setSource(builder)); } indexRandom(true, requests); + for (IndexRequestBuilder builder : requests) { + builder.request().decRef(); + } } public void testSearchSort() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java index 53e39b0f00ed2..ddad063e4512d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -118,7 +118,7 @@ public void testIssue8226() { assertAcked(prepareCreate("test_" + i).addAlias(new Alias("test"))); } if (i > 0) { - prepareIndex("test_" + i).setId("" + i).setSource("{\"entry\": " + i + "}", XContentType.JSON).get(); + indexDoc("test_" + i, "" + i, "{\"entry\": " + i + "}", XContentType.JSON); } } refresh(); @@ -174,6 +174,9 @@ public void testIssue6614() throws InterruptedException { } indexRandom(true, builders); docs += builders.size(); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } builders.clear(); } final int finalDocs = docs; @@ -281,6 +284,9 @@ public void testRandomSorting() throws IOException, InterruptedException, Execut builders[i] = prepareIndex("test").setId(docId).setSource(src); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } { int size = between(1, denseBytes.size()); assertNoFailuresAndResponse( @@ -327,7 +333,7 @@ public void test3078() { ensureGreen(); for (int i = 1; i < 101; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", Integer.toString(i)).get(); + indexDoc("test", Integer.toString(i), "field", Integer.toString(i)); } refresh(); assertResponse( @@ -339,7 +345,7 @@ public void test3078() { } ); // reindex and refresh - prepareIndex("test").setId(Integer.toString(1)).setSource("field", Integer.toString(1)).get(); + indexDoc("test", Integer.toString(1), "field", Integer.toString(1)); refresh(); assertResponse( @@ -351,7 +357,7 @@ public void test3078() { } ); // reindex - no refresh - prepareIndex("test").setId(Integer.toString(1)).setSource("field", Integer.toString(1)).get(); + indexDoc("test", Integer.toString(1), "field", Integer.toString(1)); assertResponse( prepareSearch("test").setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)), @@ -365,7 +371,7 @@ public void test3078() { forceMerge(); refresh(); - prepareIndex("test").setId(Integer.toString(1)).setSource("field", Integer.toString(1)).get(); + indexDoc("test", Integer.toString(1), "field", Integer.toString(1)); assertResponse( prepareSearch("test").setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)), response -> { @@ -389,9 +395,9 @@ public void testScoreSortDirection() throws Exception { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("field", 2).get(); - prepareIndex("test").setId("2").setSource("field", 1).get(); - prepareIndex("test").setId("3").setSource("field", 0).get(); + indexDoc("test", "1", "field", 2); + indexDoc("test", "2", "field", 1); + indexDoc("test", "3", "field", 0); refresh(); @@ -435,9 +441,9 @@ public void testScoreSortDirectionWithFunctionScore() throws Exception { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("field", 2).get(); - prepareIndex("test").setId("2").setSource("field", 1).get(); - prepareIndex("test").setId("3").setSource("field", 0).get(); + indexDoc("test", "1", "field", 2); + indexDoc("test", "2", "field", 1); + indexDoc("test", "3", "field", 0); refresh(); @@ -473,9 +479,9 @@ public void testScoreSortDirectionWithFunctionScore() throws Exception { public void testIssue2986() { assertAcked(indicesAdmin().prepareCreate("test").setMapping("field1", "type=keyword").get()); - prepareIndex("test").setId("1").setSource("{\"field1\":\"value1\"}", XContentType.JSON).get(); - prepareIndex("test").setId("2").setSource("{\"field1\":\"value2\"}", XContentType.JSON).get(); - prepareIndex("test").setId("3").setSource("{\"field1\":\"value3\"}", XContentType.JSON).get(); + indexDoc("test", "1", "{\"field1\":\"value1\"}", XContentType.JSON); + indexDoc("test", "2", "{\"field1\":\"value2\"}", XContentType.JSON); + indexDoc("test", "3", "{\"field1\":\"value3\"}", XContentType.JSON); refresh(); assertResponse(prepareSearch("test").setQuery(matchAllQuery()).setTrackScores(true).addSort("field1", SortOrder.ASC), response -> { for (SearchHit hit : response.getHits()) { @@ -493,16 +499,16 @@ public void testIssue2991() { } assertAcked(indicesAdmin().prepareCreate("test").setMapping("tag", "type=keyword").get()); ensureGreen(); - prepareIndex("test").setId("1").setSource("tag", "alpha").get(); + indexDoc("test", "1", "tag", "alpha"); refresh(); - prepareIndex("test").setId("3").setSource("tag", "gamma").get(); + indexDoc("test", "3", "tag", "gamma"); refresh(); - prepareIndex("test").setId("4").setSource("tag", "delta").get(); + indexDoc("test", "4", "tag", "delta"); refresh(); - prepareIndex("test").setId("2").setSource("tag", "beta").get(); + indexDoc("test", "2", "tag", "beta"); refresh(); assertResponse( @@ -584,6 +590,7 @@ public void testSimpleSorts() throws Exception { Collections.shuffle(builders, random); for (IndexRequestBuilder builder : builders) { builder.get(); + builder.request().decRef(); if (random.nextBoolean()) { if (random.nextInt(5) != 0) { refresh(); @@ -801,15 +808,10 @@ public void testSortMissingNumbers() throws Exception { ) ); ensureGreen(); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("id", "1").field("i_value", -1).field("d_value", -1.1).endObject()) - .get(); - - prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("id", "2").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("id", "1").field("i_value", -1).field("d_value", -1.1).endObject()); + index("test", "2", jsonBuilder().startObject().field("id", "2").endObject()); - prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("id", "1").field("i_value", 2).field("d_value", 2.2).endObject()) - .get(); + index("test", "3", jsonBuilder().startObject().field("id", "1").field("i_value", 2).field("d_value", 2.2).endObject()); flush(); refresh(); @@ -862,11 +864,11 @@ public void testSortMissingStrings() throws IOException { ) ); ensureGreen(); - prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("id", "1").field("value", "a").endObject()).get(); + index("test", "1", jsonBuilder().startObject().field("id", "1").field("value", "a").endObject()); - prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("id", "2").endObject()).get(); + index("test", "2", jsonBuilder().startObject().field("id", "2").endObject()); - prepareIndex("test").setId("3").setSource(jsonBuilder().startObject().field("id", "1").field("value", "c").endObject()).get(); + index("test", "3", jsonBuilder().startObject().field("id", "1").field("value", "c").endObject()); flush(); refresh(); @@ -946,9 +948,9 @@ public void testSortMissingDates() throws IOException { ) ); ensureGreen(); - prepareIndex(index).setId("1").setSource("mydate", "2021-01-01").get(); - prepareIndex(index).setId("2").setSource("mydate", "2021-02-01").get(); - prepareIndex(index).setId("3").setSource("other_field", "value").get(); + indexDoc(index, "1", "mydate", "2021-01-01"); + indexDoc(index, "2", "mydate", "2021-02-01"); + indexDoc(index, "3", "other_field", "value"); refresh(); @@ -1007,12 +1009,12 @@ public void testSortMissingDatesMixedTypes() throws IOException { } ensureGreen(); - prepareIndex("test_date").setId("1").setSource("mydate", "2021-01-01").get(); - prepareIndex("test_date").setId("2").setSource("mydate", "2021-02-01").get(); - prepareIndex("test_date").setId("3").setSource("other_field", 1).get(); - prepareIndex("test_date_nanos").setId("4").setSource("mydate", "2021-03-01").get(); - prepareIndex("test_date_nanos").setId("5").setSource("mydate", "2021-04-01").get(); - prepareIndex("test_date_nanos").setId("6").setSource("other_field", 2).get(); + indexDoc("test_date", "1", "mydate", "2021-01-01"); + indexDoc("test_date", "2", "mydate", "2021-02-01"); + indexDoc("test_date", "3", "other_field", 1); + indexDoc("test_date_nanos", "4", "mydate", "2021-03-01"); + indexDoc("test_date_nanos", "5", "mydate", "2021-04-01"); + indexDoc("test_date_nanos", "6", "other_field", 2); refresh(); for (boolean withFormat : List.of(true, false)) { @@ -1065,9 +1067,7 @@ private void assertHitsInOrder(SearchResponse response, String[] expectedIds) { public void testIgnoreUnmapped() throws Exception { createIndex("test"); - prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("id", "1").field("i_value", -1).field("d_value", -1.1).endObject()) - .get(); + index("test", "1", jsonBuilder().startObject().field("id", "1").field("i_value", -1).field("d_value", -1.1).endObject()); logger.info("--> sort with an unmapped field, verify it fails"); try { @@ -1140,45 +1140,45 @@ public void testSortMVField() throws Exception { ); ensureGreen(); - prepareIndex("test").setId(Integer.toString(1)) - .setSource( - jsonBuilder().startObject() - .array("long_values", 1L, 5L, 10L, 8L) - .array("int_values", 1, 5, 10, 8) - .array("short_values", 1, 5, 10, 8) - .array("byte_values", 1, 5, 10, 8) - .array("float_values", 1f, 5f, 10f, 8f) - .array("double_values", 1d, 5d, 10d, 8d) - .array("string_values", "01", "05", "10", "08") - .endObject() - ) - .get(); - prepareIndex("test").setId(Integer.toString(2)) - .setSource( - jsonBuilder().startObject() - .array("long_values", 11L, 15L, 20L, 7L) - .array("int_values", 11, 15, 20, 7) - .array("short_values", 11, 15, 20, 7) - .array("byte_values", 11, 15, 20, 7) - .array("float_values", 11f, 15f, 20f, 7f) - .array("double_values", 11d, 15d, 20d, 7d) - .array("string_values", "11", "15", "20", "07") - .endObject() - ) - .get(); - prepareIndex("test").setId(Integer.toString(3)) - .setSource( - jsonBuilder().startObject() - .array("long_values", 2L, 1L, 3L, -4L) - .array("int_values", 2, 1, 3, -4) - .array("short_values", 2, 1, 3, -4) - .array("byte_values", 2, 1, 3, -4) - .array("float_values", 2f, 1f, 3f, -4f) - .array("double_values", 2d, 1d, 3d, -4d) - .array("string_values", "02", "01", "03", "!4") - .endObject() - ) - .get(); + index( + "test", + Integer.toString(1), + jsonBuilder().startObject() + .array("long_values", 1L, 5L, 10L, 8L) + .array("int_values", 1, 5, 10, 8) + .array("short_values", 1, 5, 10, 8) + .array("byte_values", 1, 5, 10, 8) + .array("float_values", 1f, 5f, 10f, 8f) + .array("double_values", 1d, 5d, 10d, 8d) + .array("string_values", "01", "05", "10", "08") + .endObject() + ); + index( + "test", + Integer.toString(2), + jsonBuilder().startObject() + .array("long_values", 11L, 15L, 20L, 7L) + .array("int_values", 11, 15, 20, 7) + .array("short_values", 11, 15, 20, 7) + .array("byte_values", 11, 15, 20, 7) + .array("float_values", 11f, 15f, 20f, 7f) + .array("double_values", 11d, 15d, 20d, 7d) + .array("string_values", "11", "15", "20", "07") + .endObject() + ); + index( + "test", + Integer.toString(3), + jsonBuilder().startObject() + .array("long_values", 2L, 1L, 3L, -4L) + .array("int_values", 2, 1, 3, -4) + .array("short_values", 2, 1, 3, -4) + .array("byte_values", 2, 1, 3, -4) + .array("float_values", 2f, 1f, 3f, -4f) + .array("double_values", 2d, 1d, 3d, -4d) + .array("string_values", "02", "01", "03", "!4") + .endObject() + ); refresh(); @@ -1437,9 +1437,7 @@ public void testSortOnRareField() throws IOException { ) ); ensureGreen(); - prepareIndex("test").setId(Integer.toString(1)) - .setSource(jsonBuilder().startObject().array("string_values", "01", "05", "10", "08").endObject()) - .get(); + index("test", Integer.toString(1), jsonBuilder().startObject().array("string_values", "01", "05", "10", "08").endObject()); refresh(); assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(3).addSort("string_values", SortOrder.DESC), response -> { @@ -1448,13 +1446,9 @@ public void testSortOnRareField() throws IOException { assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(1))); assertThat(response.getHits().getAt(0).getSortValues()[0], equalTo("10")); }); - prepareIndex("test").setId(Integer.toString(2)) - .setSource(jsonBuilder().startObject().array("string_values", "11", "15", "20", "07").endObject()) - .get(); + index("test", Integer.toString(2), jsonBuilder().startObject().array("string_values", "11", "15", "20", "07").endObject()); for (int i = 0; i < 15; i++) { - prepareIndex("test").setId(Integer.toString(300 + i)) - .setSource(jsonBuilder().startObject().array("some_other_field", "foobar").endObject()) - .get(); + index("test", Integer.toString(300 + i), jsonBuilder().startObject().array("some_other_field", "foobar").endObject()); } refresh(); @@ -1468,13 +1462,9 @@ public void testSortOnRareField() throws IOException { assertThat(response.getHits().getAt(1).getId(), equalTo(Integer.toString(1))); assertThat(response.getHits().getAt(1).getSortValues()[0], equalTo("10")); }); - prepareIndex("test").setId(Integer.toString(3)) - .setSource(jsonBuilder().startObject().array("string_values", "02", "01", "03", "!4").endObject()) - .get(); + index("test", Integer.toString(3), jsonBuilder().startObject().array("string_values", "02", "01", "03", "!4").endObject()); for (int i = 0; i < 15; i++) { - prepareIndex("test").setId(Integer.toString(300 + i)) - .setSource(jsonBuilder().startObject().array("some_other_field", "foobar").endObject()) - .get(); + index("test", Integer.toString(300 + i), jsonBuilder().startObject().array("some_other_field", "foobar").endObject()); } refresh(); @@ -1492,9 +1482,7 @@ public void testSortOnRareField() throws IOException { assertThat(response.getHits().getAt(2).getSortValues()[0], equalTo("03")); }); for (int i = 0; i < 15; i++) { - prepareIndex("test").setId(Integer.toString(300 + i)) - .setSource(jsonBuilder().startObject().array("some_other_field", "foobar").endObject()) - .get(); + index("test", Integer.toString(300 + i), jsonBuilder().startObject().array("some_other_field", "foobar").endObject()); refresh(); } @@ -1524,6 +1512,9 @@ public void testSortMetaField() throws Exception { indexReqs[i] = prepareIndex("test").setId(Integer.toString(i)).setSource(); } indexRandom(true, indexReqs); + for (IndexRequestBuilder builder : indexReqs) { + builder.request().decRef(); + } SortOrder order = randomFrom(SortOrder.values()); assertNoFailuresAndResponse( @@ -1592,34 +1583,34 @@ public void testNestedSort() throws IOException, InterruptedException, Execution ); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .startArray("nested") - .startObject() - .field("foo", "bar bar") - .endObject() - .startObject() - .field("foo", "abc abc") - .endObject() - .endArray() - .endObject() - ) - .get(); - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .startArray("nested") - .startObject() - .field("foo", "abc abc") - .endObject() - .startObject() - .field("foo", "cba bca") - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .startArray("nested") + .startObject() + .field("foo", "bar bar") + .endObject() + .startObject() + .field("foo", "abc abc") + .endObject() + .endArray() + .endObject() + ); + index( + "test", + "2", + jsonBuilder().startObject() + .startArray("nested") + .startObject() + .field("foo", "abc abc") + .endObject() + .startObject() + .field("foo", "cba bca") + .endObject() + .endArray() + .endObject() + ); refresh(); // We sort on nested field @@ -1707,6 +1698,9 @@ public void testSortDuelBetweenSingleShardAndMultiShardIndex() throws Exception docs.add(prepareIndex(index).setId(Integer.toString(i)).setSource(sortField, i)); } indexRandom(true, docs); + for (IndexRequestBuilder builder : docs) { + builder.request().decRef(); + } } ensureSearchable("test1", "test2"); @@ -1740,11 +1734,11 @@ public void testCustomFormat() throws Exception { // representations of values, to make sure values are both correctly // rendered and parsed (search_after) assertAcked(prepareCreate("test").setMapping("ip", "type=ip")); - indexRandom( - true, - prepareIndex("test").setId("1").setSource("ip", "192.168.1.7"), - prepareIndex("test").setId("2").setSource("ip", "2001:db8::ff00:42:8329") - ); + IndexRequestBuilder builder1 = prepareIndex("test").setId("1").setSource("ip", "192.168.1.7"); + IndexRequestBuilder builder2 = prepareIndex("test").setId("2").setSource("ip", "2001:db8::ff00:42:8329"); + indexRandom(true, builder1, builder2); + builder1.request().decRef(); + builder2.request().decRef(); assertNoFailuresAndResponse(prepareSearch("test").addSort(SortBuilders.fieldSort("ip")), response -> { assertEquals(2, response.getHits().getTotalHits().value); @@ -1773,6 +1767,9 @@ public void testScriptFieldSort() throws Exception { } Collections.sort(keywords); indexRandom(true, indexReqs); + for (IndexRequestBuilder builder : indexReqs) { + builder.request().decRef(); + } { Script script = new Script(ScriptType.INLINE, NAME, "doc['number'].value", Collections.emptyMap()); @@ -1823,6 +1820,9 @@ public void testFieldAlias() throws Exception { builders.add(prepareIndex("old_index").setSource("distance", 50.5)); builders.add(prepareIndex("new_index").setSource("route_length_miles", 100.2)); indexRandom(true, true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertResponse( prepareSearch().setQuery(matchAllQuery()).setSize(builders.size()).addSort(SortBuilders.fieldSort("route_length_miles")), @@ -1849,6 +1849,9 @@ public void testFieldAliasesWithMissingValues() throws Exception { builders.add(prepareIndex("old_index").setSource(Collections.emptyMap())); builders.add(prepareIndex("new_index").setSource("route_length_miles", 100.2)); indexRandom(true, true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertResponse( prepareSearch().setQuery(matchAllQuery()) @@ -1876,6 +1879,9 @@ public void testCastNumericType() throws Exception { builders.add(prepareIndex("index_long").setSource("field", 12)); builders.add(prepareIndex("index_float").setSource("field", 12.1)); indexRandom(true, true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } { assertResponse( @@ -1924,6 +1930,9 @@ public void testCastDate() throws Exception { builders.add(prepareIndex("index_date").setSource("field", "2024-04-11T23:47:17")); builders.add(prepareIndex("index_date_nanos").setSource("field", "2024-04-11T23:47:16.854775807Z")); indexRandom(true, true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } { assertResponse( @@ -2010,6 +2019,9 @@ public void testCastDate() throws Exception { builders.clear(); builders.add(prepareIndex("index_date").setSource("field", "1905-04-11T23:47:17")); indexRandom(true, true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertResponse( prepareSearch().setQuery(matchAllQuery()).setSize(1).addSort(SortBuilders.fieldSort("field").setNumericType("date_nanos")), response -> { @@ -2024,6 +2036,9 @@ public void testCastDate() throws Exception { builders.clear(); builders.add(prepareIndex("index_date").setSource("field", "2346-04-11T23:47:17")); indexRandom(true, true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertResponse( prepareSearch().setQuery(QueryBuilders.rangeQuery("field").gt("1970-01-01")) .setSize(10) @@ -2069,7 +2084,10 @@ public void testLongSortOptimizationCorrectResults() { toClose.add(bulkBuilder); } String source = "{\"long_field\":" + randomLong() + "}"; - bulkBuilder.add(prepareIndex("test1").setId(Integer.toString(i)).setSource(source, XContentType.JSON)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test1").setId(Integer.toString(i)) + .setSource(source, XContentType.JSON); + bulkBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } } finally { for (BulkRequestBuilder bulkRequestBuilder : toClose) { @@ -2115,10 +2133,10 @@ public void testSortMixedFieldTypes() { assertAcked(prepareCreate("index_double").setMapping("foo", "type=double").get()); assertAcked(prepareCreate("index_keyword").setMapping("foo", "type=keyword").get()); - prepareIndex("index_long").setId("1").setSource("foo", "123").get(); - prepareIndex("index_integer").setId("1").setSource("foo", "123").get(); - prepareIndex("index_double").setId("1").setSource("foo", "123").get(); - prepareIndex("index_keyword").setId("1").setSource("foo", "123").get(); + indexDoc("index_long", "1", "foo", "123"); + indexDoc("index_integer", "1", "foo", "123"); + indexDoc("index_double", "1", "foo", "123"); + indexDoc("index_keyword", "1", "foo", "123"); refresh(); { // mixing long and integer types is ok, as we convert integer sort to long sort @@ -2149,9 +2167,9 @@ public void testSortMixedFieldTypesWithNoDocsForOneType() { assertAcked(prepareCreate("index_other").setMapping("bar", "type=keyword").get()); assertAcked(prepareCreate("index_double").setMapping("foo", "type=double").get()); - prepareIndex("index_long").setId("1").setSource("foo", "123").get(); - prepareIndex("index_long").setId("2").setSource("foo", "124").get(); - prepareIndex("index_other").setId("1").setSource("bar", "124").get(); + indexDoc("index_long", "1", "foo", "123"); + indexDoc("index_long", "2", "foo", "124"); + indexDoc("index_other", "1", "bar", "124"); refresh(); assertNoFailures( @@ -2159,4 +2177,17 @@ public void testSortMixedFieldTypesWithNoDocsForOneType() { .setSize(10) ); } + + private void indexDoc(String indexName, String id, String source, XContentType xContentType) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName); + try { + if (id != null) { + indexRequestBuilder.setId(id); + } + indexRequestBuilder.setSource(source, xContentType); + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java index 2d0fbb42a42e2..741cf098868ec 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.sort; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.settings.Settings; @@ -21,6 +22,7 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; +import java.util.List; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -56,89 +58,89 @@ public void testDistanceSortingMVFields() throws Exception { assertAcked(prepareCreate("test").setSettings(settings).setMapping(xContentBuilder)); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .field("names", "New York") - .startObject("locations") - .field("lat", 40.7143528) - .field("lon", -74.0059731) - .endObject() - .endObject() - ) - .get(); - - prepareIndex("test").setId("2") - .setSource( - jsonBuilder().startObject() - .field("names", "New York 2") - .startObject("locations") - .field("lat", 400.7143528) - .field("lon", 285.9990269) - .endObject() - .endObject() - ) - .get(); - - prepareIndex("test").setId("3") - .setSource( - jsonBuilder().startObject() - .array("names", "Times Square", "Tribeca") - .startArray("locations") - // to NY: 5.286 km - .startObject() - .field("lat", 40.759011) - .field("lon", -73.9844722) - .endObject() - // to NY: 0.4621 km - .startObject() - .field("lat", 40.718266) - .field("lon", -74.007819) - .endObject() - .endArray() - .endObject() - ) - .get(); - - prepareIndex("test").setId("4") - .setSource( - jsonBuilder().startObject() - .array("names", "Wall Street", "Soho") - .startArray("locations") - // to NY: 1.055 km - .startObject() - .field("lat", 40.7051157) - .field("lon", -74.0088305) - .endObject() - // to NY: 1.258 km - .startObject() - .field("lat", 40.7247222) - .field("lon", -74) - .endObject() - .endArray() - .endObject() - ) - .get(); - - prepareIndex("test").setId("5") - .setSource( - jsonBuilder().startObject() - .array("names", "Greenwich Village", "Brooklyn") - .startArray("locations") - // to NY: 2.029 km - .startObject() - .field("lat", 40.731033) - .field("lon", -73.9962255) - .endObject() - // to NY: 8.572 km - .startObject() - .field("lat", 40.65) - .field("lon", -73.95) - .endObject() - .endArray() - .endObject() - ) - .get(); + index( + "test", + "1", + jsonBuilder().startObject() + .field("names", "New York") + .startObject("locations") + .field("lat", 40.7143528) + .field("lon", -74.0059731) + .endObject() + .endObject() + ); + + index( + "test", + "2", + jsonBuilder().startObject() + .field("names", "New York 2") + .startObject("locations") + .field("lat", 400.7143528) + .field("lon", 285.9990269) + .endObject() + .endObject() + ); + + index( + "test", + "3", + jsonBuilder().startObject() + .array("names", "Times Square", "Tribeca") + .startArray("locations") + // to NY: 5.286 km + .startObject() + .field("lat", 40.759011) + .field("lon", -73.9844722) + .endObject() + // to NY: 0.4621 km + .startObject() + .field("lat", 40.718266) + .field("lon", -74.007819) + .endObject() + .endArray() + .endObject() + ); + + index( + "test", + "4", + jsonBuilder().startObject() + .array("names", "Wall Street", "Soho") + .startArray("locations") + // to NY: 1.055 km + .startObject() + .field("lat", 40.7051157) + .field("lon", -74.0088305) + .endObject() + // to NY: 1.258 km + .startObject() + .field("lat", 40.7247222) + .field("lon", -74) + .endObject() + .endArray() + .endObject() + ); + + index( + "test", + "5", + jsonBuilder().startObject() + .array("names", "Greenwich Village", "Brooklyn") + .startArray("locations") + // to NY: 2.029 km + .startObject() + .field("lat", 40.731033) + .field("lon", -73.9962255) + .endObject() + // to NY: 8.572 km + .startObject() + .field("lat", 40.65) + .field("lon", -73.95) + .endObject() + .endArray() + .endObject() + ); indicesAdmin().prepareRefresh().get(); @@ -248,27 +250,27 @@ public void testDistanceSortingWithMissingGeoPoint() throws Exception { assertAcked(prepareCreate("test").setSettings(settings).setMapping(xContentBuilder)); ensureGreen(); - prepareIndex("test").setId("1") - .setSource( - jsonBuilder().startObject() - .array("names", "Times Square", "Tribeca") - .startArray("locations") - // to NY: 5.286 km - .startObject() - .field("lat", 40.759011) - .field("lon", -73.9844722) - .endObject() - // to NY: 0.4621 km - .startObject() - .field("lat", 40.718266) - .field("lon", -74.007819) - .endObject() - .endArray() - .endObject() - ) - .get(); - - prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().array("names", "Wall Street", "Soho").endObject()).get(); + index( + "test", + "1", + jsonBuilder().startObject() + .array("names", "Times Square", "Tribeca") + .startArray("locations") + // to NY: 5.286 km + .startObject() + .field("lat", 40.759011) + .field("lon", -73.9844722) + .endObject() + // to NY: 0.4621 km + .startObject() + .field("lat", 40.718266) + .field("lon", -74.007819) + .endObject() + .endArray() + .endObject() + ); + + index("test", "2", jsonBuilder().startObject().array("names", "Wall Street", "Soho").endObject()); refresh(); @@ -320,8 +322,7 @@ public void testDistanceSortingNestedFields() throws Exception { assertAcked(prepareCreate("companies").setSettings(settings).setMapping(xContentBuilder)); ensureGreen(); - indexRandom( - true, + List builders = List.of( prepareIndex("companies").setId("1") .setSource( jsonBuilder().startObject() @@ -355,8 +356,8 @@ public void testDistanceSortingNestedFields() throws Exception { .field("lat", 40.718266) .field("lon", -74.007819) .endObject() // to NY: - // 0.4621 - // km + // 0.4621 + // km .endObject() .endArray() .endObject() @@ -379,7 +380,7 @@ public void testDistanceSortingNestedFields() throws Exception { .field("lat", 40.7247222) .field("lon", -74) .endObject() // to NY: 1.258 - // km + // km .endObject() .endArray() .endObject() @@ -395,8 +396,8 @@ public void testDistanceSortingNestedFields() throws Exception { .field("lat", 40.731033) .field("lon", -73.9962255) .endObject() // to NY: - // 2.029 - // km + // 2.029 + // km .endObject() .startObject() .field("name", "Brooklyn") @@ -404,12 +405,16 @@ public void testDistanceSortingNestedFields() throws Exception { .field("lat", 40.65) .field("lon", -73.95) .endObject() // to NY: - // 8.572 km + // 8.572 km .endObject() .endArray() .endObject() ) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // Order: Asc assertResponse( @@ -566,7 +571,9 @@ public void testGeoDistanceFilter() throws IOException { XContentBuilder source = JsonXContent.contentBuilder().startObject().field("pin", Geohash.stringEncode(lon, lat)).endObject(); assertAcked(prepareCreate("locations").setSettings(settings).setMapping(mapping)); - prepareIndex("locations").setId("1").setCreate(true).setSource(source).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("locations").setId("1").setCreate(true).setSource(source); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); client().prepareGet("locations", "1").get(); @@ -589,27 +596,27 @@ public void testDistanceSortingWithUnmappedField() throws Exception { assertAcked(prepareCreate("test2")); ensureGreen(); - prepareIndex("test1").setId("1") - .setSource( - jsonBuilder().startObject() - .array("names", "Times Square", "Tribeca") - .startArray("locations") - // to NY: 5.286 km - .startObject() - .field("lat", 40.759011) - .field("lon", -73.9844722) - .endObject() - // to NY: 0.4621 km - .startObject() - .field("lat", 40.718266) - .field("lon", -74.007819) - .endObject() - .endArray() - .endObject() - ) - .get(); - - prepareIndex("test2").setId("2").setSource(jsonBuilder().startObject().array("names", "Wall Street", "Soho").endObject()).get(); + index( + "test1", + "1", + jsonBuilder().startObject() + .array("names", "Times Square", "Tribeca") + .startArray("locations") + // to NY: 5.286 km + .startObject() + .field("lat", 40.759011) + .field("lon", -73.9844722) + .endObject() + // to NY: 0.4621 km + .startObject() + .field("lat", 40.718266) + .field("lon", -74.007819) + .endObject() + .endArray() + .endObject() + ); + + index("test2", "2", jsonBuilder().startObject().array("names", "Wall Street", "Soho").endObject()); refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java index 67426caf2faab..316ea1db65984 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.sort; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.geo.GeoDistance; @@ -71,7 +72,11 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce logger.info("d1: {}", d1Builder); logger.info("d2: {}", d2Builder); - indexRandom(true, prepareIndex("index").setId("d1").setSource(d1Builder), prepareIndex("index").setId("d2").setSource(d2Builder)); + indexRandomAndDecRefRequests( + true, + prepareIndex("index").setId("d1").setSource(d1Builder), + prepareIndex("index").setId("d2").setSource(d2Builder) + ); GeoPoint[] q = new GeoPoint[2]; if (randomBoolean()) { q[0] = new GeoPoint(2, 1); @@ -163,7 +168,11 @@ public void testSingeToManyAvgMedian() throws ExecutionException, InterruptedExc logger.info("d1: {}", d1Builder); logger.info("d2: {}", d2Builder); - indexRandom(true, prepareIndex("index").setId("d1").setSource(d1Builder), prepareIndex("index").setId("d2").setSource(d2Builder)); + indexRandomAndDecRefRequests( + true, + prepareIndex("index").setId("d1").setSource(d1Builder), + prepareIndex("index").setId("d2").setSource(d2Builder) + ); GeoPoint q = new GeoPoint(0, 0); assertResponse( @@ -234,7 +243,11 @@ public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionExcept GeoPoint[] d2Points = { new GeoPoint(4.5, 1), new GeoPoint(4.75, 2), new GeoPoint(5, 3), new GeoPoint(5.25, 4) }; createShuffeldJSONArray(d2Builder, d2Points); - indexRandom(true, prepareIndex("index").setId("d1").setSource(d1Builder), prepareIndex("index").setId("d2").setSource(d2Builder)); + indexRandomAndDecRefRequests( + true, + prepareIndex("index").setId("d1").setSource(d1Builder), + prepareIndex("index").setId("d2").setSource(d2Builder) + ); List qPoints = Arrays.asList(new GeoPoint(2, 1), new GeoPoint(2, 2), new GeoPoint(2, 3), new GeoPoint(2, 4)); Collections.shuffle(qPoints, random()); @@ -281,7 +294,7 @@ public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionExcept public void testSinglePointGeoDistanceSort() throws ExecutionException, InterruptedException, IOException { assertAcked(prepareCreate("index").setMapping(LOCATION_FIELD, "type=geo_point")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("index").setId("d1") .setSource(jsonBuilder().startObject().startObject(LOCATION_FIELD).field("lat", 1).field("lon", 1).endObject().endObject()), @@ -355,7 +368,7 @@ public void testCrossIndexIgnoreUnmapped() throws Exception { ); assertAcked(prepareCreate("test2")); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test1").setSource("str_field", "bcd", "long_field", 3, "double_field", 0.65), prepareIndex("test2").setSource() @@ -382,4 +395,14 @@ public void testCrossIndexIgnoreUnmapped() throws Exception { new Object[] { Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY } ); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java index c7b934df0394f..622c515890049 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java @@ -165,6 +165,7 @@ public void testSimpleSorts() throws Exception { Collections.shuffle(builders, random); for (IndexRequestBuilder builder : builders) { builder.get(); + builder.request().decRef(); if (random.nextBoolean()) { if (random.nextInt(5) != 0) { refresh(); @@ -239,24 +240,24 @@ public void testSortMinValueScript() throws IOException { ensureGreen(); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId("" + i) - .setSource( - jsonBuilder().startObject() - .field("ord", i) - .array("svalue", new String[] { "" + i, "" + (i + 1), "" + (i + 2) }) - .array("lvalue", new long[] { i, i + 1, i + 2 }) - .array("dvalue", new double[] { i, i + 1, i + 2 }) - .startObject("gvalue") - .field("lat", (double) i + 1) - .field("lon", (double) i) - .endObject() - .endObject() - ) - .get(); + index( + "test", + "" + i, + jsonBuilder().startObject() + .field("ord", i) + .array("svalue", new String[] { "" + i, "" + (i + 1), "" + (i + 2) }) + .array("lvalue", new long[] { i, i + 1, i + 2 }) + .array("dvalue", new double[] { i, i + 1, i + 2 }) + .startObject("gvalue") + .field("lat", (double) i + 1) + .field("lon", (double) i) + .endObject() + .endObject() + ); } for (int i = 10; i < 20; i++) { // add some docs that don't have values in those fields - prepareIndex("test").setId("" + i).setSource(jsonBuilder().startObject().field("ord", i).endObject()).get(); + index("test", "" + i, jsonBuilder().startObject().field("ord", i).endObject()); } indicesAdmin().prepareRefresh("test").get(); @@ -347,11 +348,11 @@ public void testDocumentsWithNullValue() throws Exception { assertAcked(prepareCreate("test").setMapping(mapping)); ensureGreen(); - prepareIndex("test").setSource(jsonBuilder().startObject().field("id", "1").field("svalue", "aaa").endObject()).get(); + index("test", jsonBuilder().startObject().field("id", "1").field("svalue", "aaa").endObject()); - prepareIndex("test").setSource(jsonBuilder().startObject().field("id", "2").nullField("svalue").endObject()).get(); + index("test", jsonBuilder().startObject().field("id", "2").nullField("svalue").endObject()); - prepareIndex("test").setSource(jsonBuilder().startObject().field("id", "3").field("svalue", "bbb").endObject()).get(); + index("test", jsonBuilder().startObject().field("id", "3").field("svalue", "bbb").endObject()); flush(); refresh(); @@ -430,7 +431,7 @@ public void test2920() throws IOException { ); ensureGreen(); for (int i = 0; i < 10; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource(jsonBuilder().startObject().field("value", "" + i).endObject()).get(); + index("test", Integer.toString(i), jsonBuilder().startObject().field("value", "" + i).endObject()); } refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java index bd0e7cef44d3f..98e1640266b89 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.NestedQueryBuilder; @@ -31,7 +32,7 @@ public void testSimple() { assertAcked(prepareCreate("test")); ensureGreen(); - prepareIndex("test").setId("1").setSource("field", "value").get(); + indexDoc("test", "1", "field", "value"); refresh(); assertResponse(prepareSearch("test").storedFields("_none_").setFetchSource(false).setVersion(true), response -> { @@ -50,7 +51,7 @@ public void testSimple() { public void testInnerHits() { assertAcked(prepareCreate("test").setMapping("nested", "type=nested")); ensureGreen(); - prepareIndex("test").setId("1").setSource("field", "value", "nested", Collections.singletonMap("title", "foo")).get(); + indexDoc("test", "1", "field", "value", "nested", Collections.singletonMap("title", "foo")); refresh(); assertResponse( @@ -79,7 +80,9 @@ public void testWithRouting() { assertAcked(prepareCreate("test")); ensureGreen(); - prepareIndex("test").setId("1").setSource("field", "value").setRouting("toto").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field", "value").setRouting("toto"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); refresh(); assertResponse(prepareSearch("test").storedFields("_none_").setFetchSource(false), response -> { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/source/SourceFetchingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/source/SourceFetchingIT.java index 81facfa8116c0..0b5c431d83916 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/source/SourceFetchingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/source/SourceFetchingIT.java @@ -41,7 +41,7 @@ public void testSourceFiltering() { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("field1", "value", "field2", "value2").get(); + indexDoc("test", "1", "field1", "value", "field2", "value2"); refresh(); assertResponse( @@ -78,7 +78,7 @@ public void testSourceWithWildcardFiltering() { createIndex("test"); ensureGreen(); - prepareIndex("test").setId("1").setSource("field", "value").get(); + indexDoc("test", "1", "field", "value"); refresh(); assertResponse(prepareSearch("test").setFetchSource(new String[] { "*.notexisting", "field" }, null), response -> { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java index 160cba19700ac..bca6a3aba68bd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java @@ -55,9 +55,7 @@ public void testFieldUsageStats() throws ExecutionException, InterruptedExceptio LocalDate date = LocalDate.of(2015, 9, 1); for (int i = 0; i < 30; i++) { - prepareIndex("test").setId(Integer.toString(i)) - .setSource("field", "value", "field2", "value2", "date_field", formatter.format(date.plusDays(i))) - .get(); + indexDoc("test", Integer.toString(i), "field", "value", "field2", "value2", "date_field", formatter.format(date.plusDays(i))); } indicesAdmin().prepareRefresh("test").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java index 23384d1b199f9..7c3036a30dff7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; @@ -83,7 +84,7 @@ public void testSimpleStats() throws Exception { assertAcked(prepareCreate("test1").setSettings(indexSettings(shardsIdx1, 0))); int docsTest1 = scaledRandomIntBetween(3 * shardsIdx1, 5 * shardsIdx1); for (int i = 0; i < docsTest1; i++) { - prepareIndex("test1").setId(Integer.toString(i)).setSource("field", "value").get(); + indexDoc("test1", Integer.toString(i), "field", "value"); if (rarely()) { refresh(); } @@ -91,7 +92,7 @@ public void testSimpleStats() throws Exception { assertAcked(prepareCreate("test2").setSettings(indexSettings(shardsIdx2, 0))); int docsTest2 = scaledRandomIntBetween(3 * shardsIdx2, 5 * shardsIdx2); for (int i = 0; i < docsTest2; i++) { - prepareIndex("test2").setId(Integer.toString(i)).setSource("field", "value").get(); + indexDoc("test2", Integer.toString(i), "field", "value"); if (rarely()) { refresh(); } @@ -181,7 +182,11 @@ public void testOpenContexts() { final int docs = scaledRandomIntBetween(20, 50); for (int s = 0; s < numAssignedShards(index); s++) { for (int i = 0; i < docs; i++) { - prepareIndex(index).setId(Integer.toString(s * docs + i)).setSource("field", "value").setRouting(Integer.toString(s)).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(Integer.toString(s * docs + i)) + .setSource("field", "value") + .setRouting(Integer.toString(s)); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } } indicesAdmin().prepareRefresh(index).get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java index 20c5c11f36756..9881faf831b2c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java @@ -12,6 +12,7 @@ import org.apache.lucene.analysis.TokenStreamToAutomaton; import org.apache.lucene.search.suggest.document.ContextSuggestField; import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; @@ -103,7 +104,7 @@ public void testTieBreak() throws Exception { ); } Arrays.sort(entries); - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); for (int i = 1; i < numDocs; i++) { CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("a").size(i); String[] topEntries = Arrays.copyOfRange(entries, 0, i); @@ -129,7 +130,7 @@ public void testPrefix() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg"); assertSuggestions("foo", prefix, "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6"); } @@ -156,7 +157,7 @@ public void testTextAndGlobalText() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder noText = SuggestBuilders.completionSuggestion(FIELD); assertResponse( prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", noText).setGlobalText("sugg")), @@ -194,7 +195,7 @@ public void testRegex() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).regex("sugg.*es"); assertSuggestions("foo", prefix, "sugg10estion", "sugg9estion", "sugg8estion", "sugg7estion", "sugg6estion"); } @@ -217,7 +218,7 @@ public void testFuzzy() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg", Fuzziness.ONE); assertSuggestions("foo", prefix, "sugxgestion10", "sugxgestion9", "sugxgestion8", "sugxgestion7", "sugxgestion6"); } @@ -240,7 +241,7 @@ public void testEarlyTermination() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); int size = randomIntBetween(3, 10); String[] outputs = new String[size]; for (int i = 0; i < size; i++) { @@ -274,7 +275,7 @@ public void testSuggestDocument() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg").size(numDocs); assertResponse(prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)), response -> { @@ -310,7 +311,7 @@ public void testSuggestDocumentNoSource() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg").size(numDocs); assertResponse(prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)).setFetchSource(false), response -> { @@ -348,7 +349,7 @@ public void testSuggestDocumentSourceFiltering() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg").size(numDocs); assertResponse( @@ -401,18 +402,18 @@ public void testThatWeightsAreWorking() throws Exception { List similarNames = Arrays.asList("the", "The Prodigy", "The Verve", "The the"); // the weight is 1000 divided by string length, so the results are easy to to check for (String similarName : similarNames) { - prepareIndex(INDEX).setId(similarName) - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value(similarName) - .endArray() - .field("weight", 1000 / similarName.length()) - .endObject() - .endObject() - ) - .get(); + index( + INDEX, + similarName, + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value(similarName) + .endArray() + .field("weight", 1000 / similarName.length()) + .endObject() + .endObject() + ); } refresh(); @@ -425,36 +426,37 @@ public void testThatWeightMustBeAnInteger() throws Exception { Exception e = expectThrows( DocumentParsingException.class, - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value("sth") - .endArray() - .field("weight", 2.5) - .endObject() - .endObject() - ) - ); - assertThat(e.getCause().getMessage(), equalTo("weight must be an integer, but was [2.5]")); - } - - public void testThatWeightCanBeAString() throws Exception { - createIndexAndMapping(completionMappingBuilder); - - prepareIndex(INDEX).setId("1") - .setSource( + () -> index( + INDEX, + "1", jsonBuilder().startObject() .startObject(FIELD) .startArray("input") - .value("testing") + .value("sth") .endArray() - .field("weight", "10") + .field("weight", 2.5) .endObject() .endObject() ) - .get(); + ); + assertThat(e.getCause().getMessage(), equalTo("weight must be an integer, but was [2.5]")); + } + + public void testThatWeightCanBeAString() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + index( + INDEX, + "1", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value("testing") + .endArray() + .field("weight", "10") + .endObject() + .endObject() + ); refresh(); @@ -484,17 +486,18 @@ public void testThatWeightMustNotBeANonNumberString() throws Exception { Exception e = expectThrows( DocumentParsingException.class, - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value("sth") - .endArray() - .field("weight", "thisIsNotValid") - .endObject() - .endObject() - ) + () -> index( + INDEX, + "1", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value("sth") + .endArray() + .field("weight", "thisIsNotValid") + .endObject() + .endObject() + ) ); assertThat(e.getCause().toString(), containsString("thisIsNotValid")); } @@ -506,17 +509,18 @@ public void testThatWeightAsStringMustBeInt() throws Exception { Exception e = expectThrows( DocumentParsingException.class, - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value("testing") - .endArray() - .field("weight", weight) - .endObject() - .endObject() - ) + () -> index( + INDEX, + "1", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value("testing") + .endArray() + .field("weight", weight) + .endObject() + .endObject() + ) ); assertThat(e.getCause().toString(), containsString(weight)); } @@ -524,9 +528,7 @@ public void testThatWeightAsStringMustBeInt() throws Exception { public void testThatInputCanBeAStringInsteadOfAnArray() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource(jsonBuilder().startObject().startObject(FIELD).field("input", "Foo Fighters").endObject().endObject()) - .get(); + index(INDEX, "1", jsonBuilder().startObject().startObject(FIELD).field("input", "Foo Fighters").endObject().endObject()); refresh(); @@ -537,31 +539,31 @@ public void testDisabledPreserveSeparators() throws Exception { completionMappingBuilder.preserveSeparators(false); createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value("Foo Fighters") - .endArray() - .field("weight", 10) - .endObject() - .endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value("Foo Fighters") + .endArray() + .field("weight", 10) + .endObject() + .endObject() + ); - prepareIndex(INDEX).setId("2") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value("Foof") - .endArray() - .field("weight", 20) - .endObject() - .endObject() - ) - .get(); + index( + INDEX, + "2", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value("Foof") + .endArray() + .field("weight", 20) + .endObject() + .endObject() + ); refresh(); @@ -572,15 +574,17 @@ public void testEnabledPreserveSeparators() throws Exception { completionMappingBuilder.preserveSeparators(true); createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Foo Fighters").endArray().endObject().endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Foo Fighters").endArray().endObject().endObject() + ); - prepareIndex(INDEX).setId("2") - .setSource(jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Foof").endArray().endObject().endObject()) - .get(); + index( + INDEX, + "2", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Foof").endArray().endObject().endObject() + ); refresh(); @@ -590,18 +594,18 @@ public void testEnabledPreserveSeparators() throws Exception { public void testThatMultipleInputsAreSupported() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value("Foo Fighters") - .value("Fu Fighters") - .endArray() - .endObject() - .endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value("Foo Fighters") + .value("Fu Fighters") + .endArray() + .endObject() + .endObject() + ); refresh(); @@ -612,11 +616,11 @@ public void testThatMultipleInputsAreSupported() throws Exception { public void testThatShortSyntaxIsWorking() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startArray(FIELD).value("The Prodigy Firestarter").value("Firestarter").endArray().endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startArray(FIELD).value("The Prodigy Firestarter").value("Firestarter").endArray().endObject() + ); refresh(); @@ -629,12 +633,11 @@ public void testThatDisablingPositionIncrementsWorkForStopwords() throws Excepti completionMappingBuilder.searchAnalyzer("classic").indexAnalyzer("classic").preservePositionIncrements(false); createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("The Beatles").endArray().endObject().endObject() - ) - .get(); - + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("The Beatles").endArray().endObject().endObject() + ); refresh(); assertSuggestions("b", "The Beatles"); @@ -651,10 +654,7 @@ public void testThatUpgradeToMultiFieldsWorks() throws Exception { .endObject() .endObject(); assertAcked(prepareCreate(INDEX).setMapping(mapping)); - prepareIndex(INDEX).setId("1") - .setRefreshPolicy(IMMEDIATE) - .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()) - .get(); + indexImmediate(INDEX, "1", jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()); ensureGreen(INDEX); AcknowledgedResponse putMappingResponse = indicesAdmin().preparePutMapping(INDEX) @@ -685,10 +685,7 @@ public void testThatUpgradeToMultiFieldsWorks() throws Exception { response -> assertSuggestions(response, "suggs") ); - prepareIndex(INDEX).setId("1") - .setRefreshPolicy(IMMEDIATE) - .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()) - .get(); + indexImmediate(INDEX, "1", jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()); ensureGreen(INDEX); assertResponse( @@ -702,11 +699,11 @@ public void testThatUpgradeToMultiFieldsWorks() throws Exception { public void testThatFuzzySuggesterWorks() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() + ); refresh(); @@ -731,11 +728,11 @@ public void testThatFuzzySuggesterWorks() throws Exception { public void testThatFuzzySuggesterSupportsEditDistances() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() + ); refresh(); @@ -765,11 +762,11 @@ public void testThatFuzzySuggesterSupportsEditDistances() throws Exception { public void testThatFuzzySuggesterSupportsTranspositions() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() + ); refresh(); @@ -799,11 +796,11 @@ public void testThatFuzzySuggesterSupportsTranspositions() throws Exception { public void testThatFuzzySuggesterSupportsMinPrefixLength() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() + ); refresh(); @@ -835,11 +832,11 @@ public void testThatFuzzySuggesterSupportsMinPrefixLength() throws Exception { public void testThatFuzzySuggesterSupportsNonPrefixLength() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() + ); refresh(); @@ -871,9 +868,11 @@ public void testThatFuzzySuggesterSupportsNonPrefixLength() throws Exception { public void testThatFuzzySuggesterIsUnicodeAware() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource(jsonBuilder().startObject().startObject(FIELD).startArray("input").value("ööööö").endArray().endObject().endObject()) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("ööööö").endArray().endObject().endObject() + ); refresh(); @@ -932,12 +931,8 @@ public void testThatStatsAreWorking() throws Exception { assertThat(putMappingResponse.isAcknowledged(), is(true)); // Index two entities - prepareIndex(INDEX).setId("1") - .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject()) - .get(); - prepareIndex(INDEX).setId("2") - .setSource(jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject()) - .get(); + index(INDEX, "1", jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject()); + index(INDEX, "2", jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject()); refresh(); ensureGreen(); @@ -984,11 +979,11 @@ public void testThatStatsAreWorking() throws Exception { public void testThatSortingOnCompletionFieldReturnsUsefulException() throws Exception { createIndexAndMapping(completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("Nirvana").endArray().endObject().endObject() + ); refresh(); @@ -1013,32 +1008,32 @@ public void testThatSuggestStopFilterWorks() throws Exception { builder.indexAnalyzer("simple"); createIndexAndMappingAndSettings(settingsBuilder.build(), builder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value("Feed trolls") - .endArray() - .field("weight", 5) - .endObject() - .endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value("Feed trolls") + .endArray() + .field("weight", 5) + .endObject() + .endObject() + ); // Higher weight so it's ranked first: - prepareIndex(INDEX).setId("2") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value("Feed the trolls") - .endArray() - .field("weight", 10) - .endObject() - .endObject() - ) - .get(); + index( + INDEX, + "2", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value("Feed the trolls") + .endArray() + .field("weight", 10) + .endObject() + .endObject() + ); refresh(); @@ -1059,17 +1054,17 @@ public void testThatIndexingInvalidFieldsInCompletionFieldResultsInException() t createIndexAndMapping(builder); try { - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("FRIGGININVALID") - .value("Nirvana") - .endArray() - .endObject() - .endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("FRIGGININVALID") + .value("Nirvana") + .endArray() + .endObject() + .endObject() + ); fail("Expected Exception"); } catch (DocumentParsingException e) { assertThat(e.getMessage(), containsString("failed to parse")); @@ -1101,7 +1096,7 @@ public void testSkipDuplicates() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); Arrays.sort(termIds, Comparator.comparingInt(o -> weights[(int) o]).reversed().thenComparingInt(a -> (int) a)); String[] expected = new String[numUnique]; @@ -1260,18 +1255,18 @@ private void createIndexAndMapping(CompletionMappingBuilder builder) throws IOEx public void testPrunedSegments() throws IOException { createIndexAndMappingAndSettings(indexSettings(1, 0).build(), completionMappingBuilder); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("The Beatles").endArray().endObject().endObject() - ) - .get(); + index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value("The Beatles").endArray().endObject().endObject() + ); // we have 2 docs in a segment... - prepareIndex(INDEX).setId("2").setSource(jsonBuilder().startObject().field("somefield", "somevalue").endObject()).get(); + index(INDEX, "2", jsonBuilder().startObject().field("somefield", "somevalue").endObject()); BroadcastResponse actionGet = indicesAdmin().prepareForceMerge().setFlush(true).setMaxNumSegments(1).get(); assertAllSuccessful(actionGet); refresh(); // update the first one and then merge.. the target segment will have no value in FIELD - prepareIndex(INDEX).setId("1").setSource(jsonBuilder().startObject().field("somefield", "somevalue").endObject()).get(); + index(INDEX, "1", jsonBuilder().startObject().field("somefield", "somevalue").endObject()); actionGet = indicesAdmin().prepareForceMerge().setFlush(true).setMaxNumSegments(1).get(); assertAllSuccessful(actionGet); refresh(); @@ -1304,12 +1299,11 @@ public void testVeryLongInput() throws IOException { ); // can cause stack overflow without the default max_input_length String longString = replaceReservedChars(randomRealisticUnicodeOfLength(randomIntBetween(5000, 10000)), (char) 0x01); - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value(longString).endArray().endObject().endObject() - ) - .setRefreshPolicy(IMMEDIATE) - .get(); + indexImmediate( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).startArray("input").value(longString).endArray().endObject().endObject() + ); } @@ -1333,17 +1327,18 @@ public void testReservedChars() throws IOException { String string = "foo" + (char) 0x00 + "bar"; Exception e = expectThrows( DocumentParsingException.class, - prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .startArray("input") - .value(string) - .endArray() - .field("output", "foobar") - .endObject() - .endObject() - ) + () -> index( + INDEX, + "1", + jsonBuilder().startObject() + .startObject(FIELD) + .startArray("input") + .value(string) + .endArray() + .field("output", "foobar") + .endObject() + .endObject() + ) ); assertThat(e.getMessage(), containsString("failed to parse")); } @@ -1365,10 +1360,7 @@ public void testIssue5930() throws IOException { ) ); String string = "foo bar"; - prepareIndex(INDEX).setId("1") - .setSource(jsonBuilder().startObject().field(FIELD, string).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + index(INDEX, "1", jsonBuilder().startObject().field(FIELD, string).endObject()); SearchPhaseExecutionException e = expectThrows( SearchPhaseExecutionException.class, @@ -1397,7 +1389,7 @@ public void testMultiDocSuggestions() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg").shardSize(15); assertSuggestions("foo", prefix, "suggester10", "suggester9", "suggester8", "suggester7", "suggester6"); } @@ -1423,7 +1415,7 @@ public void testSuggestWithFieldAlias() throws Exception { builders.add(prepareIndex(INDEX).setSource(FIELD, "apple")); builders.add(prepareIndex(INDEX).setSource(FIELD, "mango")); builders.add(prepareIndex(INDEX).setSource(FIELD, "papaya")); - indexRandom(true, false, builders); + indexRandomAndDecRefRequests(true, false, builders); CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("alias").text("app"); assertSuggestions("suggestion", suggestionBuilder, "apple"); @@ -1447,7 +1439,7 @@ public void testSuggestOnlyExplain() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg"); assertResponse( prepareSearch(INDEX).setExplain(true).suggest(new SuggestBuilder().addSuggestion("foo", prefix)), @@ -1479,7 +1471,7 @@ public void testCompletionWithCollapse() throws Exception { XContentBuilder builder = jsonBuilder().startObject(); builder.startObject(suggestField).field("input", "suggestion" + i).field("weight", i).endObject(); builder.field("collapse_field", "collapse me").endObject(); // all docs the same value for collapsing - prepareIndex(index).setId("" + i).setSource(builder).get(); + index(index, "" + i, builder); } indicesAdmin().prepareRefresh(index).get(); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(suggestField).prefix("sug").size(1); @@ -1558,4 +1550,34 @@ public CompletionMappingBuilder context(LinkedHashMap> return this; } } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, boolean dummyDocuments, List builders) + throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder indexRequestBuilder : builders) { + indexRequestBuilder.request().decRef(); + } + } + } + + private DocWriteResponse indexImmediate(String index, String id, XContentBuilder source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id).setSource(source).setRefreshPolicy(IMMEDIATE); + try { + return indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java index 22d48c2f282d5..b55ed60f90277 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java @@ -79,7 +79,7 @@ public void testContextPrefix() throws Exception { source.endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD) .prefix("sugg") .contexts( @@ -115,7 +115,7 @@ public void testContextRegex() throws Exception { source.endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD) .regex("sugg.*es") .contexts( @@ -151,7 +151,7 @@ public void testContextFuzzy() throws Exception { source.endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD) .prefix("sugg", Fuzziness.ONE) .contexts( @@ -168,16 +168,11 @@ public void testContextFilteringWorksWithUTF8Categories() throws Exception { LinkedHashMap> map = new LinkedHashMap<>(Collections.singletonMap("cat", contextMapping)); final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); createIndexAndMapping(mapping); - DocWriteResponse indexResponse = prepareIndex(INDEX).setId("1") - .setSource( - jsonBuilder().startObject() - .startObject(FIELD) - .field("input", "suggestion") - .endObject() - .field("cat", "ctx\\u00e4") - .endObject() - ) - .get(); + DocWriteResponse indexResponse = index( + INDEX, + "1", + jsonBuilder().startObject().startObject(FIELD).field("input", "suggestion").endObject().field("cat", "ctx\\u00e4").endObject() + ); assertThat(indexResponse.status(), equalTo(RestStatus.CREATED)); assertNoFailures(indicesAdmin().prepareRefresh(INDEX).get()); CompletionSuggestionBuilder contextSuggestQuery = SuggestBuilders.completionSuggestion(FIELD) @@ -209,7 +204,7 @@ public void testSingleContextFiltering() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD) .prefix("sugg") .contexts( @@ -240,7 +235,7 @@ public void testSingleContextBoosting() throws Exception { ) ); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD) .prefix("sugg") .contexts( @@ -274,7 +269,7 @@ public void testMultiContextFiltering() throws Exception { .endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); // filter only on context cat CompletionSuggestionBuilder catFilterSuggest = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg"); @@ -316,7 +311,7 @@ public void testMultiContextBoosting() throws Exception { .endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); // boost only on context cat CompletionSuggestionBuilder catBoostSuggest = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg"); @@ -389,7 +384,7 @@ public void testSeveralContexts() throws Exception { source.endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD) .prefix("sugg") @@ -422,7 +417,7 @@ public void testGeoFiltering() throws Exception { .endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder geoFilteringPrefix = SuggestBuilders.completionSuggestion(FIELD) .prefix("sugg") @@ -456,7 +451,7 @@ public void testGeoBoosting() throws Exception { .endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); GeoQueryContext context1 = GeoQueryContext.builder().setGeoPoint(geoPoints[0]).setBoost(11).build(); GeoQueryContext context2 = GeoQueryContext.builder().setGeoPoint(geoPoints[1]).build(); @@ -489,7 +484,7 @@ public void testGeoPointContext() throws Exception { .endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD) .prefix("sugg") .contexts( @@ -531,7 +526,7 @@ public void testGeoNeighbours() throws Exception { .endObject(); indexRequestBuilders.add(prepareIndex(INDEX).setId("" + i).setSource(source)); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); CompletionSuggestionBuilder geoNeighbourPrefix = SuggestBuilders.completionSuggestion(FIELD) .prefix("sugg") @@ -592,7 +587,7 @@ public void testGeoField() throws Exception { .array("input", "Hotel Amsterdam in Berlin") .endObject() .endObject(); - prepareIndex(INDEX).setId("1").setSource(source1).get(); + index(INDEX, "1", source1); XContentBuilder source2 = jsonBuilder().startObject() .startObject("location") @@ -602,7 +597,7 @@ public void testGeoField() throws Exception { .array("input", "Hotel Berlin in Amsterdam") .endObject() .endObject(); - prepareIndex(INDEX).setId("2").setSource(source2).get(); + index(INDEX, "2", source2); refresh(); @@ -650,7 +645,7 @@ public void testSkipDuplicatesWithContexts() throws Exception { for (int i = 0; i < numUnique; i++) { expected[i] = "suggestion" + (numUnique - 1 - i); } - indexRandom(true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, indexRequestBuilders); Map> contextMap = new HashMap<>(); contextMap.put("cat", Arrays.asList(CategoryQueryContext.builder().setCategory("cat0").build())); CompletionSuggestionBuilder completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD) @@ -726,4 +721,14 @@ private void createIndexAndMappingAndSettings(Settings settings, CompletionMappi indicesAdmin().prepareCreate(INDEX).setSettings(Settings.builder().put(indexSettings()).put(settings)).setMapping(mapping) ); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + super.indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/SuggestSearchIT.java index a7cf141eb7669..ac0c451bf409f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/SuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/SuggestSearchIT.java @@ -292,7 +292,7 @@ public void testUnmappedField() throws IOException, InterruptedException, Execut assertAcked(builder.setMapping(mapping)); ensureGreen(); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setSource("name", "I like iced tea"), prepareIndex("test").setSource("name", "I like tea."), @@ -778,7 +778,7 @@ public void testSizeParam() throws IOException { public void testDifferentShardSize() throws Exception { createIndex("test"); ensureGreen(); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test").setId("1").setSource("field1", "foobar1").setRouting("1"), prepareIndex("test").setId("2").setSource("field1", "foobar2").setRouting("2"), @@ -1120,7 +1120,7 @@ public void testSuggestWithManyCandidates() throws InterruptedException, Executi builders.add(prepareIndex("test").setSource("title", title)); } - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); PhraseSuggestionBuilder suggest = phraseSuggestion("title").addCandidateGenerator( candidateGenerator("title").suggestMode("always") @@ -1158,7 +1158,7 @@ public void testSuggestWithFieldAlias() throws Exception { builders.add(prepareIndex("test").setSource("text", "apple")); builders.add(prepareIndex("test").setSource("text", "mango")); builders.add(prepareIndex("test").setSource("text", "papaya")); - indexRandom(true, false, builders); + indexRandomAndDecRefRequests(true, false, builders); TermSuggestionBuilder termSuggest = termSuggestion("alias").text("appple"); @@ -1184,7 +1184,7 @@ public void testPhraseSuggestMinDocFreq() throws Exception { builders.add(prepareIndex("test").setSource("text", "apple")); builders.add(prepareIndex("test").setSource("text", "apple")); builders.add(prepareIndex("test").setSource("text", "appfle")); - indexRandom(true, false, builders); + indexRandomAndDecRefRequests(true, false, builders); PhraseSuggestionBuilder phraseSuggest = phraseSuggestion("text").text("appple") .size(2) @@ -1295,7 +1295,7 @@ public void testPhraseSuggesterCollate() throws InterruptedException, ExecutionE for (String title : titles) { builders.add(prepareIndex("test").setSource("title", title)); } - indexRandom(true, builders); + indexRandomAndDecRefRequests(true, builders); // suggest without collate PhraseSuggestionBuilder suggest = phraseSuggestion("title").addCandidateGenerator( @@ -1434,4 +1434,35 @@ protected Suggest searchSuggest(String suggestText, int expectShardsFailed, Map< }); return suggest[0]; } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, List builders) throws InterruptedException { + try { + indexRandom(forceRefresh, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, boolean dummyDocuments, List builders) + throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java b/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java index 55dca7810f845..6c06d2fbc3504 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.similarity; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.test.ESIntegTestCase; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -47,10 +48,11 @@ public void testCustomBM25Similarity() throws Exception { ) .get(); - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource("field1", "the quick brown fox jumped over the lazy dog", "field2", "the quick brown fox jumped over the lazy dog") - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertResponse(prepareSearch().setQuery(matchQuery("field1", "quick brown fox")), bm25SearchResponse -> { assertThat(bm25SearchResponse.getHits().getTotalHits().value, equalTo(1L)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java index 42457a12da5cb..eeea4faf5a9e3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java @@ -65,7 +65,9 @@ public void testIncrementalBehaviorOnPrimaryFailover() throws InterruptedExcepti for (int j = 0; j < randomIntBetween(1, 10); ++j) { try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) { - bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i)); + IndexRequest indexRequest = new IndexRequest(indexName).source("foo" + j, "bar" + i); + bulkRequest.add(indexRequest); + indexRequest.decRef(); } final BulkResponse bulkResponse = client().bulk(bulkRequest).get(); for (BulkItemResponse item : bulkResponse.getItems()) { @@ -144,7 +146,9 @@ public void testForceMergeCausesFullSnapshot() throws Exception { for (int j = 0; j < 2; j++) { try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) { - bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i)); + IndexRequest indexRequest = new IndexRequest(indexName).source("foo" + j, "bar" + i); + bulkRequest.add(indexRequest); + indexRequest.decRef(); } client().bulk(bulkRequest).get(); flushAndRefresh(indexName); @@ -191,10 +195,10 @@ public void testRecordCorrectSegmentCountsWithBackgroundMerges() throws Exceptio final int numDocs = scaledRandomIntBetween(100, 1000); try (BulkRequestBuilder request = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int j = 0; j < numDocs; ++j) { - request.add( - new IndexRequest(indexName).id(Long.toString(id++)) - .source(jsonBuilder().startObject().field("l", randomLong()).endObject()) - ); + IndexRequest indexRequest = new IndexRequest(indexName).id(Long.toString(id++)) + .source(jsonBuilder().startObject().field("l", randomLong()).endObject()); + request.add(indexRequest); + indexRequest.decRef(); } assertNoFailures(request.get()); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java index f507e27c6073e..e30c219092dab 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -65,7 +65,9 @@ public void testRecreateCorruptedRepositoryUnblocksIt() throws Exception { createIndex("test-idx-1"); logger.info("--> indexing some data"); - indexRandom(true, prepareIndex("test-idx-1").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test-idx-1").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); final String snapshot = "test-snap"; @@ -111,7 +113,11 @@ public void testConcurrentlyChangeRepositoryContents() throws Exception { createIndex("test-idx-1", "test-idx-2"); logger.info("--> indexing some data"); - indexRandom(true, prepareIndex("test-idx-1").setSource("foo", "bar"), prepareIndex("test-idx-2").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test-idx-1").setSource("foo", "bar"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test-idx-2").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); final String snapshot = "test-snap"; @@ -179,7 +185,11 @@ public void testFindDanglingLatestGeneration() throws Exception { createIndex("test-idx-1", "test-idx-2"); logger.info("--> indexing some data"); - indexRandom(true, prepareIndex("test-idx-1").setSource("foo", "bar"), prepareIndex("test-idx-2").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test-idx-1").setSource("foo", "bar"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test-idx-2").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); final String snapshot = "test-snap"; @@ -476,6 +486,9 @@ public void testSnapshotWithCorruptedShardIndexFile() throws Exception { documents[j] = client.prepareIndex(indexName).setSource("foo", "bar"); } indexRandom(true, documents); + for (IndexRequestBuilder builder : documents) { + builder.request().decRef(); + } flushAndRefresh(); createRepository("test-repo", "fs", repo); @@ -522,6 +535,9 @@ public void testSnapshotWithCorruptedShardIndexFile() throws Exception { documents[j] = client.prepareIndex(indexName).setSource("foo2", "bar2"); } indexRandom(true, documents); + for (IndexRequestBuilder builder : documents) { + builder.request().decRef(); + } final String snapshot2 = "test-snap-2"; logger.info("--> creating snapshot [{}]", snapshot2); @@ -544,7 +560,11 @@ public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exceptio final String[] indices = { "test-idx-1", "test-idx-2" }; createIndex(indices); logger.info("--> indexing some data"); - indexRandom(true, prepareIndex("test-idx-1").setSource("foo", "bar"), prepareIndex("test-idx-2").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test-idx-1").setSource("foo", "bar"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test-idx-2").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); logger.info("--> creating snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin() @@ -590,7 +610,11 @@ public void testDeleteSnapshotWithMissingMetadata() throws Exception { createIndex("test-idx-1", "test-idx-2"); logger.info("--> indexing some data"); - indexRandom(true, prepareIndex("test-idx-1").setSource("foo", "bar"), prepareIndex("test-idx-2").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test-idx-1").setSource("foo", "bar"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test-idx-2").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); logger.info("--> creating snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin() @@ -630,7 +654,11 @@ public void testDeleteSnapshotWithCorruptedSnapshotFile() throws Exception { createIndex("test-idx-1", "test-idx-2"); logger.info("--> indexing some data"); - indexRandom(true, prepareIndex("test-idx-1").setSource("foo", "bar"), prepareIndex("test-idx-2").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test-idx-1").setSource("foo", "bar"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test-idx-2").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); logger.info("--> creating snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin() @@ -680,12 +708,13 @@ public void testDeleteSnapshotWithCorruptedGlobalState() throws Exception { ); createIndex("test-idx-1", "test-idx-2"); - indexRandom( - true, - prepareIndex("test-idx-1").setSource("foo", "bar"), - prepareIndex("test-idx-2").setSource("foo", "bar"), - prepareIndex("test-idx-2").setSource("foo", "bar") - ); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test-idx-1").setSource("foo", "bar"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test-idx-2").setSource("foo", "bar"); + IndexRequestBuilder indexRequestBuilder3 = prepareIndex("test-idx-2").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); + indexRequestBuilder3.request().decRef(); flushAndRefresh("test-idx-1", "test-idx-2"); SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap"); @@ -726,7 +755,11 @@ public void testSnapshotWithMissingShardLevelIndexFile() throws Exception { createIndex("test-idx-1", "test-idx-2"); logger.info("--> indexing some data"); - indexRandom(true, prepareIndex("test-idx-1").setSource("foo", "bar"), prepareIndex("test-idx-2").setSource("foo", "bar")); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test-idx-1").setSource("foo", "bar"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test-idx-2").setSource("foo", "bar"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); logger.info("--> creating snapshot"); clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 6ca3fccd1e292..867c76e80b206 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -700,7 +700,7 @@ public void testSnapshotTotalAndIncrementalSizes() throws Exception { int docs = between(10, 100); for (int i = 0; i < docs; i++) { - prepareIndex(indexName).setSource("test", "init").get(); + indexDoc(indexName, null, "test", "init"); } final Path repoPath = randomRepoPath(); @@ -733,7 +733,7 @@ public void testSnapshotTotalAndIncrementalSizes() throws Exception { // add few docs - less than initially docs = between(1, 5); for (int i = 0; i < docs; i++) { - prepareIndex(indexName).setSource("test", "test" + i).get(); + indexDoc(indexName, null, "test", "test" + i); } // create another snapshot @@ -785,7 +785,7 @@ public void testDeduplicateIndexMetadata() throws Exception { int docs = between(10, 100); for (int i = 0; i < docs; i++) { - prepareIndex(indexName).setSource("test", "init").get(); + indexDoc(indexName, null, "test", "init"); } final Path repoPath = randomRepoPath(); @@ -797,7 +797,7 @@ public void testDeduplicateIndexMetadata() throws Exception { docs = between(1, 5); for (int i = 0; i < docs; i++) { - prepareIndex(indexName).setSource("test", "test" + i).get(); + indexDoc(indexName, null, "test", "test" + i); } logger.info("--> restart random data node and add new data node to change index allocation"); @@ -814,7 +814,7 @@ public void testDeduplicateIndexMetadata() throws Exception { // index to some other field to trigger a change in index metadata for (int i = 0; i < docs; i++) { - prepareIndex(indexName).setSource("new_field", "test" + i).get(); + indexDoc(indexName, null, "new_field", "test" + i); } createFullSnapshot(repositoryName, snapshot2); @@ -960,6 +960,9 @@ public void testRetentionLeasesClearedOnRestore() throws Exception { indexRequestBuilders[i] = prepareIndex(indexName).setSource("field", "value"); } indexRandom(true, indexRequestBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } assertDocCount(indexName, snapshotDocCount); final String leaseId = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); @@ -984,6 +987,9 @@ public void testRetentionLeasesClearedOnRestore() throws Exception { indexRequestBuilders[i] = prepareIndex(indexName).setSource("field", "value"); } indexRandom(true, indexRequestBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } } // Wait for green so the close does not fail in the edge case of coinciding with a shard recovery that hasn't fully synced yet diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java index 0aa3475de7be1..41c14b1b7c25d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; @@ -31,6 +32,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -55,14 +57,17 @@ protected Collection> nodePlugins() { public void testWhenMetadataAreLoaded() throws Exception { createIndex("docs"); - indexRandom( - true, + List builders = List.of( prepareIndex("docs").setId("1").setSource("rank", 1), prepareIndex("docs").setId("2").setSource("rank", 2), prepareIndex("docs").setId("3").setSource("rank", 3), prepareIndex("others").setSource("rank", 4), prepareIndex("others").setSource("rank", 5) ); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } createRepository("repository", CountingMockRepositoryPlugin.TYPE); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java index 6c452103cc014..fae2c437b427c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java @@ -672,6 +672,9 @@ public void testChangeSettingsOnRestore() throws Exception { builders[i] = prepareIndex("test-idx").setId(Integer.toString(i)).setSource("field1", "Foo bar " + i); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } flushAndRefresh(); assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "foo")), numdocs); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index c13891728f315..012eccdcee4af 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1502,7 +1502,7 @@ public void testListCorruptedSnapshot() throws Exception { createIndex("test-idx-1", "test-idx-2", "test-idx-3"); logger.info("--> indexing some data"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test-idx-1").setSource("foo", "bar"), prepareIndex("test-idx-2").setSource("foo", "bar"), @@ -1545,7 +1545,7 @@ public void testRestoreSnapshotWithCorruptedGlobalState() throws Exception { createRepository(repoName, "fs", repo); createIndex("test-idx-1", "test-idx-2"); - indexRandom( + indexRandomAndDecRefRequests( true, prepareIndex("test-idx-1").setSource("foo", "bar"), prepareIndex("test-idx-2").setSource("foo", "bar"), @@ -1612,7 +1612,7 @@ public void testRestoreSnapshotWithCorruptedIndexMetadata() throws Exception { for (int j = 0; j < nbDocs; j++) { documents[j] = client.prepareIndex(indexName).setSource("foo", "bar"); } - indexRandom(true, documents); + indexRandomAndDecRefRequests(true, documents); } flushAndRefresh(); @@ -2202,4 +2202,14 @@ private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map indexing [{}] docs into [{}]", docCount, indexName); for (int i = 0; i < docCount; i++) { - bulkRequestBuilder.add( - new IndexRequest().source( - jsonBuilder().startObject() - .field("field-" + between(1, 5), randomAlphaOfLength(10)) - .endObject() - ) + IndexRequest indexRequest = new IndexRequest().source( + jsonBuilder().startObject().field("field-" + between(1, 5), randomAlphaOfLength(10)).endObject() ); + bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } bulkRequestBuilder.execute(bulkStep); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java index 704ce8ba990fa..525ba3d08b289 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java @@ -76,6 +76,9 @@ public void testThreadNames() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } int numSearches = randomIntBetween(2, 100); for (int i = 0; i < numSearches; i++) { assertNoFailures(prepareSearch("idx").setQuery(QueryBuilders.termQuery("str_value", "s" + i))); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java index dc512cdb92cc1..105e360cd4020 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.timeseries.support; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; @@ -44,19 +45,18 @@ public void testDimensionFieldNameLimit() throws IOException { () -> List.of("routing_field"), dimensionFieldLimit ); - final Exception ex = expectThrows( - DocumentParsingException.class, - prepareIndex("test").setSource( - "routing_field", - randomAlphaOfLength(10), - dimensionFieldName, - randomAlphaOfLength(1024), - "gauge", - randomIntBetween(10, 20), - "@timestamp", - Instant.now().toEpochMilli() - ) + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource( + "routing_field", + randomAlphaOfLength(10), + dimensionFieldName, + randomAlphaOfLength(1024), + "gauge", + randomIntBetween(10, 20), + "@timestamp", + Instant.now().toEpochMilli() ); + final Exception ex = expectThrows(DocumentParsingException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertThat( ex.getCause().getMessage(), equalTo( @@ -74,19 +74,26 @@ public void testDimensionFieldValueLimit() throws IOException { dimensionFieldLimit ); long startTime = Instant.now().toEpochMilli(); - prepareIndex("test").setSource("field", randomAlphaOfLength(1024), "gauge", randomIntBetween(10, 20), "@timestamp", startTime) - .get(); - final Exception ex = expectThrows( - DocumentParsingException.class, - prepareIndex("test").setSource( - "field", - randomAlphaOfLength(1025), - "gauge", - randomIntBetween(10, 20), - "@timestamp", - startTime + 1 - ) + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test").setSource( + "field", + randomAlphaOfLength(1024), + "gauge", + randomIntBetween(10, 20), + "@timestamp", + startTime + ); + indexRequestBuilder1.get(); + indexRequestBuilder1.request().decRef(); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test").setSource( + "field", + randomAlphaOfLength(1025), + "gauge", + randomIntBetween(10, 20), + "@timestamp", + startTime + 1 ); + final Exception ex = expectThrows(DocumentParsingException.class, indexRequestBuilder2); + indexRequestBuilder2.request().decRef(); assertThat(ex.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1025].")); } @@ -143,7 +150,9 @@ public void testTotalDimensionFieldsSizeLuceneLimit() throws IOException { for (int i = 0; i < dimensionFieldLimit; i++) { source.put(dimensionFieldNames.get(i), randomAlphaOfLength(1024)); } - final DocWriteResponse indexResponse = prepareIndex("test").setSource(source).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource(source); + final DocWriteResponse indexResponse = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertEquals(RestStatus.CREATED.getStatus(), indexResponse.status().getStatus()); } @@ -169,7 +178,9 @@ public void testTotalDimensionFieldsSizeLuceneLimitPlusOne() throws IOException for (int i = 0; i < dimensionFieldLimit; i++) { source.put(dimensionFieldNames.get(i), randomAlphaOfLength(1024)); } - final Exception ex = expectThrows(DocumentParsingException.class, prepareIndex("test").setSource(source)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource(source); + final Exception ex = expectThrows(DocumentParsingException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertEquals("_tsid longer than [32766] bytes [33903].", ex.getCause().getMessage()); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateIT.java index f23b9d391e3c5..5caf14d1ab5d6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; @@ -149,10 +150,11 @@ public void testUpsert() throws Exception { ensureGreen(); Script fieldIncScript = new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, Collections.singletonMap("field", "field")); - UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "1") + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject()) - .setScript(fieldIncScript) - .get(); + .setScript(fieldIncScript); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult()); assertThat(updateResponse.getIndex(), equalTo("test")); @@ -161,10 +163,11 @@ public void testUpsert() throws Exception { assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1")); } - updateResponse = client().prepareUpdate(indexOrAlias(), "1") + updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject()) - .setScript(fieldIncScript) - .get(); + .setScript(fieldIncScript); + updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); assertThat(updateResponse.getIndex(), equalTo("test")); @@ -189,11 +192,12 @@ public void testScriptedUpsert() throws Exception { // Pay money from what will be a new account and opening balance comes from upsert doc // provided by client - UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "1") + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("balance", openingBalance).endObject()) .setScriptedUpsert(true) - .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, UPSERT_SCRIPT, params)) - .get(); + .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, UPSERT_SCRIPT, params)); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult()); assertThat(updateResponse.getIndex(), equalTo("test")); @@ -203,11 +207,12 @@ public void testScriptedUpsert() throws Exception { } // Now pay money for an existing account where balance is stored in es - updateResponse = client().prepareUpdate(indexOrAlias(), "1") + updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("balance", openingBalance).endObject()) .setScriptedUpsert(true) - .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, UPSERT_SCRIPT, params)) - .get(); + .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, UPSERT_SCRIPT, params)); + updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); assertThat(updateResponse.getIndex(), equalTo("test")); @@ -221,11 +226,12 @@ public void testUpsertDoc() throws Exception { createTestIndex(); ensureGreen(); - UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "1") + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setDoc(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject()) .setDocAsUpsert(true) - .setFetchSource(true) - .get(); + .setFetchSource(true); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertThat(updateResponse.getIndex(), equalTo("test")); assertThat(updateResponse.getGetResult(), notNullValue()); assertThat(updateResponse.getGetResult().getIndex(), equalTo("test")); @@ -237,25 +243,27 @@ public void testNotUpsertDoc() throws Exception { createTestIndex(); ensureGreen(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1"); assertFutureThrows( - client().prepareUpdate(indexOrAlias(), "1") - .setDoc(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject()) + updateRequestBuilder.setDoc(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject()) .setDocAsUpsert(false) .setFetchSource(true) .execute(), DocumentMissingException.class ); + updateRequestBuilder.request().decRef(); } public void testUpsertFields() throws Exception { createTestIndex(); ensureGreen(); - UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "1") + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject()) .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, PUT_VALUES_SCRIPT, Collections.singletonMap("extra", "foo"))) - .setFetchSource(true) - .get(); + .setFetchSource(true); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertThat(updateResponse.getIndex(), equalTo("test")); assertThat(updateResponse.getGetResult(), notNullValue()); @@ -263,11 +271,12 @@ public void testUpsertFields() throws Exception { assertThat(updateResponse.getGetResult().sourceAsMap().get("bar").toString(), equalTo("baz")); assertThat(updateResponse.getGetResult().sourceAsMap().get("extra"), nullValue()); - updateResponse = client().prepareUpdate(indexOrAlias(), "1") + updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject()) .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, PUT_VALUES_SCRIPT, Collections.singletonMap("extra", "foo"))) - .setFetchSource(true) - .get(); + .setFetchSource(true); + updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertThat(updateResponse.getIndex(), equalTo("test")); assertThat(updateResponse.getGetResult(), notNullValue()); @@ -277,11 +286,12 @@ public void testUpsertFields() throws Exception { } public void testIndexAutoCreation() throws Exception { - UpdateResponse updateResponse = client().prepareUpdate("test", "1") + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject()) .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, PUT_VALUES_SCRIPT, Collections.singletonMap("extra", "foo"))) - .setFetchSource(true) - .get(); + .setFetchSource(true); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertThat(updateResponse.getIndex(), equalTo("test")); assertThat(updateResponse.getGetResult(), notNullValue()); @@ -296,139 +306,175 @@ public void testUpdate() throws Exception { ensureGreen(); Script fieldIncScript = new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, Collections.singletonMap("field", "field")); - DocumentMissingException ex = expectThrows( - DocumentMissingException.class, - client().prepareUpdate(indexOrAlias(), "1").setScript(fieldIncScript) - ); - assertEquals("[1]: document missing", ex.getMessage()); - - prepareIndex("test").setId("1").setSource("field", 1).get(); - - UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "1").setScript(fieldIncScript).get(); - assertThat(updateResponse.getVersion(), equalTo(2L)); - assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); - assertThat(updateResponse.getIndex(), equalTo("test")); - - for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "1").get(); - assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("2")); + { + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1").setScript(fieldIncScript); + DocumentMissingException ex = expectThrows(DocumentMissingException.class, updateRequestBuilder); + updateRequestBuilder.request().decRef(); + assertEquals("[1]: document missing", ex.getMessage()); } - Map params = new HashMap<>(); - params.put("inc", 3); - params.put("field", "field"); - updateResponse = client().prepareUpdate(indexOrAlias(), "1") - .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, params)) - .get(); - assertThat(updateResponse.getVersion(), equalTo(3L)); - assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); - assertThat(updateResponse.getIndex(), equalTo("test")); - - for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "1").get(); - assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("5")); + { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field", 1); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1").setScript(fieldIncScript); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + assertThat(updateResponse.getVersion(), equalTo(2L)); + assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); + assertThat(updateResponse.getIndex(), equalTo("test")); + + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client().prepareGet("test", "1").get(); + assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("2")); + } } - // check noop - updateResponse = client().prepareUpdate(indexOrAlias(), "1") - .setScript( - new Script( - ScriptType.INLINE, - UPDATE_SCRIPTS, - PUT_VALUES_SCRIPT, - Collections.singletonMap("_ctx", Collections.singletonMap("op", "none")) - ) - ) - .get(); - assertThat(updateResponse.getVersion(), equalTo(3L)); - assertEquals(DocWriteResponse.Result.NOOP, updateResponse.getResult()); - assertThat(updateResponse.getIndex(), equalTo("test")); + { + Map params = new HashMap<>(); + params.put("inc", 3); + params.put("field", "field"); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") + .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, params)); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + assertThat(updateResponse.getVersion(), equalTo(3L)); + assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); + assertThat(updateResponse.getIndex(), equalTo("test")); + + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client().prepareGet("test", "1").get(); + assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("5")); + } + } - for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "1").get(); - assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("5")); + { + // check noop + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") + .setScript( + new Script( + ScriptType.INLINE, + UPDATE_SCRIPTS, + PUT_VALUES_SCRIPT, + Collections.singletonMap("_ctx", Collections.singletonMap("op", "none")) + ) + ); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + assertThat(updateResponse.getVersion(), equalTo(3L)); + assertEquals(DocWriteResponse.Result.NOOP, updateResponse.getResult()); + assertThat(updateResponse.getIndex(), equalTo("test")); + + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client().prepareGet("test", "1").get(); + assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("5")); + } } - // check delete - updateResponse = client().prepareUpdate(indexOrAlias(), "1") - .setScript( - new Script( - ScriptType.INLINE, - UPDATE_SCRIPTS, - PUT_VALUES_SCRIPT, - Collections.singletonMap("_ctx", Collections.singletonMap("op", "delete")) - ) - ) - .get(); - assertThat(updateResponse.getVersion(), equalTo(4L)); - assertEquals(DocWriteResponse.Result.DELETED, updateResponse.getResult()); - assertThat(updateResponse.getIndex(), equalTo("test")); + { + // check delete + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") + .setScript( + new Script( + ScriptType.INLINE, + UPDATE_SCRIPTS, + PUT_VALUES_SCRIPT, + Collections.singletonMap("_ctx", Collections.singletonMap("op", "delete")) + ) + ); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + assertThat(updateResponse.getVersion(), equalTo(4L)); + assertEquals(DocWriteResponse.Result.DELETED, updateResponse.getResult()); + assertThat(updateResponse.getIndex(), equalTo("test")); + + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client().prepareGet("test", "1").get(); + assertThat(getResponse.isExists(), equalTo(false)); + } + } - for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "1").get(); - assertThat(getResponse.isExists(), equalTo(false)); + { + // check _source parameter + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field1", 1, "field2", 2); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") + .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, Collections.singletonMap("field", "field1"))) + .setFetchSource("field1", "field2"); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + assertThat(updateResponse.getIndex(), equalTo("test")); + assertThat(updateResponse.getGetResult(), notNullValue()); + assertThat(updateResponse.getGetResult().getIndex(), equalTo("test")); + assertThat(updateResponse.getGetResult().sourceRef(), notNullValue()); + assertThat(updateResponse.getGetResult().field("field1"), nullValue()); + assertThat(updateResponse.getGetResult().sourceAsMap().size(), equalTo(1)); + assertThat(updateResponse.getGetResult().sourceAsMap().get("field1"), equalTo(2)); } - // check _source parameter - prepareIndex("test").setId("1").setSource("field1", 1, "field2", 2).get(); - updateResponse = client().prepareUpdate(indexOrAlias(), "1") - .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, Collections.singletonMap("field", "field1"))) - .setFetchSource("field1", "field2") - .get(); - assertThat(updateResponse.getIndex(), equalTo("test")); - assertThat(updateResponse.getGetResult(), notNullValue()); - assertThat(updateResponse.getGetResult().getIndex(), equalTo("test")); - assertThat(updateResponse.getGetResult().sourceRef(), notNullValue()); - assertThat(updateResponse.getGetResult().field("field1"), nullValue()); - assertThat(updateResponse.getGetResult().sourceAsMap().size(), equalTo(1)); - assertThat(updateResponse.getGetResult().sourceAsMap().get("field1"), equalTo(2)); - - // check updates without script - // add new field - prepareIndex("test").setId("1").setSource("field", 1).get(); - client().prepareUpdate(indexOrAlias(), "1") - .setDoc(XContentFactory.jsonBuilder().startObject().field("field2", 2).endObject()) - .get(); - for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "1").get(); - assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1")); - assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2")); + { + // check updates without script + // add new field + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field", 1); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") + .setDoc(XContentFactory.jsonBuilder().startObject().field("field2", 2).endObject()); + updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client().prepareGet("test", "1").get(); + assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1")); + assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2")); + } } - // change existing field - client().prepareUpdate(indexOrAlias(), "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field", 3).endObject()).get(); - for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "1").get(); - assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("3")); - assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2")); + { + // change existing field + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") + .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 3).endObject()); + updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client().prepareGet("test", "1").get(); + assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("3")); + assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2")); + } } - // recursive map - Map testMap = new HashMap<>(); - Map testMap2 = new HashMap<>(); - Map testMap3 = new HashMap<>(); - testMap3.put("commonkey", testMap); - testMap3.put("map3", 5); - testMap2.put("map2", 6); - testMap.put("commonkey", testMap2); - testMap.put("map1", 8); - - prepareIndex("test").setId("1").setSource("map", testMap).get(); - client().prepareUpdate(indexOrAlias(), "1") - .setDoc(XContentFactory.jsonBuilder().startObject().field("map", testMap3).endObject()) - .get(); - for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "1").get(); - Map map1 = get(getResponse.getSourceAsMap(), "map"); - assertThat(map1.size(), equalTo(3)); - assertThat(map1.containsKey("map1"), equalTo(true)); - assertThat(map1.containsKey("map3"), equalTo(true)); - assertThat(map1.containsKey("commonkey"), equalTo(true)); - Map map2 = get(map1, "commonkey"); - assertThat(map2.size(), equalTo(3)); - assertThat(map2.containsKey("map1"), equalTo(true)); - assertThat(map2.containsKey("map2"), equalTo(true)); - assertThat(map2.containsKey("commonkey"), equalTo(true)); + { + // recursive map + Map testMap = new HashMap<>(); + Map testMap2 = new HashMap<>(); + Map testMap3 = new HashMap<>(); + testMap3.put("commonkey", testMap); + testMap3.put("map3", 5); + testMap2.put("map2", 6); + testMap.put("commonkey", testMap2); + testMap.put("map1", 8); + + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("map", testMap); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") + .setDoc(XContentFactory.jsonBuilder().startObject().field("map", testMap3).endObject()); + updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client().prepareGet("test", "1").get(); + Map map1 = get(getResponse.getSourceAsMap(), "map"); + assertThat(map1.size(), equalTo(3)); + assertThat(map1.containsKey("map1"), equalTo(true)); + assertThat(map1.containsKey("map3"), equalTo(true)); + assertThat(map1.containsKey("commonkey"), equalTo(true)); + Map map2 = get(map1, "commonkey"); + assertThat(map2.size(), equalTo(3)); + assertThat(map2.containsKey("map1"), equalTo(true)); + assertThat(map2.containsKey("map2"), equalTo(true)); + assertThat(map2.containsKey("commonkey"), equalTo(true)); + } } } @@ -436,39 +482,46 @@ public void testUpdateWithIfSeqNo() throws Exception { createTestIndex(); ensureGreen(); - DocWriteResponse result = prepareIndex("test").setId("1").setSource("field", 1).get(); - expectThrows( - VersionConflictEngineException.class, - client().prepareUpdate(indexOrAlias(), "1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field", 1); + DocWriteResponse result = indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + { + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 2).endObject()) .setIfSeqNo(result.getSeqNo() + 1) - .setIfPrimaryTerm(result.getPrimaryTerm()) - ); + .setIfPrimaryTerm(result.getPrimaryTerm()); + expectThrows(VersionConflictEngineException.class, updateRequestBuilder); + updateRequestBuilder.request().decRef(); + } - expectThrows( - VersionConflictEngineException.class, - client().prepareUpdate(indexOrAlias(), "1") + { + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 2).endObject()) .setIfSeqNo(result.getSeqNo()) - .setIfPrimaryTerm(result.getPrimaryTerm() + 1) - ); + .setIfPrimaryTerm(result.getPrimaryTerm() + 1); + expectThrows(VersionConflictEngineException.class, updateRequestBuilder); + updateRequestBuilder.request().decRef(); + } - expectThrows( - VersionConflictEngineException.class, - client().prepareUpdate(indexOrAlias(), "1") + { + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1") .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 2).endObject()) .setIfSeqNo(result.getSeqNo() + 1) - .setIfPrimaryTerm(result.getPrimaryTerm() + 1) - ); + .setIfPrimaryTerm(result.getPrimaryTerm() + 1); + expectThrows(VersionConflictEngineException.class, updateRequestBuilder); + updateRequestBuilder.request().decRef(); + } - UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "1") - .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 2).endObject()) - .setIfSeqNo(result.getSeqNo()) - .setIfPrimaryTerm(result.getPrimaryTerm()) - .get(); + { + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1"); + UpdateResponse updateResponse = updateRequestBuilder.setDoc( + XContentFactory.jsonBuilder().startObject().field("field", 2).endObject() + ).setIfSeqNo(result.getSeqNo()).setIfPrimaryTerm(result.getPrimaryTerm()).get(); + updateRequestBuilder.request().decRef(); - assertThat(updateResponse.status(), equalTo(RestStatus.OK)); - assertThat(updateResponse.getSeqNo(), equalTo(result.getSeqNo() + 1)); + assertThat(updateResponse.status(), equalTo(RestStatus.OK)); + assertThat(updateResponse.getSeqNo(), equalTo(result.getSeqNo() + 1)); + } } public void testUpdateRequestWithBothScriptAndDoc() throws Exception { @@ -476,16 +529,18 @@ public void testUpdateRequestWithBothScriptAndDoc() throws Exception { ensureGreen(); Script fieldIncScript = new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, Collections.singletonMap("field", "field")); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1"); try { - client().prepareUpdate(indexOrAlias(), "1") - .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject()) - .setScript(fieldIncScript) - .get(); + updateRequestBuilder.setDoc(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject()) + .setScript(fieldIncScript); + updateRequestBuilder.get(); fail("Should have thrown ActionRequestValidationException"); } catch (ActionRequestValidationException e) { assertThat(e.validationErrors().size(), equalTo(1)); assertThat(e.validationErrors().get(0), containsString("can't provide both script and doc")); assertThat(e.getMessage(), containsString("can't provide both script and doc")); + } finally { + updateRequestBuilder.request().decRef(); } } @@ -493,10 +548,13 @@ public void testUpdateRequestWithScriptAndShouldUpsertDoc() throws Exception { createTestIndex(); ensureGreen(); Script fieldIncScript = new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, Collections.singletonMap("field", "field")); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "1"); try { - client().prepareUpdate(indexOrAlias(), "1").setScript(fieldIncScript).setDocAsUpsert(true).get(); + updateRequestBuilder.setScript(fieldIncScript).setDocAsUpsert(true); + updateRequestBuilder.get(); fail("Should have thrown ActionRequestValidationException"); } catch (ActionRequestValidationException e) { + updateRequestBuilder.request().decRef(); assertThat(e.validationErrors().size(), equalTo(1)); assertThat(e.validationErrors().get(0), containsString("doc must be specified if doc_as_upsert is enabled")); assertThat(e.getMessage(), containsString("doc must be specified if doc_as_upsert is enabled")); @@ -508,14 +566,21 @@ public void testContextVariables() throws Exception { ensureGreen(); // Index some documents - prepareIndex("test").setId("id1").setRouting("routing1").setSource("field1", 1, "content", "foo").get(); - prepareIndex("test").setId("id2").setSource("field1", 0, "content", "bar").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("id1") + .setRouting("routing1") + .setSource("field1", 1, "content", "foo"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex("test").setId("id2").setSource("field1", 0, "content", "bar"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); // Update the first object and note context variables values - UpdateResponse updateResponse = client().prepareUpdate("test", "id1") + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", "id1") .setRouting("routing1") - .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, EXTRACT_CTX_SCRIPT, Collections.emptyMap())) - .get(); + .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, EXTRACT_CTX_SCRIPT, Collections.emptyMap())); + UpdateResponse updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertEquals(2, updateResponse.getVersion()); @@ -527,9 +592,10 @@ public void testContextVariables() throws Exception { assertEquals("routing1", updateContext.get("_routing")); // Idem with the second object - updateResponse = client().prepareUpdate("test", "id2") - .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, EXTRACT_CTX_SCRIPT, Collections.emptyMap())) - .get(); + updateRequestBuilder = client().prepareUpdate("test", "id2") + .setScript(new Script(ScriptType.INLINE, UPDATE_SCRIPTS, EXTRACT_CTX_SCRIPT, Collections.emptyMap())); + updateResponse = updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); assertEquals(2, updateResponse.getVersion()); @@ -576,13 +642,19 @@ public void run() { .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { bulkRequestBuilder.add(updateRequestBuilder).get(); + } finally { + updateRequestBuilder.request().decRef(); } } else { - client().prepareUpdate(indexOrAlias(), Integer.toString(i)) - .setScript(fieldIncScript) - .setRetryOnConflict(Integer.MAX_VALUE) - .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()) - .get(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), Integer.toString(i)); + try { + updateRequestBuilder.setScript(fieldIncScript) + .setRetryOnConflict(Integer.MAX_VALUE) + .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()) + .get(); + } finally { + updateRequestBuilder.request().decRef(); + } } } logger.info("Client [{}] issued all [{}] requests.", Thread.currentThread().getName(), numberOfUpdatesPerThread); @@ -698,27 +770,35 @@ public void run() { for (int j = 0; j < numberOfIds; j++) { for (int k = 0; k < numberOfUpdatesPerId; ++k) { updateRequestsOutstanding.acquire(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", Integer.toString(j)); + try { - UpdateRequest ur = client().prepareUpdate("test", Integer.toString(j)) - .setScript(fieldIncScript) + UpdateRequest ur = updateRequestBuilder.setScript(fieldIncScript) .setRetryOnConflict(retryOnConflict) .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()) .request(); if (randomBoolean()) { - client().update(ur, new UpdateListener(j)); + client().update(ur, ActionListener.runAfter(new UpdateListener(j), ur::decRef)); } else { - try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - bulkRequestBuilder.add(ur).execute(new UpdateListener(j).map(br -> { + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + try { + bulkRequestBuilder.add(ur); + ur.decRef(); + bulkRequestBuilder.execute(ActionListener.runAfter(new UpdateListener(j).map(br -> { final BulkItemResponse ir = br.getItems()[0]; if (ir.isFailed()) { throw ir.getFailure().getCause(); } else { return ir.getResponse(); } - })); + }), bulkRequestBuilder::close)); + } catch (Exception e) { + bulkRequestBuilder.close(); + throw e; } } } catch (NoNodeAvailableException nne) { + updateRequestBuilder.request().decRef(); updateRequestsOutstanding.release(); synchronized (failedMap) { incrementMapValue(j, failedMap); @@ -820,11 +900,12 @@ private void waitForOutstandingRequests(TimeValue timeOut, Semaphore requestsOut // This means that we add 1 to the expected versions and attempts // All the previous operations should be complete or failed at this point for (int i = 0; i < numberOfIdsPerThread; ++i) { - client().prepareUpdate("test", Integer.toString(i)) + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("test", Integer.toString(i)) .setScript(fieldIncScript) .setRetryOnConflict(Integer.MAX_VALUE) - .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()) - .get(); + .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()); + updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); } refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateNoopIT.java b/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateNoopIT.java index eb3f248d38044..58a3ecd6825b8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateNoopIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateNoopIT.java @@ -311,6 +311,7 @@ private UpdateResponse update(Boolean detectNoop, long expectedSeqNo, long expec updateRequest.setDetectNoop(detectNoop); } UpdateResponse updateResponse = updateRequest.get(); + updateRequest.request().decRef(); assertThat(updateResponse.getGetResult(), notNullValue()); assertThat(updateResponse.getSeqNo(), equalTo(expectedSeqNo)); assertThat(updateResponse.getVersion(), equalTo(expectedVersion)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index 1e8cc954cd850..8c0ec62e4fc3e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; @@ -138,7 +139,7 @@ public void testExplainValidateQueryTwoNodes() throws IOException { .get(); for (int i = 0; i < 10; i++) { - prepareIndex("test").setSource("foo", "text", "bar", i, "baz", "blort").setId(Integer.toString(i)).get(); + indexDocs("test", Integer.toString(i), "foo", "text", "bar", i, "baz", "blort"); } refresh(); @@ -183,7 +184,7 @@ public void testExplainDateRangeInQueryString() { String aMonthAgo = DateTimeFormatter.ISO_LOCAL_DATE.format(now.plus(1, ChronoUnit.MONTHS)); String aMonthFromNow = DateTimeFormatter.ISO_LOCAL_DATE.format(now.minus(1, ChronoUnit.MONTHS)); - prepareIndex("test").setId("1").setSource("past", aMonthAgo, "future", aMonthFromNow).get(); + indexDocs("test", "1", "past", aMonthAgo, "future", aMonthFromNow); refresh(); @@ -247,10 +248,10 @@ public void testExplainWithRewriteValidateQuery() { .setMapping("field", "type=text,analyzer=whitespace") .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1)) .get(); - prepareIndex("test").setId("1").setSource("field", "quick lazy huge brown pidgin").get(); - prepareIndex("test").setId("2").setSource("field", "the quick brown fox").get(); - prepareIndex("test").setId("3").setSource("field", "the quick lazy huge brown fox jumps over the tree").get(); - prepareIndex("test").setId("4").setSource("field", "the lazy dog quacks like a duck").get(); + indexDocs("test", "1", "field", "quick lazy huge brown pidgin"); + indexDocs("test", "2", "field", "the quick brown fox"); + indexDocs("test", "3", "field", "the quick lazy huge brown fox jumps over the tree"); + indexDocs("test", "4", "field", "the lazy dog quacks like a duck"); refresh(); // prefix queries @@ -293,10 +294,10 @@ public void testExplainWithRewriteValidateQueryAllShards() { .get(); // We are relying on specific routing behaviors for the result to be right, so // we cannot randomize the number of shards or change ids here. - prepareIndex("test").setId("1").setSource("field", "quick lazy huge brown pidgin").get(); - prepareIndex("test").setId("2").setSource("field", "the quick brown fox").get(); - prepareIndex("test").setId("3").setSource("field", "the quick lazy huge brown fox jumps over the tree").get(); - prepareIndex("test").setId("4").setSource("field", "the lazy dog quacks like a duck").get(); + indexDocs("test", "1", "field", "quick lazy huge brown pidgin"); + indexDocs("test", "2", "field", "the quick brown fox"); + indexDocs("test", "3", "field", "the quick lazy huge brown fox jumps over the tree"); + indexDocs("test", "4", "field", "the lazy dog quacks like a duck"); refresh(); // prefix queries @@ -371,11 +372,20 @@ public void testExplainTermsQueryWithLookup() { .setMapping("user", "type=integer", "followers", "type=integer") .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2)) .get(); - prepareIndex("twitter").setId("1").setSource("followers", new int[] { 1, 2, 3 }).get(); + indexDocs("twitter", "1", "followers", new int[] { 1, 2, 3 }); refresh(); TermsQueryBuilder termsLookupQuery = QueryBuilders.termsLookupQuery("user", new TermsLookup("twitter", "1", "followers")); ValidateQueryResponse response = indicesAdmin().prepareValidateQuery("twitter").setQuery(termsLookupQuery).setExplain(true).get(); assertThat(response.isValid(), is(true)); } + + private void indexDocs(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id).setSource(source); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java index 69c10edf89809..5885807e3e376 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; @@ -31,7 +32,8 @@ public void testConcurrentOperationOnSameDoc() throws Exception { final AtomicReference failure = new AtomicReference<>(); final CountDownLatch latch = new CountDownLatch(numberOfUpdates); for (int i = 0; i < numberOfUpdates; i++) { - prepareIndex("test").setId("1").setSource("field1", i).execute(new ActionListener<>() { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field1", i); + indexRequestBuilder.execute(ActionListener.runAfter(new ActionListener<>() { @Override public void onResponse(DocWriteResponse response) { latch.countDown(); @@ -43,7 +45,7 @@ public void onFailure(Exception e) { failure.set(e); latch.countDown(); } - }); + }, () -> indexRequestBuilder.request().decRef())); } latch.await(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java index 2b804293cd506..474a313bc438b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.coordination.LinearizabilityChecker; import org.elasticsearch.cluster.coordination.LinearizabilityChecker.LinearizabilityCheckAborted; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -135,10 +136,14 @@ public void testSeqNoCASLinearizability() { int numberOfKeys = randomIntBetween(1, 10); logger.info("--> Indexing initial doc for {} keys", numberOfKeys); - List partitions = IntStream.range(0, numberOfKeys) - .mapToObj(i -> prepareIndex("test").setId("ID:" + i).setSource("value", -1).get()) - .map(response -> new Partition(response.getId(), new Version(response.getPrimaryTerm(), response.getSeqNo()))) - .toList(); + List partitions = IntStream.range(0, numberOfKeys).mapToObj(i -> { + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("ID:" + i).setSource("value", -1); + try { + return indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + }).map(response -> new Partition(response.getId(), new Version(response.getPrimaryTerm(), response.getSeqNo()))).toList(); int threadCount = randomIntBetween(3, 20); CyclicBarrier roundBarrier = new CyclicBarrier(threadCount + 1); // +1 for main thread. @@ -231,11 +236,11 @@ public void run() { version = version.previousTerm(); } + Consumer historyResponse = partition.invoke(version); IndexRequest indexRequest = new IndexRequest("test").id(partition.id) .source("value", random.nextInt()) .setIfPrimaryTerm(version.primaryTerm) .setIfSeqNo(version.seqNo); - Consumer historyResponse = partition.invoke(version); try { // we should be able to remove timeout or fail hard on timeouts DocWriteResponse indexResponse = client().index(indexRequest).actionGet(timeout, TimeUnit.SECONDS); @@ -269,6 +274,8 @@ public void run() { // interrupt often comes as a RuntimeException so check to stop here too. return; } + } finally { + indexRequest.decRef(); } } } catch (InterruptedException e) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java index 6f2feb3e05f9d..580bb23b9db07 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.uid.Versions; @@ -52,48 +53,35 @@ public void testExternalVersioningInitialDelete() throws Exception { assertEquals(DocWriteResponse.Result.NOT_FOUND, deleteResponse.getResult()); // this should conflict with the delete command transaction which told us that the object was deleted at version 17. - assertFutureThrows( - prepareIndex("test").setId("1").setSource("field1", "value1_1").setVersion(13).setVersionType(VersionType.EXTERNAL).execute(), - VersionConflictEngineException.class - ); - - DocWriteResponse indexResponse = prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource("field1", "value1_1") - .setVersion(18) - .setVersionType(VersionType.EXTERNAL) - .get(); + .setVersion(13) + .setVersionType(VersionType.EXTERNAL); + assertFutureThrows(indexRequestBuilder.execute(), VersionConflictEngineException.class); + indexRequestBuilder.request().decRef(); + + DocWriteResponse indexResponse = indexDocWithVersion("test", "1", 18, VersionType.EXTERNAL, "field1", "value1_1"); assertThat(indexResponse.getVersion(), equalTo(18L)); } public void testExternalGTE() throws Exception { createIndex("test"); - DocWriteResponse indexResponse = prepareIndex("test").setId("1") - .setSource("field1", "value1_1") - .setVersion(12) - .setVersionType(VersionType.EXTERNAL_GTE) - .get(); + DocWriteResponse indexResponse = indexDocWithVersion("test", "1", 12, VersionType.EXTERNAL_GTE, "field1", "value1_1"); assertThat(indexResponse.getVersion(), equalTo(12L)); - indexResponse = prepareIndex("test").setId("1") - .setSource("field1", "value1_2") - .setVersion(12) - .setVersionType(VersionType.EXTERNAL_GTE) - .get(); + indexResponse = indexDocWithVersion("test", "1", 12, VersionType.EXTERNAL_GTE, "field1", "value1_2"); assertThat(indexResponse.getVersion(), equalTo(12L)); - indexResponse = prepareIndex("test").setId("1") - .setSource("field1", "value1_2") - .setVersion(14) - .setVersionType(VersionType.EXTERNAL_GTE) - .get(); + indexResponse = indexDocWithVersion("test", "1", 14, VersionType.EXTERNAL_GTE, "field1", "value1_2"); assertThat(indexResponse.getVersion(), equalTo(14L)); - ActionRequestBuilder builder1 = prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource("field1", "value1_1") .setVersion(13) .setVersionType(VersionType.EXTERNAL_GTE); - expectThrows(VersionConflictEngineException.class, builder1); + expectThrows(VersionConflictEngineException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); client().admin().indices().prepareRefresh().get(); if (randomBoolean()) { @@ -129,24 +117,18 @@ public void testExternalVersioning() throws Exception { createIndex("test"); ensureGreen(); - DocWriteResponse indexResponse = prepareIndex("test").setId("1") - .setSource("field1", "value1_1") - .setVersion(12) - .setVersionType(VersionType.EXTERNAL) - .get(); + DocWriteResponse indexResponse = indexDocWithVersion("test", "1", 12, VersionType.EXTERNAL, "field1", "value1_1"); assertThat(indexResponse.getVersion(), equalTo(12L)); - indexResponse = prepareIndex("test").setId("1") - .setSource("field1", "value1_1") - .setVersion(14) - .setVersionType(VersionType.EXTERNAL) - .get(); + indexResponse = indexDocWithVersion("test", "1", 14, VersionType.EXTERNAL, "field1", "value1_1"); assertThat(indexResponse.getVersion(), equalTo(14L)); - assertFutureThrows( - prepareIndex("test").setId("1").setSource("field1", "value1_1").setVersion(13).setVersionType(VersionType.EXTERNAL).execute(), - VersionConflictEngineException.class - ); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setSource("field1", "value1_1") + .setVersion(13) + .setVersionType(VersionType.EXTERNAL); + assertFutureThrows(indexRequestBuilder.execute(), VersionConflictEngineException.class); + indexRequestBuilder.request().decRef(); if (randomBoolean()) { refresh(); @@ -179,11 +161,7 @@ public void testExternalVersioning() throws Exception { // TODO: This behavior breaks rest api returning http status 201 // good news is that it this is only the case until deletes GC kicks in. - indexResponse = prepareIndex("test").setId("1") - .setSource("field1", "value1_1") - .setVersion(19) - .setVersionType(VersionType.EXTERNAL) - .get(); + indexResponse = indexDocWithVersion("test", "1", 19, VersionType.EXTERNAL, "field1", "value1_1"); assertThat(indexResponse.getVersion(), equalTo(19L)); deleteResponse = client().prepareDelete("test", "1").setVersion(20).setVersionType(VersionType.EXTERNAL).get(); @@ -195,11 +173,7 @@ public void testExternalVersioning() throws Exception { Thread.sleep(300); // gc works based on estimated sampled time. Give it a chance... // And now we have previous version return -1 - indexResponse = prepareIndex("test").setId("1") - .setSource("field1", "value1_1") - .setVersion(20) - .setVersionType(VersionType.EXTERNAL) - .get(); + indexResponse = indexDocWithVersion("test", "1", 20, VersionType.EXTERNAL, "field1", "value1_1"); assertThat(indexResponse.getVersion(), equalTo(20L)); } @@ -229,7 +203,7 @@ public void testCompareAndSetInitialDelete() throws Exception { VersionConflictEngineException.class ); - DocWriteResponse indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_1").setCreate(true).get(); + DocWriteResponse indexResponse = indexDocWithCreateTrue("test", "1", "field1", "value1_1"); assertThat(indexResponse.getVersion(), equalTo(1L)); } @@ -237,28 +211,28 @@ public void testCompareAndSet() { createIndex("test"); ensureGreen(); - DocWriteResponse indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_1").get(); + DocWriteResponse indexResponse = indexDoc("test", "1", "field1", "value1_1"); assertThat(indexResponse.getSeqNo(), equalTo(0L)); assertThat(indexResponse.getPrimaryTerm(), equalTo(1L)); - indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_2").setIfSeqNo(0L).setIfPrimaryTerm(1).get(); + indexResponse = indexDocWithSeqNoAndPrimaryTerm("test", "1", 0L, 1, "field1", "value1_2"); assertThat(indexResponse.getSeqNo(), equalTo(1L)); assertThat(indexResponse.getPrimaryTerm(), equalTo(1L)); - assertFutureThrows( - prepareIndex("test").setId("1").setSource("field1", "value1_1").setIfSeqNo(10).setIfPrimaryTerm(1).execute(), - VersionConflictEngineException.class - ); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setSource("field1", "value1_1") + .setIfSeqNo(10) + .setIfPrimaryTerm(1); + assertFutureThrows(indexRequestBuilder.execute(), VersionConflictEngineException.class); + indexRequestBuilder.request().decRef(); - assertFutureThrows( - prepareIndex("test").setId("1").setSource("field1", "value1_1").setIfSeqNo(10).setIfPrimaryTerm(2).execute(), - VersionConflictEngineException.class - ); + indexRequestBuilder = prepareIndex("test").setId("1").setSource("field1", "value1_1").setIfSeqNo(10).setIfPrimaryTerm(2); + assertFutureThrows(indexRequestBuilder.execute(), VersionConflictEngineException.class); + indexRequestBuilder.request().decRef(); - assertFutureThrows( - prepareIndex("test").setId("1").setSource("field1", "value1_1").setIfSeqNo(1).setIfPrimaryTerm(2).execute(), - VersionConflictEngineException.class - ); + indexRequestBuilder = prepareIndex("test").setId("1").setSource("field1", "value1_1").setIfSeqNo(1).setIfPrimaryTerm(2); + assertFutureThrows(indexRequestBuilder.execute(), VersionConflictEngineException.class); + indexRequestBuilder.request().decRef(); ActionRequestBuilder builder6 = client().prepareDelete("test", "1").setIfSeqNo(10).setIfPrimaryTerm(1); expectThrows(VersionConflictEngineException.class, builder6); @@ -312,22 +286,24 @@ public void testSimpleVersioningWithFlush() throws Exception { createIndex("test"); ensureGreen(); - DocWriteResponse indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_1").get(); + DocWriteResponse indexResponse = indexDoc("test", "1", "field1", "value1_1"); assertThat(indexResponse.getSeqNo(), equalTo(0L)); client().admin().indices().prepareFlush().get(); - indexResponse = prepareIndex("test").setId("1").setSource("field1", "value1_2").setIfSeqNo(0).setIfPrimaryTerm(1).get(); + indexResponse = indexDocWithSeqNoAndPrimaryTerm("test", "1", 0, 1, "field1", "value1_2"); assertThat(indexResponse.getSeqNo(), equalTo(1L)); client().admin().indices().prepareFlush().get(); - ActionRequestBuilder builder2 = prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setSource("field1", "value1_1") .setIfSeqNo(0) .setIfPrimaryTerm(1); - expectThrows(VersionConflictEngineException.class, builder2); + expectThrows(VersionConflictEngineException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); - ActionRequestBuilder builder1 = prepareIndex("test").setId("1").setCreate(true).setSource("field1", "value1_1"); - expectThrows(VersionConflictEngineException.class, builder1); + indexRequestBuilder = prepareIndex("test").setId("1").setCreate(true).setSource("field1", "value1_1"); + expectThrows(VersionConflictEngineException.class, indexRequestBuilder); + indexRequestBuilder.request().decRef(); ActionRequestBuilder builder = client().prepareDelete("test", "1").setIfSeqNo(0).setIfPrimaryTerm(1); expectThrows(VersionConflictEngineException.class, builder); @@ -352,7 +328,9 @@ public void testVersioningWithBulk() { ensureGreen(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - BulkResponse bulkResponse = bulkRequestBuilder.add(prepareIndex("test").setId("1").setSource("field1", "value1_1")).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field1", "value1_1"); + BulkResponse bulkResponse = bulkRequestBuilder.add(indexRequestBuilder).get(); + indexRequestBuilder.request().decRef(); assertThat(bulkResponse.hasFailures(), equalTo(false)); assertThat(bulkResponse.getItems().length, equalTo(1)); IndexResponse indexResponse = bulkResponse.getItems()[0].getResponse(); @@ -633,12 +611,7 @@ public void run() { } } else { try { - idVersion.response = prepareIndex("test").setId(id) - .setSource("foo", "bar") - .setVersion(version) - .setVersionType(VersionType.EXTERNAL) - .get(); - + idVersion.response = indexDocWithVersion("test", id, version, VersionType.EXTERNAL, "foo", "bar"); } catch (VersionConflictEngineException vcee) { // OK: our version is too old assertThat(version, lessThanOrEqualTo(truth.get(id).version)); @@ -708,12 +681,13 @@ public void testDeleteNotLost() throws Exception { updateIndexSettings(Settings.builder().put("index.gc_deletes", "10ms").put("index.refresh_interval", "-1"), "test"); // Index a doc: - prepareIndex("test").setId("id") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("id") .setSource("foo", "bar") .setOpType(DocWriteRequest.OpType.INDEX) .setVersion(10) - .setVersionType(VersionType.EXTERNAL) - .get(); + .setVersionType(VersionType.EXTERNAL); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); if (randomBoolean()) { // Force refresh so the add is sometimes visible in the searcher: @@ -744,12 +718,13 @@ public void testGCDeletesZero() throws Exception { // We test deletes, but can't rely on wall-clock delete GC: updateIndexSettings(Settings.builder().put("index.gc_deletes", "0ms"), "test"); // Index a doc: - prepareIndex("test").setId("id") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("id") .setSource("foo", "bar") .setOpType(DocWriteRequest.OpType.INDEX) .setVersion(10) - .setVersionType(VersionType.EXTERNAL) - .get(); + .setVersionType(VersionType.EXTERNAL); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); if (randomBoolean()) { // Force refresh so the add is sometimes visible in the searcher: @@ -766,33 +741,50 @@ public void testGCDeletesZero() throws Exception { public void testSpecialVersioning() { internalCluster().ensureAtLeastNumDataNodes(2); createIndex("test", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()); - DocWriteResponse doc1 = prepareIndex("test").setId("1") - .setSource("field", "value1") - .setVersion(0) - .setVersionType(VersionType.EXTERNAL) - .get(); + DocWriteResponse doc1 = indexDocWithVersion("test", "1", 0, VersionType.EXTERNAL, "field", "value1"); assertThat(doc1.getVersion(), equalTo(0L)); - DocWriteResponse doc2 = prepareIndex("test").setId("1") - .setSource("field", "value2") - .setVersion(Versions.MATCH_ANY) - .setVersionType(VersionType.INTERNAL) - .get(); + DocWriteResponse doc2 = indexDocWithVersion("test", "1", Versions.MATCH_ANY, VersionType.INTERNAL, "field", "value2"); assertThat(doc2.getVersion(), equalTo(1L)); client().prepareDelete("test", "1").get(); // v2 - DocWriteResponse doc3 = prepareIndex("test").setId("1") - .setSource("field", "value3") - .setVersion(Versions.MATCH_DELETED) - .setVersionType(VersionType.INTERNAL) - .get(); + DocWriteResponse doc3 = indexDocWithVersion("test", "1", Versions.MATCH_DELETED, VersionType.INTERNAL, "field", "value3"); assertThat(doc3.getVersion(), equalTo(3L)); - DocWriteResponse doc4 = prepareIndex("test").setId("1") - .setSource("field", "value4") - .setVersion(4L) - .setVersionType(VersionType.EXTERNAL_GTE) - .get(); + DocWriteResponse doc4 = indexDocWithVersion("test", "1", 4L, VersionType.EXTERNAL_GTE, "field", "value4"); assertThat(doc4.getVersion(), equalTo(4L)); // Make sure that these versions are replicated correctly setReplicaCount(1, "test"); ensureGreen("test"); } + + private DocWriteResponse indexDocWithVersion(String index, String id, long version, VersionType versionType, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id) + .setVersion(version) + .setVersionType(versionType) + .setSource(source); + try { + return indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private DocWriteResponse indexDocWithCreateTrue(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id).setCreate(true).setSource(source); + try { + return indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private DocWriteResponse indexDocWithSeqNoAndPrimaryTerm(String index, String id, long seqNo, long primaryTerm, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id) + .setIfSeqNo(seqNo) + .setIfPrimaryTerm(primaryTerm) + .setSource(source); + try { + return indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java index 425461d1f4ba1..762e8ec7eecef 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java @@ -15,23 +15,29 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; +import org.elasticsearch.core.Releasable; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.transport.LeakTracker; import java.io.IOException; import java.util.Objects; -public class BulkItemRequest implements Writeable, Accountable { +public class BulkItemRequest implements Writeable, Accountable, RefCounted, Releasable { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(BulkItemRequest.class); private final int id; private final DocWriteRequest request; private volatile BulkItemResponse primaryResponse; + private final RefCounted refCounted; BulkItemRequest(@Nullable ShardId shardId, StreamInput in) throws IOException { id = in.readVInt(); request = DocWriteRequest.readDocumentRequest(shardId, in); + this.refCounted = LeakTracker.wrap(new BulkItemRequestRefCounted()); if (in.readBoolean()) { if (shardId == null) { primaryResponse = new BulkItemResponse(in); @@ -45,6 +51,10 @@ public class BulkItemRequest implements Writeable, Accountable { public BulkItemRequest(int id, DocWriteRequest request) { this.id = id; this.request = request; + this.refCounted = LeakTracker.wrap(new BulkItemRequestRefCounted()); + if (this.request instanceof RefCounted refCountedRequest) { + refCountedRequest.incRef(); + } } public int id() { @@ -111,4 +121,43 @@ public void writeThin(StreamOutput out) throws IOException { public long ramBytesUsed() { return SHALLOW_SIZE + request.ramBytesUsed(); } + + @Override + public void incRef() { + refCounted.incRef(); + } + + @Override + public boolean tryIncRef() { + return refCounted.tryIncRef(); + } + + @Override + public boolean decRef() { + assert refCounted.hasReferences() : "Attempt to decRef BulkItemRequest that is already closed"; + boolean droppedToZero = refCounted.decRef(); + if (droppedToZero && this.request instanceof RefCounted refCountedRequest) { + refCountedRequest.decRef(); + } + return droppedToZero; + } + + @Override + public boolean hasReferences() { + return refCounted.hasReferences(); + } + + @Override + public void close() { + boolean closed = decRef(); + assert closed : "Attempt to close BulkItemRequest but it still has references"; + } + + private static class BulkItemRequestRefCounted extends AbstractRefCounted { + @Override + protected void closeInternal() { + // nothing to close + } + } + } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java index 78df7fdc25542..e74a163709a9d 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java @@ -321,7 +321,14 @@ public void markAsCompleted(BulkItemResponse translatedResponse) { assert translatedResponse.getItemId() == getCurrentItem().id(); if (translatedResponse.isFailed() == false && requestToExecute != null && requestToExecute != getCurrent()) { - request.items()[currentIndex] = new BulkItemRequest(request.items()[currentIndex].id(), requestToExecute); + BulkItemRequest oldBulkItemRequest = request.items()[currentIndex]; + try { + request.items()[currentIndex] = new BulkItemRequest(request.items()[currentIndex].id(), requestToExecute); + } finally { + if (oldBulkItemRequest != null) { + oldBulkItemRequest.decRef(); + } + } } getCurrentItem().setPrimaryResponse(translatedResponse); currentItemState = ItemProcessingState.COMPLETED; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 87412666c1773..3873fbca61c32 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -159,7 +159,7 @@ public BulkRequest add(IndexRequest request) { BulkRequest internalAdd(IndexRequest request) { Objects.requireNonNull(request, "'request' must not be null"); applyGlobalMandatoryParameters(request); - + request.incRef(); requests.add(request); // lack of source is validated in validate() method sizeInBytes += (request.source() != null ? request.source().length() : 0) + REQUEST_OVERHEAD; @@ -175,6 +175,7 @@ public BulkRequest add(UpdateRequest request) { } BulkRequest internalAdd(UpdateRequest request) { + request.incRef(); Objects.requireNonNull(request, "'request' must not be null"); applyGlobalMandatoryParameters(request); @@ -209,7 +210,7 @@ public BulkRequest add(DeleteRequest request) { * The list of requests in this bulk request. */ public List> requests() { - return this.requests; + return Collections.unmodifiableList(this.requests); } /** @@ -494,7 +495,15 @@ public boolean tryIncRef() { @Override public boolean decRef() { - return refCounted.decRef(); + boolean droppedToZero = refCounted.decRef(); + if (droppedToZero) { + for (DocWriteRequest request : requests) { + if (request instanceof RefCounted refCountedRequest) { + refCountedRequest.decRef(); + } + } + } + return droppedToZero; } @Override diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java index 6690f4d964fba..9dc9cb55b6796 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java @@ -161,6 +161,6 @@ public BulkRequestBuilder routing(String globalRouting) { @Override public void close() { - request.close(); + request.decRef(); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java index 5dccd1b55f554..412c4aecb929e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java @@ -343,42 +343,47 @@ public void parse( // of index request. if ("index".equals(action)) { if (opType == null) { - indexRequestConsumer.accept( - new IndexRequest(index).id(id) - .routing(routing) - .version(version) - .versionType(versionType) - .setPipeline(pipeline) - .setIfSeqNo(ifSeqNo) - .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) - .setDynamicTemplates(dynamicTemplates) - .setRequireAlias(requireAlias) - .setRequireDataStream(requireDataStream) - .setListExecutedPipelines(listExecutedPipelines), - type - ); + IndexRequest indexRequest = new IndexRequest(index).id(id) + .routing(routing) + .version(version) + .versionType(versionType) + .setPipeline(pipeline) + .setIfSeqNo(ifSeqNo) + .setIfPrimaryTerm(ifPrimaryTerm) + .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) + .setDynamicTemplates(dynamicTemplates) + .setRequireAlias(requireAlias) + .setRequireDataStream(requireDataStream) + .setListExecutedPipelines(listExecutedPipelines); + try { + indexRequestConsumer.accept(indexRequest, type); + } finally { + indexRequest.decRef(); + } } else { - indexRequestConsumer.accept( - new IndexRequest(index).id(id) - .routing(routing) - .version(version) - .versionType(versionType) - .create("create".equals(opType)) - .setPipeline(pipeline) - .setIfSeqNo(ifSeqNo) - .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) - .setDynamicTemplates(dynamicTemplates) - .setRequireAlias(requireAlias) - .setRequireDataStream(requireDataStream) - .setListExecutedPipelines(listExecutedPipelines), - type - ); + IndexRequest indexRequest = new IndexRequest(index).id(id) + .routing(routing) + .version(version) + .versionType(versionType) + .create("create".equals(opType)) + .setPipeline(pipeline) + .setIfSeqNo(ifSeqNo) + .setIfPrimaryTerm(ifPrimaryTerm) + .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) + .setDynamicTemplates(dynamicTemplates) + .setRequireAlias(requireAlias) + .setRequireDataStream(requireDataStream) + .setListExecutedPipelines(listExecutedPipelines); + try { + indexRequestConsumer.accept(indexRequest, type); + } finally { + indexRequest.decRef(); + } } } else if ("create".equals(action)) { - indexRequestConsumer.accept( - new IndexRequest(index).id(id) + IndexRequest indexRequest = new IndexRequest(index); + try { + indexRequest.id(id) .routing(routing) .version(version) .versionType(versionType) @@ -390,9 +395,11 @@ public void parse( .setDynamicTemplates(dynamicTemplates) .setRequireAlias(requireAlias) .setRequireDataStream(requireDataStream) - .setListExecutedPipelines(listExecutedPipelines), - type - ); + .setListExecutedPipelines(listExecutedPipelines); + indexRequestConsumer.accept(indexRequest, type); + } finally { + indexRequest.decRef(); + } } else if ("update".equals(action)) { if (version != Versions.MATCH_ANY || versionType != VersionType.INTERNAL) { throw new IllegalArgumentException( @@ -419,23 +426,27 @@ public void parse( .setIfPrimaryTerm(ifPrimaryTerm) .setRequireAlias(requireAlias) .routing(routing); - try ( - XContentParser sliceParser = createParser( - xContent, - sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType) - ) - ) { - updateRequest.fromXContent(sliceParser); - } - if (fetchSourceContext != null) { - updateRequest.fetchSource(fetchSourceContext); - } - IndexRequest upsertRequest = updateRequest.upsertRequest(); - if (upsertRequest != null) { - upsertRequest.setPipeline(pipeline).setListExecutedPipelines(listExecutedPipelines); - } + try { + try ( + XContentParser sliceParser = createParser( + xContent, + sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType) + ) + ) { + updateRequest.fromXContent(sliceParser); + } + if (fetchSourceContext != null) { + updateRequest.fetchSource(fetchSourceContext); + } + IndexRequest upsertRequest = updateRequest.upsertRequest(); + if (upsertRequest != null) { + upsertRequest.setPipeline(pipeline).setListExecutedPipelines(listExecutedPipelines); + } - updateRequestConsumer.accept(updateRequest); + updateRequestConsumer.accept(updateRequest); + } finally { + updateRequest.decRef(); + } } // move pointers from = nextMarker + 1; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java index dcc883758e742..a24f55d444189 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java @@ -48,6 +48,11 @@ public BulkShardRequest(StreamInput in) throws IOException { public BulkShardRequest(ShardId shardId, RefreshPolicy refreshPolicy, BulkItemRequest[] items) { super(shardId); this.items = items; + for (BulkItemRequest item : items) { + if (item != null) { + item.incRef(); + } + } this.refCounted = LeakTracker.wrap(new BulkRequestRefCounted()); setRefreshPolicy(refreshPolicy); } @@ -175,7 +180,16 @@ public boolean tryIncRef() { @Override public boolean decRef() { - return refCounted.decRef(); + assert refCounted.hasReferences() : "Attempt to decRef BulkShardRequest that is already closed"; + boolean droppedToZero = refCounted.decRef(); + if (droppedToZero) { + for (BulkItemRequest item : items) { + if (item != null) { + item.decRef(); + } + } + } + return droppedToZero; } @Override @@ -189,6 +203,9 @@ public void close() { } private static class BulkRequestRefCounted extends AbstractRefCounted { + // BulkRequestRefCounted() { + // super(true); + // } @Override protected void closeInternal() { // nothing to close diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 08f74d996c2b2..8aebd86dce9e2 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -55,6 +55,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Assertions; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -295,10 +296,12 @@ public void onTimeout(TimeValue timeout) { } private void forkAndExecute(Task task, BulkRequest bulkRequest, String executorName, ActionListener releasingListener) { - threadPool.executor(Names.WRITE).execute(new ActionRunnable<>(releasingListener) { + bulkRequest.incRef(); + ActionListener listener = ActionListener.runAfter(releasingListener, bulkRequest::decRef); + threadPool.executor(Names.WRITE).execute(new ActionRunnable<>(listener) { @Override protected void doRun() { - doInternalExecute(task, bulkRequest, executorName, releasingListener); + doInternalExecute(task, bulkRequest, executorName, listener); } }); } @@ -473,6 +476,9 @@ private static void failRequestsWhenPrerequisiteActionFailed( for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest request = bulkRequest.requests.get(i); if (request != null && setResponseFailureIfIndexMatches(responses, i, request, target, error)) { + if (bulkRequest.requests.get(i) instanceof RefCounted refCounted) { + refCounted.decRef(); + } bulkRequest.requests.set(i, null); } } @@ -716,6 +722,9 @@ private Map> groupRequestsByShards(ClusterState c BulkItemResponse bulkItemResponse = BulkItemResponse.failure(i, docWriteRequest.opType(), failure); responses.set(i, bulkItemResponse); // make sure the request gets never processed again + if (bulkRequest.requests.get(i) instanceof RefCounted refCounted) { + refCounted.decRef(); + } bulkRequest.requests.set(i, null); } } @@ -749,6 +758,10 @@ private void executeBulkRequestsByShard(Map> requ bulkRequest.getRefreshPolicy(), requests.toArray(new BulkItemRequest[0]) ); + for (BulkItemRequest request : requests) { + // The BulkShardRequest constructor has incremented the ref, and we are no longer directly referencing this object + request.decRef(); + } bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); bulkShardRequest.timeout(bulkRequest.timeout()); bulkShardRequest.routedBasedOnClusterVersion(clusterState.version()); @@ -897,6 +910,9 @@ private void addFailure(DocWriteRequest request, int idx, Exception unavailab responses.set(idx, bulkItemResponse); // make sure the request gets never processed again bulkRequest.requests.set(idx, null); + if (bulkRequest.requests.get(idx) instanceof RefCounted refCounted) { + refCounted.decRef(); + } } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index e6d5bdcc46696..413e1c76fe614 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -263,7 +264,8 @@ protected void doRun() { docWriteRequest.id() ), context, - null + null, + () -> {} ); } finishRequest(); @@ -310,10 +312,19 @@ static boolean executeBulkItemRequest( // Translate update requests into index or delete requests which can be executed directly final UpdateHelper.Result updateResult; + final Runnable releaseRequest; if (opType == DocWriteRequest.OpType.UPDATE) { final UpdateRequest updateRequest = (UpdateRequest) context.getCurrent(); try { updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier); + if (updateResult != null + && updateResult.action() != updateRequest.doc() + && updateResult.action() != updateRequest.upsertRequest() + && updateResult.action() instanceof RefCounted refCounted) { + releaseRequest = refCounted::decRef; + } else { + releaseRequest = () -> {}; + } } catch (Exception failure) { // we may fail translating a update to index or delete operation // we use index result to communicate failure while translating update request @@ -332,6 +343,7 @@ static boolean executeBulkItemRequest( } else { context.setRequestToExecute(context.getCurrent()); updateResult = null; + releaseRequest = () -> {}; } assert context.getRequestToExecute() != null; // also checks that we're in TRANSLATED state @@ -370,7 +382,6 @@ static boolean executeBulkItemRequest( ); } if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) { - try { Optional mergedSource = Optional.ofNullable( primary.mapperService() @@ -384,19 +395,20 @@ static boolean executeBulkItemRequest( .map(DocumentMapper::mappingSource); if (mergedSource.equals(previousSource)) { + releaseRequest.run(); context.resetForNoopMappingUpdateRetry(primary.mapperService().mappingVersion()); return true; } } catch (Exception e) { logger.info(() -> format("%s mapping update rejected by primary", primary.shardId()), e); assert result.getId() != null; - onComplete(exceptionToResult(e, primary, isDelete, version, result.getId()), context, updateResult); + onComplete(exceptionToResult(e, primary, isDelete, version, result.getId()), context, updateResult, releaseRequest); return true; } - mappingUpdater.updateMappings(result.getRequiredMappingUpdate(), primary.shardId(), new ActionListener<>() { @Override public void onResponse(Void v) { + releaseRequest.run(); context.markAsRequiringMappingUpdate(); waitForMappingUpdate.accept(ActionListener.runAfter(new ActionListener<>() { @Override @@ -414,7 +426,7 @@ public void onFailure(Exception e) { @Override public void onFailure(Exception e) { - onComplete(exceptionToResult(e, primary, isDelete, version, result.getId()), context, updateResult); + onComplete(exceptionToResult(e, primary, isDelete, version, result.getId()), context, updateResult, releaseRequest); // Requesting mapping update failed, so we don't have to wait for a cluster state update assert context.isInitial(); itemDoneListener.onResponse(null); @@ -422,7 +434,7 @@ public void onFailure(Exception e) { }); return false; } else { - onComplete(result, context, updateResult); + onComplete(result, context, updateResult, releaseRequest); } return true; } @@ -432,48 +444,62 @@ private static Engine.Result exceptionToResult(Exception e, IndexShard primary, return isDelete ? primary.getFailedDeleteResult(e, version, id) : primary.getFailedIndexResult(e, version, id); } - private static void onComplete(Engine.Result r, BulkPrimaryExecutionContext context, UpdateHelper.Result updateResult) { - context.markOperationAsExecuted(r); - final DocWriteRequest docWriteRequest = context.getCurrent(); - final DocWriteRequest.OpType opType = docWriteRequest.opType(); - final boolean isUpdate = opType == DocWriteRequest.OpType.UPDATE; - final BulkItemResponse executionResult = context.getExecutionResult(); - final boolean isFailed = executionResult.isFailed(); - if (isUpdate - && isFailed - && isConflictException(executionResult.getFailure().getCause()) - && context.getUpdateRetryCounter() < ((UpdateRequest) docWriteRequest).retryOnConflict()) { - context.resetForUpdateRetry(); - return; - } - final BulkItemResponse response; - if (isUpdate) { - response = processUpdateResponse((UpdateRequest) docWriteRequest, context.getConcreteIndex(), executionResult, updateResult); - } else { - if (isFailed) { - final Exception failure = executionResult.getFailure().getCause(); - Level level; - if (TransportShardBulkAction.isConflictException(failure)) { - level = Level.TRACE; - } else { - level = Level.DEBUG; - } - logger.log( - level, - () -> Strings.format( - "%s failed to execute bulk item (%s) %s", - context.getPrimary().shardId(), - opType.getLowercase(), - docWriteRequest - ), - failure + private static void onComplete( + Engine.Result r, + BulkPrimaryExecutionContext context, + UpdateHelper.Result updateResult, + Runnable onComplete + ) { + try { + context.markOperationAsExecuted(r); + final DocWriteRequest docWriteRequest = context.getCurrent(); + final DocWriteRequest.OpType opType = docWriteRequest.opType(); + final boolean isUpdate = opType == DocWriteRequest.OpType.UPDATE; + final BulkItemResponse executionResult = context.getExecutionResult(); + final boolean isFailed = executionResult.isFailed(); + if (isUpdate + && isFailed + && isConflictException(executionResult.getFailure().getCause()) + && context.getUpdateRetryCounter() < ((UpdateRequest) docWriteRequest).retryOnConflict()) { + context.resetForUpdateRetry(); + return; + } + final BulkItemResponse response; + if (isUpdate) { + response = processUpdateResponse( + (UpdateRequest) docWriteRequest, + context.getConcreteIndex(), + executionResult, + updateResult ); + } else { + if (isFailed) { + final Exception failure = executionResult.getFailure().getCause(); + Level level; + if (TransportShardBulkAction.isConflictException(failure)) { + level = Level.TRACE; + } else { + level = Level.DEBUG; + } + logger.log( + level, + () -> Strings.format( + "%s failed to execute bulk item (%s) %s", + context.getPrimary().shardId(), + opType.getLowercase(), + docWriteRequest + ), + failure + ); + } + response = executionResult; } - response = executionResult; + context.markAsCompleted(response); + assert context.isInitial(); + } finally { + onComplete.run(); } - context.markAsCompleted(response); - assert context.isInitial(); } private static boolean isConflictException(final Exception e) { diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index eda28eb4e139e..891e8d0e4418c 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -31,13 +31,17 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.StringLiteralDeduplicator; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; +import org.elasticsearch.core.Releasable; import org.elasticsearch.index.Index; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.plugins.internal.DocumentParsingObserver; +import org.elasticsearch.transport.LeakTracker; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; @@ -69,7 +73,11 @@ * @see IndexResponse * @see org.elasticsearch.client.internal.Client#index(IndexRequest) */ -public class IndexRequest extends ReplicatedWriteRequest implements DocWriteRequest, CompositeIndicesRequest { +public class IndexRequest extends ReplicatedWriteRequest + implements + DocWriteRequest, + CompositeIndicesRequest, + Releasable { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(IndexRequest.class); private static final TransportVersion PIPELINES_HAVE_RUN_FIELD_ADDED = TransportVersions.V_8_10_X; @@ -140,6 +148,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement */ private Object rawTimestamp; private boolean pipelinesHaveRun = false; + private final RefCounted refCounted; public IndexRequest(StreamInput in) throws IOException { this(null, in); @@ -147,6 +156,7 @@ public IndexRequest(StreamInput in) throws IOException { public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOException { super(shardId, in); + this.refCounted = LeakTracker.wrap(new IndexRequestRefCounted()); if (in.getTransportVersion().before(TransportVersions.V_8_0_0)) { String type = in.readOptionalString(); assert MapperService.SINGLE_MAPPING_NAME.equals(type) : "Expected [_doc] but received [" + type + "]"; @@ -200,6 +210,7 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio public IndexRequest() { super(NO_SHARD_ID); + this.refCounted = LeakTracker.wrap(new IndexRequestRefCounted()); } /** @@ -209,6 +220,7 @@ public IndexRequest() { public IndexRequest(String index) { super(NO_SHARD_ID); this.index = index; + this.refCounted = LeakTracker.wrap(new IndexRequestRefCounted()); } private static final StringLiteralDeduplicator pipelineNameDeduplicator = new StringLiteralDeduplicator(); @@ -904,4 +916,37 @@ public List getExecutedPipelines() { return Collections.unmodifiableList(executedPipelines); } } + + @Override + public void incRef() { + refCounted.incRef(); + } + + @Override + public boolean tryIncRef() { + return refCounted.tryIncRef(); + } + + @Override + public boolean decRef() { + assert hasReferences() : "Attempt to decRef IndexRequest that is already closed"; + return refCounted.decRef(); + } + + @Override + public boolean hasReferences() { + return refCounted.hasReferences(); + } + + @Override + public void close() { + decRef(); + } + + private static class IndexRequestRefCounted extends AbstractRefCounted { + @Override + protected void closeInternal() { + // nothing to close + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index 80b7a95bbe0de..51f7bf7059c92 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -259,7 +259,18 @@ public void onTimeout(TimeValue timeout) { } private void handleShardRequest(Request request, TransportChannel channel, Task task) { - threadPool.executor(executor(request.shardId)) - .execute(ActionRunnable.wrap(new ChannelActionListener(channel), l -> shardOperation(request, l))); + request.incRef(); + try { + threadPool.executor(executor(request.shardId)) + .execute( + ActionRunnable.wrap( + ActionListener.runAfter(new ChannelActionListener(channel), request::decRef), + l -> shardOperation(request, l) + ) + ); + } catch (Exception e) { + request.decRef(); + throw e; + } } } diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index c73679a2527df..febc819e39827 100644 --- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -179,6 +179,7 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< protected void shardOperation(final UpdateRequest request, final ActionListener listener, final int retryCount) throws IOException { + assert request.hasReferences() : "Upsert request is already closed"; final ShardId shardId = request.getShardId(); final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); final IndexShard indexShard = indexService.getShard(shardId.getId()); @@ -189,76 +190,84 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< // we fetch it from the index request so we don't generate the bytes twice, its already done in the index request final BytesReference upsertSourceBytes = upsertRequest.source(); final BulkRequest bulkRequest = toSingleItemBulkRequest(upsertRequest); + final Runnable maybeDecRefIndexRequest = UpdateHelper.getMaybeDecRefUpdateHelerResult(request, result); client.bulk( bulkRequest, - ActionListener.releaseAfter(unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { - UpdateResponse update = new UpdateResponse( - response.getShardInfo(), - response.getShardId(), - response.getId(), - response.getSeqNo(), - response.getPrimaryTerm(), - response.getVersion(), - response.getResult() - ); - if (request.fetchSource() != null && request.fetchSource().fetchSource()) { - Tuple> sourceAndContent = XContentHelper.convertToMap( - upsertSourceBytes, - true, - upsertRequest.getContentType() - ); - update.setGetResult( - UpdateHelper.extractGetResult( - request, - request.concreteIndex(), - response.getSeqNo(), - response.getPrimaryTerm(), - response.getVersion(), - sourceAndContent.v2(), - sourceAndContent.v1(), - upsertSourceBytes - ) + ActionListener.runAfter( + ActionListener.runAfter(unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { + UpdateResponse update = new UpdateResponse( + response.getShardInfo(), + response.getShardId(), + response.getId(), + response.getSeqNo(), + response.getPrimaryTerm(), + response.getVersion(), + response.getResult() ); - } else { - update.setGetResult(null); - } - update.setForcedRefresh(response.forcedRefresh()); - listener.onResponse(update); - }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount))), bulkRequest) + if (request.fetchSource() != null && request.fetchSource().fetchSource()) { + Tuple> sourceAndContent = XContentHelper.convertToMap( + upsertSourceBytes, + true, + upsertRequest.getContentType() + ); + update.setGetResult( + UpdateHelper.extractGetResult( + request, + request.concreteIndex(), + response.getSeqNo(), + response.getPrimaryTerm(), + response.getVersion(), + sourceAndContent.v2(), + sourceAndContent.v1(), + upsertSourceBytes + ) + ); + } else { + update.setGetResult(null); + } + update.setForcedRefresh(response.forcedRefresh()); + listener.onResponse(update); + }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount))), bulkRequest::decRef), + maybeDecRefIndexRequest + ) ); } case UPDATED -> { IndexRequest indexRequest = result.action(); + final Runnable maybeDecRefIndexRequest = UpdateHelper.getMaybeDecRefUpdateHelerResult(request, result); // we fetch it from the index request so we don't generate the bytes twice, its already done in the index request final BytesReference indexSourceBytes = indexRequest.source(); final BulkRequest bulkRequest = toSingleItemBulkRequest(indexRequest); client.bulk( bulkRequest, - ActionListener.releaseAfter(unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { - UpdateResponse update = new UpdateResponse( - response.getShardInfo(), - response.getShardId(), - response.getId(), - response.getSeqNo(), - response.getPrimaryTerm(), - response.getVersion(), - response.getResult() - ); - update.setGetResult( - UpdateHelper.extractGetResult( - request, - request.concreteIndex(), + ActionListener.runAfter( + ActionListener.releaseAfter(unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { + UpdateResponse update = new UpdateResponse( + response.getShardInfo(), + response.getShardId(), + response.getId(), response.getSeqNo(), response.getPrimaryTerm(), response.getVersion(), - result.updatedSourceAsMap(), - result.updateSourceContentType(), - indexSourceBytes - ) - ); - update.setForcedRefresh(response.forcedRefresh()); - listener.onResponse(update); - }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount))), bulkRequest) + response.getResult() + ); + update.setGetResult( + UpdateHelper.extractGetResult( + request, + request.concreteIndex(), + response.getSeqNo(), + response.getPrimaryTerm(), + response.getVersion(), + result.updatedSourceAsMap(), + result.updateSourceContentType(), + indexSourceBytes + ) + ); + update.setForcedRefresh(response.forcedRefresh()); + listener.onResponse(update); + }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount))), bulkRequest), + maybeDecRefIndexRequest + ) ); } case DELETED -> { @@ -290,19 +299,23 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< ); update.setForcedRefresh(response.forcedRefresh()); listener.onResponse(update); - }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount))), bulkRequest) + }, exception -> { handleUpdateFailureWithRetry(listener, request, exception, retryCount); })), bulkRequest) ); } case NOOP -> { UpdateResponse update = result.action(); - IndexService indexServiceOrNull = indicesService.indexService(shardId.getIndex()); - if (indexServiceOrNull != null) { - IndexShard shard = indexService.getShardOrNull(shardId.getId()); - if (shard != null) { - shard.noopUpdate(); + try { + IndexService indexServiceOrNull = indicesService.indexService(shardId.getIndex()); + if (indexServiceOrNull != null) { + IndexShard shard = indexService.getShardOrNull(shardId.getId()); + if (shard != null) { + shard.noopUpdate(); + } } + listener.onResponse(update); + } finally { + update.decRef(); } - listener.onResponse(update); } default -> throw new IllegalStateException("Illegal result " + result.getResponseResult()); } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index de6cffa10f48d..ea28e6da10691 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.DocumentMissingException; @@ -53,7 +54,10 @@ public UpdateHelper(ScriptService scriptService) { } /** - * Prepares an update request by converting it into an index or delete request or an update response (no action). + * Prepares an update request by converting it into an index or delete request or an update response (no action). The caller must be + * very careful with the response from this method, because sometimes it contains a new index request that must be decRef'd when the + * client is finished with it, and sometimes it contains a reference to an index request already referenced from the update request. + * It is unfortunately on the client to check whether or not this IndexRequest is the same as one already on the input UpdateRequest. */ public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis) throws IOException { final GetResult getResult = indexShard.getService().getForUpdate(request.id(), request.ifSeqNo(), request.ifPrimaryTerm()); @@ -80,6 +84,29 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult } } + /** + * This is a helper for the prepare* methods of this class. The prepare* methods return a Result that might contain a newly-created + * IndexRequest, or might return an IndexRequest that already existed on the UpdateRequest as the doc or upsertRequest. If the + * former, then the caller needs to make sure to decRef the IndexRequest in the Result once done with it. This method returns a + * Runnable that will decref the IndexRequest if needed. + * @param request The UpdateRequest previously passed to a prepare* method on this class + * @param result The Result previously returned by a prepare* method on this class + * @return A Runnable that can be used to decref any IndexRequests created by the prepare* method in this class (possibly a no-op). + */ + public static Runnable getMaybeDecRefUpdateHelerResult(UpdateRequest request, Result result) { + final Runnable maybeDecRefIndexRequest; + if (result.action instanceof RefCounted refCounted) { + if (refCounted != request && refCounted != request.doc() && refCounted != request.upsertRequest()) { + maybeDecRefIndexRequest = refCounted::decRef; + } else { + maybeDecRefIndexRequest = () -> {}; + } + } else { + maybeDecRefIndexRequest = () -> {}; + } + return maybeDecRefIndexRequest; + } + /** * Execute a scripted upsert, where there is an existing upsert document and a script to be executed. The script is executed and a new * Tuple of operation and updated {@code _source} is returned. diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 73cb6ebe2e39a..5986bbb368de2 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -25,13 +25,16 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.transport.LeakTracker; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -126,8 +129,11 @@ public class UpdateRequest extends InstanceShardOperationRequest @Nullable private IndexRequest doc; + private final RefCounted refCounted; - public UpdateRequest() {} + public UpdateRequest() { + this.refCounted = LeakTracker.wrap(new UpdateRequestRefCounted()); + } public UpdateRequest(StreamInput in) throws IOException { this(null, in); @@ -164,11 +170,13 @@ public UpdateRequest(@Nullable ShardId shardId, StreamInput in) throws IOExcepti } else { requireAlias = false; } + this.refCounted = LeakTracker.wrap(new UpdateRequestRefCounted()); } public UpdateRequest(String index, String id) { super(index); this.id = id; + this.refCounted = LeakTracker.wrap(new UpdateRequestRefCounted()); } @Override @@ -616,6 +624,7 @@ public UpdateRequest waitForActiveShards(final int waitForActiveShards) { */ public UpdateRequest doc(IndexRequest doc) { this.doc = doc; + this.doc.incRef(); return this; } @@ -710,6 +719,7 @@ private IndexRequest safeDoc() { */ public UpdateRequest upsert(IndexRequest upsertRequest) { this.upsertRequest = upsertRequest; + this.upsertRequest.incRef(); return this; } @@ -1003,4 +1013,42 @@ public long ramBytesUsed() { } return SHALLOW_SIZE + RamUsageEstimator.sizeOf(id) + childRequestBytes; } + + @Override + public void incRef() { + refCounted.incRef(); + } + + @Override + public boolean tryIncRef() { + return refCounted.tryIncRef(); + } + + @Override + public boolean decRef() { + assert refCounted.hasReferences() : "Attempt to decRef UpdateRequest that is already closed"; + boolean droppedToZero = refCounted.decRef(); + if (droppedToZero) { + if (doc != null) { + doc.decRef(); + } + if (upsertRequest != null) { + upsertRequest.decRef(); + } + } + return droppedToZero; + } + + @Override + public boolean hasReferences() { + return refCounted.hasReferences(); + } + + private static class UpdateRequestRefCounted extends AbstractRefCounted { + + @Override + protected void closeInternal() { + // nothing to close + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java index a1f741d7d51d6..832e84041450e 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java @@ -19,6 +19,8 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.AbstractRefCounted; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.VersionType; @@ -26,6 +28,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.transport.LeakTracker; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; @@ -59,9 +62,11 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest request, private static void failOnSizeSpecified() { throw new IllegalArgumentException("invalid parameter [size], use [max_docs] instead"); } + + @Override + public void incRef() { + refCounted.incRef(); + } + + @Override + public boolean tryIncRef() { + return refCounted.tryIncRef(); + } + + @Override + public boolean decRef() { + boolean success = refCounted.decRef(); + if (refCounted.hasReferences() == false) { + success = destination.decRef() && success; + } + return success; + } + + @Override + public boolean hasReferences() { + return refCounted.hasReferences(); + } + + private static class ReindexRequestRefCounted extends AbstractRefCounted { + + @Override + protected void closeInternal() { + // nothing to close + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java index 88a851bee15e0..3c13e3e774c61 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java @@ -17,6 +17,7 @@ public class ReindexRequestBuilder extends AbstractBulkIndexByScrollRequestBuild public ReindexRequestBuilder(ElasticsearchClient client) { this(client, new SearchRequestBuilder(client), new IndexRequestBuilder(client)); + request.getDestination().decRef(); } private ReindexRequestBuilder(ElasticsearchClient client, SearchRequestBuilder search, IndexRequestBuilder destination) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index cdda3ea38129f..f524083788b4d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.rest.action.document; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.index.IndexRequest; @@ -122,31 +123,41 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } IndexRequest indexRequest = new IndexRequest(request.param("index")); - indexRequest.id(request.param("id")); - indexRequest.routing(request.param("routing")); - indexRequest.setPipeline(request.param("pipeline")); - indexRequest.source(request.requiredContent(), request.getXContentType()); - indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT)); - indexRequest.setRefreshPolicy(request.param("refresh")); - indexRequest.version(RestActions.parseVersion(request)); - indexRequest.versionType(VersionType.fromString(request.param("version_type"), indexRequest.versionType())); - indexRequest.setIfSeqNo(request.paramAsLong("if_seq_no", indexRequest.ifSeqNo())); - indexRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", indexRequest.ifPrimaryTerm())); - indexRequest.setRequireAlias(request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, indexRequest.isRequireAlias())); - indexRequest.setRequireDataStream(request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, indexRequest.isRequireDataStream())); - String sOpType = request.param("op_type"); - String waitForActiveShards = request.param("wait_for_active_shards"); - if (waitForActiveShards != null) { - indexRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); - } - if (sOpType != null) { - indexRequest.opType(sOpType); - } + try { + indexRequest.id(request.param("id")); + indexRequest.routing(request.param("routing")); + indexRequest.setPipeline(request.param("pipeline")); + indexRequest.source(request.requiredContent(), request.getXContentType()); + indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT)); + indexRequest.setRefreshPolicy(request.param("refresh")); + indexRequest.version(RestActions.parseVersion(request)); + indexRequest.versionType(VersionType.fromString(request.param("version_type"), indexRequest.versionType())); + indexRequest.setIfSeqNo(request.paramAsLong("if_seq_no", indexRequest.ifSeqNo())); + indexRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", indexRequest.ifPrimaryTerm())); + indexRequest.setRequireAlias(request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, indexRequest.isRequireAlias())); + indexRequest.setRequireDataStream( + request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, indexRequest.isRequireDataStream()) + ); + String sOpType = request.param("op_type"); + String waitForActiveShards = request.param("wait_for_active_shards"); + if (waitForActiveShards != null) { + indexRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); + } + if (sOpType != null) { + indexRequest.opType(sOpType); + } - return channel -> client.index( - indexRequest, - new RestToXContentListener<>(channel, DocWriteResponse::status, r -> r.getLocation(indexRequest.routing())) - ); + return channel -> client.index( + indexRequest, + ActionListener.runAfter( + new RestToXContentListener<>(channel, DocWriteResponse::status, r -> r.getLocation(indexRequest.routing())), + indexRequest::decRef + ) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java index aff71cba2d858..ba5637d169f95 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.rest.action.document; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexRequest; @@ -54,52 +55,59 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC request.param("type"); } UpdateRequest updateRequest = new UpdateRequest(request.param("index"), request.param("id")); - updateRequest.routing(request.param("routing")); - updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout())); - updateRequest.setRefreshPolicy(request.param("refresh")); - String waitForActiveShards = request.param("wait_for_active_shards"); - if (waitForActiveShards != null) { - updateRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); - } - updateRequest.docAsUpsert(request.paramAsBoolean("doc_as_upsert", updateRequest.docAsUpsert())); - FetchSourceContext fetchSourceContext = FetchSourceContext.parseFromRestRequest(request); - if (fetchSourceContext != null) { - updateRequest.fetchSource(fetchSourceContext); - } - - updateRequest.retryOnConflict(request.paramAsInt("retry_on_conflict", updateRequest.retryOnConflict())); - if (request.hasParam("version") || request.hasParam("version_type")) { - final ActionRequestValidationException versioningError = new ActionRequestValidationException(); - versioningError.addValidationError( - "internal versioning can not be used for optimistic concurrency control. " - + "Please use `if_seq_no` and `if_primary_term` instead" - ); - throw versioningError; - } - - updateRequest.setIfSeqNo(request.paramAsLong("if_seq_no", updateRequest.ifSeqNo())); - updateRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", updateRequest.ifPrimaryTerm())); - updateRequest.setRequireAlias(request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, updateRequest.isRequireAlias())); - - request.applyContentParser(parser -> { - updateRequest.fromXContent(parser); - IndexRequest upsertRequest = updateRequest.upsertRequest(); - if (upsertRequest != null) { - upsertRequest.routing(request.param("routing")); - upsertRequest.version(RestActions.parseVersion(request)); - upsertRequest.versionType(VersionType.fromString(request.param("version_type"), upsertRequest.versionType())); + try { + updateRequest.routing(request.param("routing")); + updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout())); + updateRequest.setRefreshPolicy(request.param("refresh")); + String waitForActiveShards = request.param("wait_for_active_shards"); + if (waitForActiveShards != null) { + updateRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); + } + updateRequest.docAsUpsert(request.paramAsBoolean("doc_as_upsert", updateRequest.docAsUpsert())); + FetchSourceContext fetchSourceContext = FetchSourceContext.parseFromRestRequest(request); + if (fetchSourceContext != null) { + updateRequest.fetchSource(fetchSourceContext); } - IndexRequest doc = updateRequest.doc(); - if (doc != null) { - doc.routing(request.param("routing")); - doc.version(RestActions.parseVersion(request)); - doc.versionType(VersionType.fromString(request.param("version_type"), doc.versionType())); + + updateRequest.retryOnConflict(request.paramAsInt("retry_on_conflict", updateRequest.retryOnConflict())); + if (request.hasParam("version") || request.hasParam("version_type")) { + final ActionRequestValidationException versioningError = new ActionRequestValidationException(); + versioningError.addValidationError( + "internal versioning can not be used for optimistic concurrency control. " + + "Please use `if_seq_no` and `if_primary_term` instead" + ); + throw versioningError; } - }); + updateRequest.setIfSeqNo(request.paramAsLong("if_seq_no", updateRequest.ifSeqNo())); + updateRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", updateRequest.ifPrimaryTerm())); + updateRequest.setRequireAlias(request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, updateRequest.isRequireAlias())); + + request.applyContentParser(parser -> { + updateRequest.fromXContent(parser); + IndexRequest upsertRequest = updateRequest.upsertRequest(); + if (upsertRequest != null) { + upsertRequest.routing(request.param("routing")); + upsertRequest.version(RestActions.parseVersion(request)); + upsertRequest.versionType(VersionType.fromString(request.param("version_type"), upsertRequest.versionType())); + } + IndexRequest doc = updateRequest.doc(); + if (doc != null) { + doc.routing(request.param("routing")); + doc.version(RestActions.parseVersion(request)); + doc.versionType(VersionType.fromString(request.param("version_type"), doc.versionType())); + } + }); + } catch (Exception e) { + updateRequest.decRef(); + throw e; + } return channel -> client.update( updateRequest, - new RestToXContentListener<>(channel, UpdateResponse::status, r -> r.getLocation(updateRequest.routing())) + ActionListener.runAfter( + new RestToXContentListener<>(channel, UpdateResponse::status, r -> r.getLocation(updateRequest.routing())), + updateRequest::decRef + ) ); } diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java index a72ef9e83ccf2..cb875da54253c 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java @@ -87,7 +87,7 @@ public void storeResult(TaskResult taskResult, ActionListener listener) { } catch (IOException e) { throw new ElasticsearchException("Couldn't convert task result to XContent for [{}]", e, taskResult.getTask()); } - doStoreResult(STORE_BACKOFF_POLICY.iterator(), index, listener); + doStoreResult(STORE_BACKOFF_POLICY.iterator(), index, ActionListener.runAfter(listener, () -> index.request().decRef())); } private void doStoreResult(Iterator backoff, IndexRequestBuilder index, ActionListener listener) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java index 065dea4be743c..8effa2859267e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java @@ -8,6 +8,8 @@ package org.elasticsearch.action.admin.indices.segments; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.MergePolicyConfig; @@ -39,7 +41,14 @@ public void setupIndex() { int numDocs = scaledRandomIntBetween(100, 1000); for (int j = 0; j < numDocs; ++j) { String id = Integer.toString(j); - prepareIndex("test").setId(id).setSource("text", "sometext").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(id).setSource("text", "sometext"); + IndexRequest indexRequest = indexRequestBuilder.request(); + try { + indexRequestBuilder.get(); + } finally { + indexRequest.decRef(); + } + } client().admin().indices().prepareFlush("test").get(); client().admin().indices().prepareRefresh().get(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java index 661727d832747..e8f34b3dd15da 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; @@ -72,7 +73,9 @@ public void testSegmentStats() throws Exception { .setSettings(Settings.builder().put("index.store.type", storeType.getSettingsKey())) ); ensureGreen("test"); - prepareIndex("test").setId("1").setSource("foo", "bar", "bar", "baz", "baz", 42).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("foo", "bar", "bar", "baz", "baz", 42); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); indicesAdmin().prepareRefresh("test").get(); IndicesStatsResponse rsp = indicesAdmin().prepareStats("test").get(); @@ -80,7 +83,9 @@ public void testSegmentStats() throws Exception { assertThat(stats.getCount(), greaterThan(0L)); // now check multiple segments stats are merged together - prepareIndex("test").setId("2").setSource("foo", "bar", "bar", "baz", "baz", 43).get(); + indexRequestBuilder = prepareIndex("test").setId("2").setSource("foo", "bar", "bar", "baz", "baz", 43); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); indicesAdmin().prepareRefresh("test").get(); rsp = indicesAdmin().prepareStats("test").get(); @@ -107,10 +112,10 @@ public void testRefreshListeners() throws Exception { createIndex("test", Settings.builder().put("refresh_interval", -1).build()); // Index a document asynchronously so the request will only return when document is refreshed - ActionFuture index = prepareIndex("test").setId("test") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("test") .setSource("test", "test") - .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) - .execute(); + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + ActionFuture index = indexRequestBuilder.execute(); // Wait for the refresh listener to appear in the stats. Wait a long time because NFS tests can be quite slow! logger.info("starting to wait"); @@ -132,6 +137,7 @@ public void testRefreshListeners() throws Exception { // Refresh the index and wait for the request to come back indicesAdmin().prepareRefresh("test").get(); index.get(); + indexRequestBuilder.request().decRef(); // The document should appear in the statistics and the refresh listener should be gone IndicesStatsResponse stats = indicesAdmin().prepareStats("test").clear().setRefresh(true).setDocs(true).get(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java index 0a41451cbc57b..cbca12b76f9cf 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; @@ -69,11 +70,25 @@ private BulkShardRequest generateRandomRequest() { }; items[i] = new BulkItemRequest(i, request); } - return new BulkShardRequest(new ShardId("index", "_na_", 0), randomFrom(WriteRequest.RefreshPolicy.values()), items); + BulkShardRequest bulkShardRequest = new BulkShardRequest( + new ShardId("index", "_na_", 0), + randomFrom(WriteRequest.RefreshPolicy.values()), + items + ); + /* + * We're responsible for the lifecycle of the requests and the BulkItemRequests since they were created here, so we decref them + * here, so that when the returned BulkShardRequest is decRef'd, the reference counts correctly go to 0. + */ + for (BulkItemRequest bulkItemRequest : items) { + bulkItemRequest.decRef(); + if (bulkItemRequest.request() instanceof RefCounted refCountedRequest) { + refCountedRequest.decRef(); + } + } + return bulkShardRequest; } public void testTranslogLocation() { - try (BulkShardRequest shardRequest = generateRandomRequest()) { Translog.Location expectedLocation = null; @@ -85,6 +100,7 @@ public void testTranslogLocation() { BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(shardRequest, primary); while (context.hasMoreOperationsToExecute()) { + IndexRequest nullableIndexRequest = null; final Engine.Result result; final DocWriteRequest current = context.getCurrent(); final boolean failure = rarely(); @@ -106,7 +122,9 @@ public void testTranslogLocation() { } } case UPDATE -> { - context.setRequestToExecute(new IndexRequest(current.index()).id(current.id())); + IndexRequest indexRequest = new IndexRequest(current.index()).id(current.id()); + context.setRequestToExecute(indexRequest); + nullableIndexRequest = indexRequest; if (failure) { result = new Engine.IndexResult(new ElasticsearchException("bla"), 1, 1, 1, current.id()); } else { @@ -128,6 +146,9 @@ public void testTranslogLocation() { } context.markOperationAsExecuted(result); context.markAsCompleted(context.getExecutionResult()); + if (nullableIndexRequest != null) { + nullableIndexRequest.decRef(); + } } assertThat(context.getLocationToSync(), equalTo(expectedLocation)); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessor2Tests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessor2Tests.java index 30810e85382ca..fc9991bc7f463 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessor2Tests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessor2Tests.java @@ -103,7 +103,12 @@ public void afterBulk(long executionId, BulkRequest request, Exception failure) .setFlushInterval(TimeValue.timeValueMillis(1)) .build(); try { - bulkProcessor.add(new IndexRequest()); + IndexRequest indexRequest = new IndexRequest(); + try { + bulkProcessor.add(indexRequest); + } finally { + indexRequest.decRef(); + } assertTrue(countDownLatch.await(5, TimeUnit.MINUTES)); assertThat(bulkProcessor.getTotalBytesInFlight(), equalTo(0L)); } finally { @@ -180,8 +185,12 @@ public void testConcurrentExecutions() throws Exception { ); try { IndexRequest indexRequest = new IndexRequest(); - for (final AtomicInteger i = new AtomicInteger(0); i.getAndIncrement() < maxDocuments;) { - bulkProcessor.add(indexRequest); + try { + for (final AtomicInteger i = new AtomicInteger(0); i.getAndIncrement() < maxDocuments;) { + bulkProcessor.add(indexRequest); + } + } finally { + indexRequest.decRef(); } assertBusy(() -> { @@ -289,6 +298,7 @@ public void testConcurrentExecutionsWithFlush() throws Exception { IndexRequest indexRequest = new IndexRequest(); List> futures = new ArrayList<>(); CountDownLatch startGate = new CountDownLatch(1 + concurrentClients); + CountDownLatch allIndexRequestsAdded = new CountDownLatch(maxDocuments); for (final AtomicInteger i = new AtomicInteger(0); i.getAndIncrement() < maxDocuments;) { futures.add(executorService.submit(() -> { try { @@ -297,6 +307,7 @@ public void testConcurrentExecutionsWithFlush() throws Exception { startGate.await(); // alternate between ways to add to the bulk processor bulkProcessor.add(indexRequest); + allIndexRequestsAdded.countDown(); } catch (Exception e) { throw ExceptionsHelper.convertToRuntime(e); } @@ -304,6 +315,8 @@ public void testConcurrentExecutionsWithFlush() throws Exception { } startGate.countDown(); startGate.await(); + allIndexRequestsAdded.await(); + indexRequest.decRef(); for (Future f : futures) { try { @@ -399,6 +412,7 @@ public void testRejections() throws Exception { break; } } + indexRequest.decRef(); assertThat(rejectedRequests, equalTo(true)); } finally { bulkProcessor.awaitClose(1, TimeUnit.SECONDS); @@ -448,7 +462,12 @@ public void afterBulk(long executionId, BulkRequest request, Exception failure) .build(); try { for (int i = 0; i < numberOfRequests; i++) { - bulkProcessor.add(new IndexRequest().source(Collections.singletonMap("this_is_a_key" + i, "this_is_a_value" + i))); + IndexRequest indexRequest = new IndexRequest().source(Collections.singletonMap("this_is_a_key" + i, "this_is_a_value" + i)); + try { + bulkProcessor.add(indexRequest); + } finally { + indexRequest.decRef(); + } } } catch (EsRejectedExecutionException e) { // We are throwing more data at the processor than the max bytes in flight setting can handle @@ -507,19 +526,20 @@ public void afterBulk(long executionId, BulkRequest request, Exception failure) final AtomicBoolean haveSeenRejections = new AtomicBoolean(false); try { for (int i = 0; i < numberOfRequests; i++) { - bulkProcessor.addWithBackpressure( - new IndexRequest().source(Collections.singletonMap("this_is_a_key" + i, "this_is_a_value" + i)), - abort::get - ); + IndexRequest indexRequest = new IndexRequest().source(Collections.singletonMap("this_is_a_key" + i, "this_is_a_value" + i)); + bulkProcessor.addWithBackpressure(indexRequest, abort::get); + indexRequest.decRef(); } assertTrue(countDownLatch.await(5, TimeUnit.MINUTES)); assertThat(bulkProcessor.getTotalBytesInFlight(), equalTo(0L)); abort.set(true); try { - bulkProcessor.addWithBackpressure( - new IndexRequest().source(Collections.singletonMap("this_is_a_key", "this_is_a_value")), - abort::get - ); + IndexRequest indexRequest = new IndexRequest().source(Collections.singletonMap("this_is_a_key", "this_is_a_value")); + try { + bulkProcessor.addWithBackpressure(indexRequest, abort::get); + } finally { + indexRequest.decRef(); + } } catch (EsRejectedExecutionException e) { haveSeenRejections.set(true); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java index b3a1006fe49d1..5c04bf77a4601 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java @@ -108,7 +108,9 @@ public void testBulkProcessorFlushPreservesContext() throws InterruptedException assertNull(threadPool.getThreadContext().getTransient(transientKey)); // add a single item which won't be over the size or number of items - bulkProcessor.add(new IndexRequest()); + IndexRequest indexRequest = new IndexRequest(); + bulkProcessor.add(indexRequest); + indexRequest.decRef(); // wait for flush to execute latch.await(); @@ -160,7 +162,9 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) .setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.ZERO, Integer.MAX_VALUE)) .build() ) { - bulkProcessor.add(new IndexRequest()); + IndexRequest indexRequest = new IndexRequest(); + bulkProcessor.add(indexRequest); + indexRequest.decRef(); bulkProcessor.flush(); assertTrue(countDownLatch.await(5, TimeUnit.SECONDS)); } @@ -233,6 +237,7 @@ public void testConcurrentExecutions() throws Exception { ExecutorService executorService = Executors.newFixedThreadPool(concurrentClients); CountDownLatch startGate = new CountDownLatch(1 + concurrentClients); + CountDownLatch allIndexRequestsAdded = new CountDownLatch(maxDocuments); IndexRequest indexRequest = new IndexRequest(); String bulkRequest = """ @@ -255,6 +260,7 @@ public void testConcurrentExecutions() throws Exception { } else { bulkProcessor.add(bytesReference, null, null, XContentType.JSON); } + allIndexRequestsAdded.countDown(); } catch (Exception e) { throw ExceptionsHelper.convertToRuntime(e); } @@ -262,6 +268,8 @@ public void testConcurrentExecutions() throws Exception { } startGate.countDown(); startGate.await(); + allIndexRequestsAdded.await(); + indexRequest.decRef(); for (Future f : futures) { try { @@ -371,6 +379,7 @@ public void testConcurrentExecutionsWithFlush() throws Exception { ); List> futures = new ArrayList<>(); CountDownLatch startGate = new CountDownLatch(1 + concurrentClients); + CountDownLatch allIndexRequestsAdded = new CountDownLatch(maxDocuments); for (final AtomicInteger i = new AtomicInteger(0); i.getAndIncrement() < maxDocuments;) { futures.add(executorService.submit(() -> { try { @@ -383,6 +392,7 @@ public void testConcurrentExecutionsWithFlush() throws Exception { } else { bulkProcessor.add(bytesReference, null, null, XContentType.JSON); } + allIndexRequestsAdded.countDown(); } catch (Exception e) { throw ExceptionsHelper.convertToRuntime(e); } @@ -390,7 +400,8 @@ public void testConcurrentExecutionsWithFlush() throws Exception { } startGate.countDown(); startGate.await(); - + allIndexRequestsAdded.await(); + indexRequest.decRef(); for (Future f : futures) { try { f.get(); @@ -487,7 +498,9 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) .setFlushCondition(flushEnabled::get) .build() ) { - bulkProcessor.add(new IndexRequest()); + IndexRequest indexRequest = new IndexRequest(); + bulkProcessor.add(indexRequest); + indexRequest.decRef(); bulkProcessor.flush(); assertThat(attemptRef.get(), equalTo(0)); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java index 2acd0b5d6bebf..1aaa6dc3b4297 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java @@ -34,7 +34,12 @@ public void testBulkRequestModifier() { int numRequests = scaledRandomIntBetween(8, 64); try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < numRequests; i++) { - bulkRequest.add(new IndexRequest("_index").id(String.valueOf(i)).source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest("_index").id(String.valueOf(i)).source("{}", XContentType.JSON); + try { + bulkRequest.add(indexRequest); + } finally { + indexRequest.decRef(); + } } // wrap the bulk request and fail some of the item requests at random @@ -91,7 +96,12 @@ public void testBulkRequestModifier() { public void testPipelineFailures() { try (BulkRequest originalBulkRequest = new BulkRequest()) { for (int i = 0; i < 32; i++) { - originalBulkRequest.add(new IndexRequest("index").id(String.valueOf(i))); + IndexRequest indexRequest = new IndexRequest("index").id(String.valueOf(i)); + try { + originalBulkRequest.add(indexRequest); + } finally { + indexRequest.decRef(); + } } TransportBulkAction.BulkRequestModifier modifier = new TransportBulkAction.BulkRequestModifier(originalBulkRequest); @@ -151,7 +161,12 @@ public void onFailure(Exception e) {} public void testNoFailures() { try (BulkRequest originalBulkRequest = new BulkRequest()) { for (int i = 0; i < 32; i++) { - originalBulkRequest.add(new IndexRequest("index").id(String.valueOf(i))); + IndexRequest indexRequest = new IndexRequest("index").id(String.valueOf(i)); + try { + originalBulkRequest.add(indexRequest); + } finally { + indexRequest.decRef(); + } } TransportBulkAction.BulkRequestModifier modifier = new TransportBulkAction.BulkRequestModifier(originalBulkRequest); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index d2dae1c20d8c9..3c07ec1104872 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -130,10 +130,14 @@ public void testBulkAllowExplicitIndex() throws Exception { public void testBulkAddIterable() { try (BulkRequest bulkRequest = new BulkRequest()) { List> requests = new ArrayList<>(); - requests.add(new IndexRequest("test").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value")); - requests.add(new UpdateRequest("test", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value")); + IndexRequest indexRequest = new IndexRequest("test").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + requests.add(indexRequest); + UpdateRequest updateRequest = new UpdateRequest("test", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); + requests.add(updateRequest); requests.add(new DeleteRequest("test", "id")); bulkRequest.add(requests); + indexRequest.decRef(); + updateRequest.decRef(); assertThat(bulkRequest.requests().size(), equalTo(3)); assertThat(bulkRequest.requests().get(0), instanceOf(IndexRequest.class)); assertThat(bulkRequest.requests().get(1), instanceOf(UpdateRequest.class)); @@ -259,8 +263,15 @@ public void testBulkRequestWithRefresh() throws Exception { // We force here a "id is missing" validation error bulkRequest.add(new DeleteRequest("index", null).setRefreshPolicy(RefreshPolicy.IMMEDIATE)); bulkRequest.add(new DeleteRequest("index", "id").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); - bulkRequest.add(new UpdateRequest("index", "id").doc("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE)); - bulkRequest.add(new IndexRequest("index").id("id").source("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE)); + UpdateRequest updateRequest = new UpdateRequest("index", "id").doc("{}", XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE); + bulkRequest.add(updateRequest); + updateRequest.decRef(); + IndexRequest indexRequest = new IndexRequest("index").id("id") + .source("{}", XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE); + bulkRequest.add(indexRequest); + indexRequest.decRef(); ActionRequestValidationException validate = bulkRequest.validate(); assertThat(validate, notNullValue()); assertThat(validate.validationErrors(), not(empty())); @@ -280,8 +291,12 @@ public void testBulkRequestWithRefresh() throws Exception { // issue 15120 public void testBulkNoSource() throws Exception { try (BulkRequest bulkRequest = new BulkRequest()) { - bulkRequest.add(new UpdateRequest("index", "id")); - bulkRequest.add(new IndexRequest("index").id("id")); + UpdateRequest updateRequest = new UpdateRequest("index", "id"); + bulkRequest.add(updateRequest); + updateRequest.decRef(); + IndexRequest indexRequest = new IndexRequest("index").id("id"); + bulkRequest.add(indexRequest); + indexRequest.decRef(); ActionRequestValidationException validate = bulkRequest.validate(); assertThat(validate, notNullValue()); assertThat(validate.validationErrors(), not(empty())); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/Retry2Tests.java b/server/src/test/java/org/elasticsearch/action/bulk/Retry2Tests.java index fde1f15d97d89..971d848b9fe00 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/Retry2Tests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/Retry2Tests.java @@ -70,11 +70,13 @@ public void tearDown() throws Exception { private BulkRequest createBulkRequest() { BulkRequest request = new BulkRequest(); - request.add(new UpdateRequest("shop", "1").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))); - request.add(new UpdateRequest("shop", "2").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))); - request.add(new UpdateRequest("shop", "3").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))); - request.add(new UpdateRequest("shop", "4").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))); - request.add(new UpdateRequest("shop", "5").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))); + for (int i = 0; i < 5; i++) { + UpdateRequest updateRequest = new UpdateRequest("shop", Integer.toString(i + 1)).doc( + Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)) + ); + request.add(updateRequest); + updateRequest.decRef(); + } return request; } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/RetryTests.java b/server/src/test/java/org/elasticsearch/action/bulk/RetryTests.java index 83000856045c5..57e0e6ce75fc6 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/RetryTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/RetryTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteRequest.OpType; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.support.PlainActionFuture; @@ -72,6 +73,9 @@ private BulkRequest createBulkRequest() { request.add(new UpdateRequest("shop", "3")); request.add(new UpdateRequest("shop", "4")); request.add(new UpdateRequest("shop", "5")); + for (DocWriteRequest docWriteRequest : request.requests()) { + ((UpdateRequest) docWriteRequest).decRef(); + } return request; } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index ecb526a38576f..82949480a86f8 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -49,10 +49,16 @@ public class TransportBulkActionIndicesThatCannotBeCreatedTests extends ESTestCa public void testNonExceptional() { try (BulkRequest bulkRequest = new BulkRequest()) { - bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); - bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); + IndexRequest indexRequest1 = new IndexRequest(randomAlphaOfLength(5)); + bulkRequest.add(indexRequest1); + indexRequest1.decRef(); + IndexRequest indexRequest2 = new IndexRequest(randomAlphaOfLength(5)); + bulkRequest.add(indexRequest2); + indexRequest2.decRef(); bulkRequest.add(new DeleteRequest(randomAlphaOfLength(5))); - bulkRequest.add(new UpdateRequest(randomAlphaOfLength(5), randomAlphaOfLength(5))); + UpdateRequest updateRequest = new UpdateRequest(randomAlphaOfLength(5), randomAlphaOfLength(5)); + bulkRequest.add(updateRequest); + updateRequest.decRef(); // Test emulating that index can be auto-created indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true, noop); // Test emulating that index cannot be auto-created @@ -64,10 +70,16 @@ public void testNonExceptional() { public void testAllFail() { try (BulkRequest bulkRequest = new BulkRequest()) { - bulkRequest.add(new IndexRequest("no")); - bulkRequest.add(new IndexRequest("can't")); + IndexRequest indexRequest1 = new IndexRequest("no"); + bulkRequest.add(indexRequest1); + indexRequest1.decRef(); + IndexRequest indexRequest2 = new IndexRequest("can't"); + bulkRequest.add(indexRequest2); + indexRequest2.decRef(); bulkRequest.add(new DeleteRequest("do").version(0).versionType(VersionType.EXTERNAL)); - bulkRequest.add(new UpdateRequest("nothin", randomAlphaOfLength(5))); + UpdateRequest updateRequest = new UpdateRequest("nothin", randomAlphaOfLength(5)); + bulkRequest.add(updateRequest); + updateRequest.decRef(); indicesThatCannotBeCreatedTestCase(Set.of("no", "can't", "do", "nothin"), bulkRequest, index -> true, index -> { throw new IndexNotFoundException("Can't make it because I say so"); }); @@ -76,8 +88,12 @@ public void testAllFail() { public void testSomeFail() { try (BulkRequest bulkRequest = new BulkRequest()) { - bulkRequest.add(new IndexRequest("ok")); - bulkRequest.add(new IndexRequest("bad")); + IndexRequest indexRequest1 = new IndexRequest("ok"); + bulkRequest.add(indexRequest1); + indexRequest1.decRef(); + IndexRequest indexRequest2 = new IndexRequest("bad"); + bulkRequest.add(indexRequest2); + indexRequest2.decRef(); // Emulate auto_create_index=-bad,+* indicesThatCannotBeCreatedTestCase(Set.of("bad"), bulkRequest, index -> true, index -> { if (index.equals("bad")) { @@ -139,6 +155,7 @@ void executeBulk( Map indicesThatCannotBeCreated ) { assertEquals(expected, indicesThatCannotBeCreated.keySet()); + listener.onResponse(null); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 1b664c0297346..7f5b98468a93c 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -61,6 +61,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; @@ -245,6 +246,7 @@ public void testIngestSkipped() throws Exception { IndexRequest indexRequest = new IndexRequest("index").id("id"); indexRequest.source(Collections.emptyMap()); bulkRequest.add(indexRequest); + indexRequest.decRef(); ActionTestUtils.execute(action, null, bulkRequest, ActionTestUtils.assertNoFailureListener(response -> {})); assertTrue(action.isExecuted); verifyNoMoreInteractions(ingestService); @@ -257,6 +259,7 @@ public void testSingleItemBulkActionIngestSkipped() throws Exception { ActionTestUtils.execute(singleItemBulkWriteAction, null, indexRequest, ActionTestUtils.assertNoFailureListener(response -> {})); assertTrue(action.isExecuted); verifyNoMoreInteractions(ingestService); + indexRequest.decRef(); } public void testIngestLocal() throws Exception { @@ -270,6 +273,8 @@ public void testIngestLocal() throws Exception { indexRequest2.setPipeline("testpipeline"); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); + indexRequest1.decRef(); + indexRequest2.decRef(); AtomicBoolean responseCalled = new AtomicBoolean(false); AtomicBoolean failureCalled = new AtomicBoolean(false); @@ -382,6 +387,7 @@ public void testSingleItemBulkActionIngestLocal() throws Exception { assertTrue(action.isExecuted); assertTrue(responseCalled.get()); verifyNoMoreInteractions(transportService); + indexRequest.decRef(); } public void testIngestSystemLocal() throws Exception { @@ -395,6 +401,8 @@ public void testIngestSystemLocal() throws Exception { indexRequest2.setPipeline("testpipeline"); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); + indexRequest1.decRef(); + indexRequest2.decRef(); AtomicBoolean responseCalled = new AtomicBoolean(false); AtomicBoolean failureCalled = new AtomicBoolean(false); @@ -458,13 +466,16 @@ public void testIngestForward() throws Exception { indexRequest.source(Collections.emptyMap()); indexRequest.setPipeline("testpipeline"); bulkRequest.add(indexRequest); + indexRequest.decRef(); BulkResponse bulkResponse = mock(BulkResponse.class); AtomicBoolean responseCalled = new AtomicBoolean(false); ActionListener listener = ActionTestUtils.assertNoFailureListener(response -> { responseCalled.set(true); assertSame(bulkResponse, response); }); - ActionTestUtils.execute(action, null, bulkRequest, listener); + // We're going to use the same bulk request twice, so we want to know when we're done with it: + CountDownLatch executionLatch = new CountDownLatch(2); + ActionTestUtils.execute(action, null, bulkRequest, ActionListener.runAfter(listener, executionLatch::countDown)); // should not have executed ingest locally verify(ingestService, never()).executeBulkRequest(anyInt(), any(), any(), any(), any(), any()); @@ -484,13 +495,15 @@ public void testIngestForward() throws Exception { // now make sure ingest nodes are rotated through with a subsequent request reset(transportService); - ActionTestUtils.execute(action, null, bulkRequest, listener); + ActionTestUtils.execute(action, null, bulkRequest, ActionListener.runAfter(listener, executionLatch::countDown)); verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture()); if (usedNode1) { assertSame(remoteNode2, node.getValue()); } else { assertSame(remoteNode1, node.getValue()); } + remoteResponseHandler.getValue().handleResponse(bulkResponse); // call the listener for the remote node + executionLatch.await(); } } @@ -544,14 +557,25 @@ public void testSingleItemBulkActionIngestForward() throws Exception { * not using the real TransportService that would do this for us). */ remoteResponseHandler.getValue().handleException(exception); + indexRequest.decRef(); } public void testUseDefaultPipeline() throws Exception { - validateDefaultPipeline(new IndexRequest(WITH_DEFAULT_PIPELINE).id("id")); + IndexRequest indexRequest = new IndexRequest(WITH_DEFAULT_PIPELINE).id("id"); + try { + validateDefaultPipeline(indexRequest); + } finally { + indexRequest.decRef(); + } } public void testUseDefaultPipelineWithAlias() throws Exception { - validateDefaultPipeline(new IndexRequest(WITH_DEFAULT_PIPELINE_ALIAS).id("id")); + IndexRequest indexRequest = new IndexRequest(WITH_DEFAULT_PIPELINE_ALIAS).id("id"); + try { + validateDefaultPipeline(indexRequest); + } finally { + indexRequest.decRef(); + } } public void testUseDefaultPipelineWithBulkUpsert() throws Exception { @@ -578,6 +602,9 @@ private void validatePipelineWithBulkUpsert(@Nullable String indexRequestIndexNa .script(mockScript("1")) .scriptedUpsert(true); bulkRequest.add(upsertRequest).add(docAsUpsertRequest).add(scriptedUpsert); + scriptedUpsert.decRef(); + docAsUpsertRequest.decRef(); + upsertRequest.decRef(); AtomicBoolean responseCalled = new AtomicBoolean(false); AtomicBoolean failureCalled = new AtomicBoolean(false); @@ -639,6 +666,9 @@ private void validatePipelineWithBulkUpsert(@Nullable String indexRequestIndexNa assertTrue(action.isExecuted); assertTrue(responseCalled.get()); verifyNoMoreInteractions(transportService); + indexRequest1.decRef(); + indexRequest2.decRef(); + indexRequest3.decRef(); } } @@ -703,6 +733,7 @@ public void testDoExecuteCalledTwiceCorrectly() throws Exception { assertTrue(action.indexCreated); // now the index is created since we skipped the ingest node path. assertTrue(responseCalled.get()); verifyNoMoreInteractions(transportService); + indexRequest.decRef(); } public void testNotFindDefaultPipelineFromTemplateMatches() { @@ -722,6 +753,7 @@ public void testNotFindDefaultPipelineFromTemplateMatches() { ); assertEquals(IngestService.NOOP_PIPELINE_NAME, indexRequest.getPipeline()); verifyNoMoreInteractions(ingestService); + indexRequest.decRef(); } @@ -792,6 +824,7 @@ public void testFindDefaultPipelineFromTemplateMatch() { */ completionHandler.getValue().accept(DUMMY_WRITE_THREAD, exception); verifyNoMoreInteractions(transportService); + indexRequest.decRef(); } public void testFindDefaultPipelineFromV2TemplateMatch() { @@ -831,6 +864,7 @@ public void testFindDefaultPipelineFromV2TemplateMatch() { eq(Names.WRITE) ); completionHandler.getValue().accept(DUMMY_WRITE_THREAD, exception); + indexRequest.decRef(); } public void testIngestCallbackExceptionHandled() throws Exception { @@ -865,6 +899,7 @@ public void testIngestCallbackExceptionHandled() throws Exception { assertFalse(action.isExecuted); assertFalse(responseCalled.get()); assertTrue(failureCalled.get()); + indexRequest1.decRef(); } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 1542be9af8e34..7e47aeda589d1 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; @@ -190,12 +191,17 @@ public void testGetIndexWriteRequest() throws Exception { assertEquals(TransportBulkAction.getIndexWriteRequest(upsertRequest), indexRequest); assertEquals(TransportBulkAction.getIndexWriteRequest(docAsUpsertRequest), indexRequest); assertEquals(TransportBulkAction.getIndexWriteRequest(scriptedUpsert), indexRequest); + indexRequest.decRef(); + upsertRequest.decRef(); + docAsUpsertRequest.decRef(); + scriptedUpsert.decRef(); DeleteRequest deleteRequest = new DeleteRequest("index", "id"); assertNull(TransportBulkAction.getIndexWriteRequest(deleteRequest)); UpdateRequest badUpsertRequest = new UpdateRequest("index", "id1"); assertNull(TransportBulkAction.getIndexWriteRequest(badUpsertRequest)); + badUpsertRequest.decRef(); } public void testProhibitAppendWritesInBackingIndices() throws Exception { @@ -217,6 +223,7 @@ public void testProhibitAppendWritesInBackingIndices() throws Exception { + "target corresponding data stream [logs-foobar] instead" ) ); + invalidRequest1.decRef(); // Testing index op against backing index fails: IndexRequest invalidRequest2 = new IndexRequest(backingIndexName).opType(DocWriteRequest.OpType.INDEX); @@ -231,33 +238,41 @@ public void testProhibitAppendWritesInBackingIndices() throws Exception { + "targeting backing indices is disallowed, target corresponding data stream [logs-foobar] instead" ) ); + invalidRequest2.decRef(); // Testing valid writes ops against a backing index: DocWriteRequest validRequest = new IndexRequest(backingIndexName).opType(DocWriteRequest.OpType.INDEX) .setIfSeqNo(1) .setIfPrimaryTerm(1); TransportBulkAction.prohibitAppendWritesInBackingIndices(validRequest, metadata); + ((IndexRequest) validRequest).decRef(); validRequest = new DeleteRequest(backingIndexName); TransportBulkAction.prohibitAppendWritesInBackingIndices(validRequest, metadata); validRequest = new UpdateRequest(backingIndexName, "_id"); TransportBulkAction.prohibitAppendWritesInBackingIndices(validRequest, metadata); + ((UpdateRequest) validRequest).decRef(); // Testing append only write via ds name validRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE); TransportBulkAction.prohibitAppendWritesInBackingIndices(validRequest, metadata); + ((IndexRequest) validRequest).decRef(); validRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.INDEX); TransportBulkAction.prohibitAppendWritesInBackingIndices(validRequest, metadata); + ((IndexRequest) validRequest).decRef(); // Append only for a backing index that doesn't exist is allowed: validRequest = new IndexRequest(DataStream.getDefaultBackingIndexName("logs-barbaz", 1)).opType(DocWriteRequest.OpType.CREATE); TransportBulkAction.prohibitAppendWritesInBackingIndices(validRequest, metadata); + ((IndexRequest) validRequest).decRef(); // Some other index names: validRequest = new IndexRequest("my-index").opType(DocWriteRequest.OpType.CREATE); TransportBulkAction.prohibitAppendWritesInBackingIndices(validRequest, metadata); + ((IndexRequest) validRequest).decRef(); validRequest = new IndexRequest("foobar").opType(DocWriteRequest.OpType.CREATE); TransportBulkAction.prohibitAppendWritesInBackingIndices(validRequest, metadata); + ((IndexRequest) validRequest).decRef(); } public void testProhibitCustomRoutingOnDataStream() throws Exception { @@ -266,7 +281,7 @@ public void testProhibitCustomRoutingOnDataStream() throws Exception { Metadata metadata = clusterState.metadata(); // custom routing requests against the data stream are prohibited - DocWriteRequest writeRequestAgainstDataStream = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.INDEX) + IndexRequest writeRequestAgainstDataStream = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.INDEX) .routing("custom"); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, @@ -279,12 +294,14 @@ public void testProhibitCustomRoutingOnDataStream() throws Exception { + "but the [allow_custom_routing] setting was not enabled in the data stream's template." ) ); + writeRequestAgainstDataStream.decRef(); // test custom routing is allowed when the index request targets the backing index - DocWriteRequest writeRequestAgainstIndex = new IndexRequest(DataStream.getDefaultBackingIndexName(dataStreamName, 1L)).opType( + IndexRequest writeRequestAgainstIndex = new IndexRequest(DataStream.getDefaultBackingIndexName(dataStreamName, 1L)).opType( DocWriteRequest.OpType.INDEX ).routing("custom"); prohibitCustomRoutingOnDataStream(writeRequestAgainstIndex, metadata); + writeRequestAgainstIndex.decRef(); } public void testOnlySystem() { @@ -302,20 +319,30 @@ public void testOnlySystem() { List.of(new SystemIndices.Feature("plugin", "test feature", List.of(SystemIndexDescriptorUtils.createUnmanaged(".test*", "")))) ); List onlySystem = List.of(".foo", ".bar"); - assertTrue(TransportBulkAction.isOnlySystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices)); + try (BulkRequest bulkRequest = buildBulkRequest(onlySystem)) { + assertTrue(TransportBulkAction.isOnlySystem(bulkRequest, indicesLookup, systemIndices)); + } onlySystem = List.of(".foo", ".bar", ".test"); - assertTrue(TransportBulkAction.isOnlySystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices)); + try (BulkRequest bulkRequest = buildBulkRequest(onlySystem)) { + assertTrue(TransportBulkAction.isOnlySystem(bulkRequest, indicesLookup, systemIndices)); + } List nonSystem = List.of("foo", "bar"); - assertFalse(TransportBulkAction.isOnlySystem(buildBulkRequest(nonSystem), indicesLookup, systemIndices)); + try (BulkRequest bulkRequest = buildBulkRequest(nonSystem)) { + assertFalse(TransportBulkAction.isOnlySystem(bulkRequest, indicesLookup, systemIndices)); + } List mixed = List.of(".foo", ".test", "other"); - assertFalse(TransportBulkAction.isOnlySystem(buildBulkRequest(mixed), indicesLookup, systemIndices)); + try (BulkRequest bulkRequest = buildBulkRequest(mixed)) { + assertFalse(TransportBulkAction.isOnlySystem(bulkRequest, indicesLookup, systemIndices)); + } } public void testRejectCoordination() throws Exception { - try (BulkRequest bulkRequest = new BulkRequest().add(new IndexRequest("index").id("id").source(Collections.emptyMap()))) { + IndexRequest indexRequest = new IndexRequest("index").id("id").source(Collections.emptyMap()); + try (BulkRequest bulkRequest = new BulkRequest().add(indexRequest)) { + indexRequest.decRef(); threadPool.startForcingRejections(); PlainActionFuture future = new PlainActionFuture<>(); ActionTestUtils.execute(bulkAction, null, bulkRequest, future); @@ -327,7 +354,9 @@ public void testRejectCoordination() throws Exception { public void testRejectionAfterCreateIndexIsPropagated() throws Exception { bulkAction.failIndexCreation = randomBoolean(); - try (BulkRequest bulkRequest = new BulkRequest().add(new IndexRequest("index").id("id").source(Collections.emptyMap()))) { + IndexRequest indexRequest = new IndexRequest("index").id("id").source(Collections.emptyMap()); + try (BulkRequest bulkRequest = new BulkRequest().add(indexRequest)) { + indexRequest.decRef(); bulkAction.beforeIndexCreation = threadPool::startForcingRejections; PlainActionFuture future = new PlainActionFuture<>(); ActionTestUtils.execute(bulkAction, null, bulkRequest, future); @@ -339,17 +368,19 @@ public void testRejectionAfterCreateIndexIsPropagated() throws Exception { } private BulkRequest buildBulkRequest(List indices) { - try (BulkRequest request = new BulkRequest()) { - for (String index : indices) { - final DocWriteRequest subRequest = switch (randomIntBetween(1, 3)) { - case 1 -> new IndexRequest(index); - case 2 -> new DeleteRequest(index).id("0"); - case 3 -> new UpdateRequest(index, "0"); - default -> throw new IllegalStateException("only have 3 cases"); - }; - request.add(subRequest); + BulkRequest request = new BulkRequest(); + for (String index : indices) { + final DocWriteRequest subRequest = switch (randomIntBetween(1, 3)) { + case 1 -> new IndexRequest(index); + case 2 -> new DeleteRequest(index).id("0"); + case 3 -> new UpdateRequest(index, "0"); + default -> throw new IllegalStateException("only have 3 cases"); + }; + request.add(subRequest); + if (subRequest instanceof RefCounted refCountedRequest) { + refCountedRequest.decRef(); } - return request; } + return request; } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 7089862f5d59c..a25abf7c6dd35 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -197,7 +197,7 @@ private void runTestTook(boolean controlled) throws Exception { bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, XContentType.JSON); AtomicLong expected = new AtomicLong(); TransportBulkAction action = createAction(controlled, expected); - action.doExecute(null, bulkRequest, new ActionListener() { + action.doExecute(null, bulkRequest, ActionListener.runAfter(new ActionListener<>() { @Override public void onResponse(BulkResponse bulkItemResponses) { if (controlled) { @@ -211,14 +211,11 @@ public void onResponse(BulkResponse bulkItemResponses) { greaterThanOrEqualTo(TimeUnit.MILLISECONDS.convert(expected.get(), TimeUnit.NANOSECONDS)) ); } - bulkRequest.close(); } @Override - public void onFailure(Exception e) { - bulkRequest.close(); - } - }); + public void onFailure(Exception e) {} + }, bulkRequest::decRef)); } static class Resolver extends IndexNameExpressionResolver { diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java index b5e1d84026a59..9d976fbc830cd 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java @@ -106,8 +106,9 @@ public void testExecuteBulkIndexRequest() throws Exception { BulkItemRequest[] items = new BulkItemRequest[1]; boolean create = randomBoolean(); - DocWriteRequest writeRequest = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE).create(create); + IndexRequest writeRequest = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE).create(create); BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest); + writeRequest.decRef(); items[0] = primaryRequest; try (BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items)) { @@ -137,9 +138,11 @@ public void testExecuteBulkIndexRequest() throws Exception { // Assert that the document actually made it there assertDocCount(shard, 1); } + primaryRequest.decRef(); writeRequest = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE).create(true); primaryRequest = new BulkItemRequest(0, writeRequest); + writeRequest.decRef(); items[0] = primaryRequest; try (BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items)) { @@ -181,6 +184,7 @@ public void testExecuteBulkIndexRequest() throws Exception { assertDocCount(shard, 1); closeShards(shard); } + primaryRequest.decRef(); } public void testSkipBulkIndexRequestIfAborted() throws Exception { @@ -188,10 +192,11 @@ public void testSkipBulkIndexRequestIfAborted() throws Exception { BulkItemRequest[] items = new BulkItemRequest[randomIntBetween(2, 5)]; for (int i = 0; i < items.length; i++) { - DocWriteRequest writeRequest = new IndexRequest("index").id("id_" + i) + IndexRequest writeRequest = new IndexRequest("index").id("id_" + i) .source(Requests.INDEX_CONTENT_TYPE) .opType(DocWriteRequest.OpType.INDEX); items[i] = new BulkItemRequest(i, writeRequest); + writeRequest.decRef(); } try (BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items)) { @@ -246,13 +251,17 @@ public void testSkipBulkIndexRequestIfAborted() throws Exception { latch.await(); } + for (BulkItemRequest item : items) { + item.decRef(); + } } public void testExecuteBulkIndexRequestWithMappingUpdates() throws Exception { BulkItemRequest[] items = new BulkItemRequest[1]; - DocWriteRequest writeRequest = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); + IndexRequest writeRequest = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); items[0] = new BulkItemRequest(0, writeRequest); + writeRequest.decRef(); try (BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items)) { Engine.IndexResult mappingUpdate = new Engine.IndexResult( @@ -328,14 +337,16 @@ public void testExecuteBulkIndexRequestWithMappingUpdates() throws Exception { closeShards(shard); } + items[0].decRef(); } public void testExecuteBulkIndexRequestWithErrorWhileUpdatingMapping() throws Exception { IndexShard shard = newStartedShard(true); BulkItemRequest[] items = new BulkItemRequest[1]; - DocWriteRequest writeRequest = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); + IndexRequest writeRequest = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); items[0] = new BulkItemRequest(0, writeRequest); + writeRequest.decRef(); try (BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items)) { // Return an exception when trying to update the mapping, or when waiting for it to come @@ -385,6 +396,7 @@ public void onFailure(final Exception e) { closeShards(shard); } + items[0].decRef(); } public void testExecuteBulkDeleteRequest() throws Exception { @@ -399,6 +411,7 @@ public void testExecuteBulkDeleteRequest() throws Exception { Translog.Location location = new Translog.Location(0, 0, 0); randomlySetIgnoredPrimaryResponse(items[0]); + items[0].decRef(); BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard); TransportShardBulkAction.executeBulkItemRequest( @@ -446,6 +459,7 @@ public void testExecuteBulkDeleteRequest() throws Exception { try (BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items)) { randomlySetIgnoredPrimaryResponse(items[0]); + items[0].decRef(); BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard); TransportShardBulkAction.executeBulkItemRequest( @@ -489,8 +503,9 @@ public void testExecuteBulkDeleteRequest() throws Exception { } public void testNoopUpdateRequest() throws Exception { - DocWriteRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); + UpdateRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest); + writeRequest.decRef(); DocWriteResponse noopUpdateResponse = new UpdateResponse(shardId, "id", 0, 2, 1, DocWriteResponse.Result.NOOP); @@ -535,14 +550,14 @@ public void testNoopUpdateRequest() throws Exception { assertEquals(primaryRequest, bulkShardRequest.items()[0]); // check that bulk item was not mutated assertThat(primaryResponse.getResponse().getSeqNo(), equalTo(0L)); } + primaryRequest.decRef(); } public void testUpdateRequestWithFailure() throws Exception { IndexSettings indexSettings = new IndexSettings(indexMetadata(), Settings.EMPTY); - DocWriteRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); + UpdateRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest); - - IndexRequest updateResponse = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + writeRequest.decRef(); Exception err = new ElasticsearchException("I'm dead <(x.x)>"); Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0, "id"); @@ -553,9 +568,9 @@ public void testUpdateRequestWithFailure() throws Exception { when(shard.indexSettings()).thenReturn(indexSettings); UpdateHelper updateHelper = mock(UpdateHelper.class); - when(updateHelper.prepare(any(), eq(shard), any())).thenReturn( - new UpdateHelper.Result( - updateResponse, + when(updateHelper.prepare(any(), eq(shard), any())).thenAnswer( + invocation -> new UpdateHelper.Result( + new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"), randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED, Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE @@ -593,16 +608,16 @@ public void testUpdateRequestWithFailure() throws Exception { assertThat(failure.getCause(), equalTo(err)); assertThat(failure.getStatus(), equalTo(RestStatus.INTERNAL_SERVER_ERROR)); } + primaryRequest.decRef(); } public void testUpdateRequestWithConflictFailure() throws Exception { IndexSettings indexSettings = new IndexSettings(indexMetadata(), Settings.EMPTY); int retries = randomInt(4); - DocWriteRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value") + UpdateRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value") .retryOnConflict(retries); BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest); - - IndexRequest updateResponse = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + writeRequest.decRef(); Exception err = new VersionConflictEngineException(shardId, "id", "I'm conflicted <(;_;)>"); Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0, "id"); @@ -613,9 +628,9 @@ public void testUpdateRequestWithConflictFailure() throws Exception { when(shard.indexSettings()).thenReturn(indexSettings); UpdateHelper updateHelper = mock(UpdateHelper.class); - when(updateHelper.prepare(any(), eq(shard), any())).thenReturn( - new UpdateHelper.Result( - updateResponse, + when(updateHelper.prepare(any(), eq(shard), any())).thenAnswer( + invocation -> new UpdateHelper.Result( + new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"), randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED, Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE @@ -654,14 +669,14 @@ public void testUpdateRequestWithConflictFailure() throws Exception { assertThat(failure.getCause(), equalTo(err)); assertThat(failure.getStatus(), equalTo(RestStatus.CONFLICT)); } + assert primaryRequest.decRef(); } public void testUpdateRequestWithSuccess() throws Exception { IndexSettings indexSettings = new IndexSettings(indexMetadata(), Settings.EMPTY); - DocWriteRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); + UpdateRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest); - - IndexRequest updateResponse = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + writeRequest.decRef(); boolean created = randomBoolean(); Translog.Location resultLocation = new Translog.Location(42, 42, 42); @@ -674,9 +689,9 @@ public void testUpdateRequestWithSuccess() throws Exception { when(shard.shardId()).thenReturn(shardId); UpdateHelper updateHelper = mock(UpdateHelper.class); - when(updateHelper.prepare(any(), eq(shard), any())).thenReturn( - new UpdateHelper.Result( - updateResponse, + when(updateHelper.prepare(any(), eq(shard), any())).thenAnswer( + invocation -> new UpdateHelper.Result( + new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"), created ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED, Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE @@ -701,7 +716,6 @@ public void testUpdateRequestWithSuccess() throws Exception { // Check that the translog is successfully advanced assertThat(context.getLocationToSync(), equalTo(resultLocation)); - assertThat(bulkShardRequest.items()[0].request(), equalTo(updateResponse)); // Since this was not a conflict failure, the primary response // should be filled out with the failure information BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse(); @@ -712,12 +726,14 @@ public void testUpdateRequestWithSuccess() throws Exception { assertThat(response.status(), equalTo(created ? RestStatus.CREATED : RestStatus.OK)); assertThat(response.getSeqNo(), equalTo(13L)); } + primaryRequest.decRef(); } public void testUpdateWithDelete() throws Exception { IndexSettings indexSettings = new IndexSettings(indexMetadata(), Settings.EMPTY); - DocWriteRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); + UpdateRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest); + writeRequest.decRef(); DeleteRequest updateResponse = new DeleteRequest("index", "id"); @@ -767,11 +783,13 @@ public void testUpdateWithDelete() throws Exception { assertThat(response.status(), equalTo(RestStatus.OK)); assertThat(response.getSeqNo(), equalTo(resultSeqNo)); } + primaryRequest.decRef(); } public void testFailureDuringUpdateProcessing() throws Exception { - DocWriteRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); + UpdateRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest); + writeRequest.decRef(); IndexShard shard = mock(IndexShard.class); @@ -807,6 +825,7 @@ public void testFailureDuringUpdateProcessing() throws Exception { assertThat(failure.getCause(), equalTo(err)); assertThat(failure.getStatus(), equalTo(RestStatus.INTERNAL_SERVER_ERROR)); } + primaryRequest.decRef(); } public void testTranslogPositionToSync() throws Exception { @@ -815,12 +834,16 @@ public void testTranslogPositionToSync() throws Exception { try (var operationPermit = getOperationPermit(shard)) { BulkItemRequest[] items = new BulkItemRequest[randomIntBetween(2, 5)]; for (int i = 0; i < items.length; i++) { - DocWriteRequest writeRequest = new IndexRequest("index").id("id_" + i) + IndexRequest writeRequest = new IndexRequest("index").id("id_" + i) .source(Requests.INDEX_CONTENT_TYPE) .opType(DocWriteRequest.OpType.INDEX); items[i] = new BulkItemRequest(i, writeRequest); + writeRequest.decRef(); } try (BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items)) { + for (BulkItemRequest item : items) { + item.decRef(); + } BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard); while (context.hasMoreOperationsToExecute()) { @@ -855,7 +878,9 @@ public void testTranslogPositionToSync() throws Exception { public void testNoOpReplicationOnPrimaryDocumentFailure() throws Exception { final IndexShard shard = spy(newStartedShard(false)); - BulkItemRequest itemRequest = new BulkItemRequest(0, new IndexRequest("index").source(Requests.INDEX_CONTENT_TYPE)); + IndexRequest indexRequest = new IndexRequest("index").source(Requests.INDEX_CONTENT_TYPE); + BulkItemRequest itemRequest = new BulkItemRequest(0, indexRequest); + indexRequest.decRef(); final String failureMessage = "simulated primary failure"; final IOException exception = new IOException(failureMessage); itemRequest.setPrimaryResponse( @@ -872,6 +897,7 @@ public void testNoOpReplicationOnPrimaryDocumentFailure() throws Exception { verify(shard, times(1)).markSeqNoAsNoop(1, 1, exception.toString()); closeShards(shard); } + itemRequest.decRef(); } public void testRetries() throws Exception { @@ -880,8 +906,7 @@ public void testRetries() throws Exception { // the beating will continue until success has come. writeRequest.retryOnConflict(Integer.MAX_VALUE); BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest); - - IndexRequest updateResponse = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); + writeRequest.decRef(); Exception err = new VersionConflictEngineException(shardId, "id", "I'm conflicted <(;_;)>"); Engine.IndexResult conflictedResult = new Engine.IndexResult(err, 0, "id"); @@ -914,9 +939,9 @@ public void testRetries() throws Exception { when(mapperService.merge(any(), any(CompressedXContent.class), any())).thenReturn(mergedDocMapper); UpdateHelper updateHelper = mock(UpdateHelper.class); - when(updateHelper.prepare(any(), eq(shard), any())).thenReturn( - new UpdateHelper.Result( - updateResponse, + when(updateHelper.prepare(any(), eq(shard), any())).thenAnswer( + invocation -> new UpdateHelper.Result( + new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"), randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED, Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE @@ -949,6 +974,7 @@ public void testRetries() throws Exception { ); latch.await(); } + primaryRequest.decRef(); } public void testForceExecutionOnRejectionAfterMappingUpdate() throws Exception { @@ -976,9 +1002,8 @@ public void testForceExecutionOnRejectionAfterMappingUpdate() throws Exception { rejectingThreadPool.executor(ThreadPool.Names.WRITE).execute(() -> {}); BulkItemRequest[] items = new BulkItemRequest[2]; - DocWriteRequest writeRequest1 = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "foo", 1); - DocWriteRequest writeRequest2 = new IndexRequest("index").id("id") - .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); + IndexRequest writeRequest1 = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "foo", 1); + IndexRequest writeRequest2 = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); items[0] = new BulkItemRequest(0, writeRequest1); items[1] = new BulkItemRequest(1, writeRequest2); try (BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items)) { @@ -1009,6 +1034,8 @@ public void testForceExecutionOnRejectionAfterMappingUpdate() throws Exception { when(mergedDoc.mappingSource()).thenReturn(CompressedXContent.fromJSON("{}")); randomlySetIgnoredPrimaryResponse(items[0]); + items[0].decRef(); + items[1].decRef(); AtomicInteger updateCalled = new AtomicInteger(); @@ -1073,6 +1100,8 @@ public void testForceExecutionOnRejectionAfterMappingUpdate() throws Exception { closeShards(shard); } + writeRequest1.decRef(); + writeRequest2.decRef(); } finally { rejectingThreadPool.shutdownNow(); } @@ -1082,12 +1111,13 @@ public void testPerformOnPrimaryReportsBulkStats() throws Exception { IndexShard shard = newStartedShard(true); BulkItemRequest[] items = new BulkItemRequest[5]; for (int i = 0; i < items.length; i++) { - DocWriteRequest writeRequest = new IndexRequest("index").id("id_" + i) + IndexRequest writeRequest = new IndexRequest("index").id("id_" + i) // we need to induce mapping updates because it's on the mapping update code path where introduce delays to make sure the // average operation time is gt 1ms (so we can assert it's *always* gt 0) .source(Requests.INDEX_CONTENT_TYPE, "foo" + i, "bar") .opType(DocWriteRequest.OpType.INDEX); items[i] = new BulkItemRequest(i, writeRequest); + writeRequest.decRef(); } try (BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items)) { @@ -1124,6 +1154,9 @@ public void testPerformOnPrimaryReportsBulkStats() throws Exception { latch.await(); } + for (BulkItemRequest item : items) { + item.decRef(); + } } public void testNoopMappingUpdateInfiniteLoopPrevention() throws Exception { @@ -1146,8 +1179,8 @@ public void testNoopMappingUpdateInfiniteLoopPrevention() throws Exception { when(mapperService.merge(any(), any(CompressedXContent.class), any())).thenReturn(documentMapper); UpdateHelper updateHelper = mock(UpdateHelper.class); - when(updateHelper.prepare(any(), eq(shard), any())).thenReturn( - new UpdateHelper.Result( + when(updateHelper.prepare(any(), eq(shard), any())).thenAnswer( + invocation -> new UpdateHelper.Result( new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"), randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED, Collections.singletonMap("field", "value"), @@ -1155,9 +1188,14 @@ public void testNoopMappingUpdateInfiniteLoopPrevention() throws Exception { ) ); + UpdateRequest updateRequest = new UpdateRequest("index", "id"); BulkItemRequest[] items = new BulkItemRequest[] { - new BulkItemRequest(0, new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value")) }; + new BulkItemRequest(0, updateRequest.doc(Requests.INDEX_CONTENT_TYPE, "field", "value")) }; + updateRequest.decRef(); BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items); + for (BulkItemRequest item : items) { + item.decRef(); + } try { AssertionError error = expectThrows( AssertionError.class, @@ -1220,8 +1258,8 @@ public void testNoopMappingUpdateSuccessOnRetry() throws Exception { when(mapperService.mappingVersion()).thenReturn(0L, 1L); UpdateHelper updateHelper = mock(UpdateHelper.class); - when(updateHelper.prepare(any(), eq(shard), any())).thenReturn( - new UpdateHelper.Result( + when(updateHelper.prepare(any(), eq(shard), any())).thenAnswer( + invocation -> new UpdateHelper.Result( new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"), randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED, Collections.singletonMap("field", "value"), @@ -1229,9 +1267,14 @@ public void testNoopMappingUpdateSuccessOnRetry() throws Exception { ) ); + UpdateRequest updateRequest = new UpdateRequest("index", "id"); BulkItemRequest[] items = new BulkItemRequest[] { - new BulkItemRequest(0, new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value")) }; + new BulkItemRequest(0, updateRequest.doc(Requests.INDEX_CONTENT_TYPE, "field", "value")) }; + updateRequest.decRef(); BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items); + for (BulkItemRequest item : items) { + item.decRef(); + } final CountDownLatch latch = new CountDownLatch(1); TransportShardBulkAction.performOnPrimary( diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index 7167c389059bd..fb9efe3888144 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.client.internal.Requests; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; @@ -135,11 +136,14 @@ public void testIndexData() { int bulkItemCount = randomIntBetween(0, 200); for (int i = 0; i < bulkItemCount; i++) { Map source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5)); - IndexRequest indexRequest = new IndexRequest(randomAlphaOfLength(10)).id(randomAlphaOfLength(10)).source(source); + IndexRequest indexRequest = new IndexRequest(randomAlphaOfLength(10)).id(randomAlphaOfLength(10)) + .setListExecutedPipelines(true) + .source(source); for (int j = 0; j < randomIntBetween(0, 10); j++) { indexRequest.addPipeline(randomAlphaOfLength(12)); } - bulkRequest.add(); + bulkRequest.add(indexRequest); + indexRequest.decRef(); } AtomicBoolean onResponseCalled = new AtomicBoolean(false); ActionListener listener = new ActionListener<>() { @@ -166,12 +170,16 @@ public void onResponse(BulkResponse response) { Strings.format( """ { + "_id": "%s", "_index": "%s", + "_version": %s, "_source": %s, "executed_pipelines": [%s] }""", + indexRequest.id(), indexRequest.index(), - indexRequest.source(), + indexRequest.version(), + XContentHelper.convertToJson(indexRequest.source(), false, Requests.INDEX_CONTENT_TYPE), indexRequest.getExecutedPipelines() .stream() .map(pipeline -> "\"" + pipeline + "\"") diff --git a/server/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java b/server/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java index e2f67d9387ff5..b11737c5fa781 100644 --- a/server/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java @@ -76,5 +76,7 @@ public void testSetSource() throws Exception { doc.close(); indexRequestBuilder.setSource(doc); assertEquals(EXPECTED_SOURCE, XContentHelper.convertToJson(indexRequestBuilder.request().source(), true)); + + indexRequestBuilder.request().decRef(); } } diff --git a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index e505f04e0ce06..499a11b40fb11 100644 --- a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -69,10 +69,18 @@ public void testIndexRequestOpTypeFromString() throws Exception { assertThat(indexRequest.opType(), equalTo(DocWriteRequest.OpType.INDEX)); indexRequest.opType(indexUpper); assertThat(indexRequest.opType(), equalTo(DocWriteRequest.OpType.INDEX)); + indexRequest.decRef(); } public void testReadIncorrectOpType() { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new IndexRequest("").opType("foobar")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + IndexRequest indexRequest = new IndexRequest(""); + try { + indexRequest.opType("foobar"); + } finally { + indexRequest.decRef(); + } + }); assertThat(e.getMessage(), equalTo("opType must be 'create' or 'index', found: [foobar]")); } @@ -88,6 +96,7 @@ public void testCreateOperationRejectsVersions() { request.versionType(VersionType.INTERNAL); request.version(randomIntBetween(0, Integer.MAX_VALUE)); assertThat(request.validate().validationErrors(), not(empty())); + request.decRef(); } public void testIndexingRejectsLongIds() { @@ -96,12 +105,14 @@ public void testIndexingRejectsLongIds() { request.source("{}", XContentType.JSON); ActionRequestValidationException validate = request.validate(); assertNull(validate); + request.decRef(); id = randomAlphaOfLength(512); request = new IndexRequest("index").id(id); request.source("{}", XContentType.JSON); validate = request.validate(); assertNull(validate); + request.decRef(); id = randomAlphaOfLength(513); request = new IndexRequest("index").id(id); @@ -109,6 +120,7 @@ public void testIndexingRejectsLongIds() { validate = request.validate(); assertThat(validate, notNullValue()); assertThat(validate.getMessage(), containsString("id [" + id + "] is too long, must be no longer than 512 bytes but was: 513")); + request.decRef(); } public void testWaitForActiveShards() { @@ -118,12 +130,14 @@ public void testWaitForActiveShards() { assertEquals(request.waitForActiveShards(), ActiveShardCount.from(count)); // test negative shard count value not allowed expectThrows(IllegalArgumentException.class, () -> request.waitForActiveShards(ActiveShardCount.from(randomIntBetween(-10, -1)))); + request.decRef(); } public void testAutoGenerateId() { IndexRequest request = new IndexRequest("index"); request.autoGenerateId(); assertTrue("expected > 0 but got: " + request.getAutoGeneratedTimestamp(), request.getAutoGeneratedTimestamp() > 0); + request.decRef(); } public void testIndexResponse() { @@ -182,6 +196,8 @@ public void testIndexRequestXContentSerialization() throws IOException { assertEquals(new BytesArray("{}"), serialized.source()); assertEquals(isRequireAlias, serialized.isRequireAlias()); assertThat(serialized.getDynamicTemplates(), equalTo(dynamicTemplates)); + indexRequest.decRef(); + serialized.decRef(); } // reindex makes use of index requests without a source so this needs to be handled @@ -196,6 +212,8 @@ public void testSerializationOfEmptyRequestWorks() throws IOException { IndexRequest serialized = new IndexRequest(in); assertNull(serialized.getContentType()); assertEquals("index", serialized.index()); + request.decRef(); + serialized.decRef(); } } } @@ -216,6 +234,7 @@ public void testSerializeDynamicTemplates() throws Exception { in.setTransportVersion(ver); IndexRequest serialized = new IndexRequest(in); assertThat(serialized.getDynamicTemplates(), anEmptyMap()); + serialized.decRef(); } // old version { @@ -254,7 +273,9 @@ public void testSerializeDynamicTemplates() throws Exception { in.setTransportVersion(ver); IndexRequest serialized = new IndexRequest(in); assertThat(serialized.getDynamicTemplates(), equalTo(dynamicTemplates)); + serialized.decRef(); } + indexRequest.decRef(); } public void testToStringSizeLimit() { @@ -277,6 +298,7 @@ public void testToStringSizeLimit() { + "]}", request.toString() ); + request.decRef(); } public void testRejectsEmptyStringPipeline() { @@ -286,6 +308,7 @@ public void testRejectsEmptyStringPipeline() { ActionRequestValidationException validate = request.validate(); assertThat(validate, notNullValue()); assertThat(validate.getMessage(), containsString("pipeline cannot be an empty string")); + request.decRef(); } public void testGetConcreteWriteIndex() { @@ -313,6 +336,7 @@ public void testGetConcreteWriteIndex() { var result = request.getConcreteWriteIndex(metadata.getIndicesLookup().get(tsdbDataStream), metadata); assertThat(result, equalTo(metadata.dataStreams().get(tsdbDataStream).getIndices().get(1))); + request.decRef(); } { // Target is a regular index => resolve to this index only @@ -322,6 +346,7 @@ public void testGetConcreteWriteIndex() { var result = request.getConcreteWriteIndex(metadata.getIndicesLookup().get(indexName), metadata); assertThat(result.getName(), equalTo(indexName)); + request.decRef(); } { String regularDataStream = "logs_another-app_prod"; @@ -345,6 +370,7 @@ public void testGetConcreteWriteIndex() { var result = request.getConcreteWriteIndex(metadata2.getIndicesLookup().get(regularDataStream), metadata2); assertThat(result.getName(), equalTo(backingIndex2.getIndex().getName())); + request.decRef(); } { // provided timestamp resolves to the first backing index @@ -354,6 +380,7 @@ public void testGetConcreteWriteIndex() { var result = request.getConcreteWriteIndex(metadata.getIndicesLookup().get(tsdbDataStream), metadata); assertThat(result, equalTo(metadata.dataStreams().get(tsdbDataStream).getIndices().get(0))); + request.decRef(); } { // provided timestamp as millis since epoch resolves to the first backing index @@ -363,6 +390,7 @@ public void testGetConcreteWriteIndex() { var result = request.getConcreteWriteIndex(metadata.getIndicesLookup().get(tsdbDataStream), metadata); assertThat(result, equalTo(metadata.dataStreams().get(tsdbDataStream).getIndices().get(0))); + request.decRef(); } { IndexRequest request = new IndexRequest(tsdbDataStream); @@ -374,6 +402,7 @@ public void testGetConcreteWriteIndex() { var result = request.getConcreteWriteIndex(metadata.getIndicesLookup().get(tsdbDataStream), metadata); assertThat(result, equalTo(metadata.dataStreams().get(tsdbDataStream).getIndices().get(0))); + request.decRef(); } { // provided timestamp resolves to the latest backing index @@ -383,6 +412,7 @@ public void testGetConcreteWriteIndex() { var result = request.getConcreteWriteIndex(metadata.getIndicesLookup().get(tsdbDataStream), metadata); assertThat(result, equalTo(metadata.dataStreams().get(tsdbDataStream).getIndices().get(1))); + request.decRef(); } { // provided timestamp resolves to no index => fail with an exception @@ -405,6 +435,7 @@ public void testGetConcreteWriteIndex() { .replace("$end2", formatInstant(end2)) ) ); + request.decRef(); } { @@ -423,6 +454,7 @@ public void testGetConcreteWriteIndex() { + "Failed to parse object: expecting token of type [START_OBJECT] but found [null]" ) ); + request.decRef(); } { @@ -439,6 +471,7 @@ public void testGetConcreteWriteIndex() { e.getMessage(), equalTo("Error get data stream timestamp field: timestamp [10.0] type [class java.lang.Double] error") ); + request.decRef(); } } @@ -462,6 +495,8 @@ public void testSerialization() throws IOException { assertThat(copy.getFinalPipeline(), equalTo(indexRequest.getFinalPipeline())); assertThat(copy.ifPrimaryTerm(), equalTo(indexRequest.ifPrimaryTerm())); assertThat(copy.isRequireDataStream(), equalTo(indexRequest.isRequireDataStream())); + indexRequest.decRef(); + copy.decRef(); } private IndexRequest createTestInstance() { diff --git a/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java index a678956b20e59..7ca017d331394 100644 --- a/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java @@ -8,7 +8,9 @@ package org.elasticsearch.action.search; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; @@ -54,12 +56,16 @@ public void testKnnSearchRemovedVector() throws IOException { createIndex("index", indexSettings, builder); for (int doc = 0; doc < 10; doc++) { - prepareIndex("index").setId(Integer.toString(doc)).setSource("vector", randomVector(), "text", "hello world").get(); - prepareIndex("index").setSource("text", "goodnight world").get(); + indexData("index", Integer.toString(doc), "vector", randomVector(), "text", "hello world"); + indexData("index", null, "text", "goodnight world"); } indicesAdmin().prepareRefresh("index").get(); - client().prepareUpdate("index", "0").setDoc("vector", (Object) null).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate("index", "0") + .setDoc("vector", (Object) null) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + updateRequestBuilder.get(); + updateRequestBuilder.request().decRef(); float[] queryVector = randomVector(); KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 20, 50, null).boost(5.0f); @@ -99,8 +105,8 @@ public void testKnnWithQuery() throws IOException { createIndex("index", indexSettings, builder); for (int doc = 0; doc < 10; doc++) { - prepareIndex("index").setSource("vector", randomVector(), "text", "hello world").get(); - prepareIndex("index").setSource("text", "goodnight world").get(); + indexData("index", null, "vector", randomVector(), "text", "hello world"); + indexData("index", null, "text", "goodnight world"); } indicesAdmin().prepareRefresh("index").get(); @@ -147,7 +153,7 @@ public void testKnnFilter() throws IOException { for (int doc = 0; doc < 10; doc++) { String value = doc % 2 == 0 ? "first" : "second"; - prepareIndex("index").setId(String.valueOf(doc)).setSource("vector", randomVector(), "field", value).get(); + indexData("index", String.valueOf(doc), "vector", randomVector(), "field", value); } indicesAdmin().prepareRefresh("index").get(); @@ -189,9 +195,9 @@ public void testKnnFilterWithRewrite() throws IOException { createIndex("index", indexSettings, builder); for (int doc = 0; doc < 10; doc++) { - prepareIndex("index").setId(String.valueOf(doc)).setSource("vector", randomVector(), "field", "value").get(); + indexData("index", String.valueOf(doc), "vector", randomVector(), "field", "value"); } - prepareIndex("index").setId("lookup-doc").setSource("other-field", "value").get(); + indexData("index", "lookup-doc", "other-field", "value"); indicesAdmin().prepareRefresh("index").get(); @@ -236,9 +242,9 @@ public void testMultiKnnClauses() throws IOException { createIndex("index", indexSettings, builder); for (int doc = 0; doc < 10; doc++) { - prepareIndex("index").setSource("vector", randomVector(1.0f, 2.0f), "text", "hello world", "number", 1).get(); - prepareIndex("index").setSource("vector_2", randomVector(20f, 21f), "text", "hello world", "number", 2).get(); - prepareIndex("index").setSource("text", "goodnight world", "number", 3).get(); + indexData("index", null, "vector", randomVector(1.0f, 2.0f), "text", "hello world", "number", 1); + indexData("index", null, "vector_2", randomVector(20f, 21f), "text", "hello world", "number", 2); + indexData("index", null, "text", "goodnight world", "number", 3); } indicesAdmin().prepareRefresh("index").get(); @@ -299,7 +305,7 @@ public void testMultiKnnClausesSameDoc() throws IOException { for (int doc = 0; doc < 10; doc++) { // Make them have hte same vector. This will allow us to test the recall is the same but scores take into account both fields float[] vector = randomVector(); - prepareIndex("index").setSource("vector", vector, "vector_2", vector, "number", doc).get(); + indexData("index", null, "vector", vector, "vector_2", vector, "number", doc); } indicesAdmin().prepareRefresh("index").get(); @@ -369,10 +375,10 @@ public void testKnnFilteredAlias() throws IOException { int expectedHits = 0; for (int doc = 0; doc < 10; doc++) { if (randomBoolean()) { - prepareIndex("index").setId(String.valueOf(doc)).setSource("vector", randomVector(), "field", "hit").get(); + indexData("index", String.valueOf(doc), "vector", randomVector(), "field", "hit"); ++expectedHits; } else { - prepareIndex("index").setId(String.valueOf(doc)).setSource("vector", randomVector(), "field", "not hit").get(); + indexData("index", String.valueOf(doc), "vector", randomVector(), "field", "not hit"); } } indicesAdmin().prepareRefresh("index").get(); @@ -403,8 +409,8 @@ public void testKnnSearchAction() throws IOException { createIndex("index2", indexSettings, builder); for (int doc = 0; doc < 10; doc++) { - prepareIndex("index1").setId(String.valueOf(doc)).setSource("vector", randomVector()).get(); - prepareIndex("index2").setId(String.valueOf(doc)).setSource("vector", randomVector()).get(); + indexData("index1", String.valueOf(doc), "vector", randomVector()); + indexData("index2", String.valueOf(doc), "vector", randomVector()); } indicesAdmin().prepareForceMerge("index1", "index2").setMaxNumSegments(1).get(); @@ -441,7 +447,7 @@ public void testKnnVectorsWith4096Dims() throws IOException { createIndex("index", indexSettings, builder); for (int doc = 0; doc < 10; doc++) { - prepareIndex("index").setSource("vector", randomVector(4096)).get(); + indexData("index", null, "vector", randomVector(4096)); } indicesAdmin().prepareRefresh("index").get(); @@ -478,4 +484,18 @@ private float[] randomVector(float dimLower, float dimUpper) { } return vector; } + + private void indexData(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + if (id != null) { + indexRequestBuilder = indexRequestBuilder.setId(id); + } + indexRequestBuilder = indexRequestBuilder.setSource(source); + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + + } } diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java index 1de057f0446ee..e3705fed0ca49 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.action.search; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.core.RefCounted; import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ToXContent; @@ -114,7 +113,7 @@ public final void testFromXContent() throws IOException { .numberOfTestRuns(20) .supportsUnknownFields(supportsUnknownFields()) .assertEqualsConsumer(this::assertEqualInstances) - .dispose(RefCounted::decRef) + .dispose(MultiSearchResponse::decRef) .test(); } @@ -133,8 +132,8 @@ public void testFromXContentWithFailures() throws IOException { .supportsUnknownFields(true) // exceptions are not of the same type whenever parsed back .assertToXContentEquivalence(false) + .dispose(MultiSearchResponse::decRef) .assertEqualsConsumer(this::assertEqualInstances) - .dispose(RefCounted::decRef) .test(); } diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java index f1bf8d1dcb0dd..3e83966638d4a 100644 --- a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java +++ b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; import org.elasticsearch.index.analysis.PreConfiguredTokenizer; @@ -165,9 +166,13 @@ public void testRandomPayloadWithDelimitedPayloadTokenFilter() throws IOExceptio .build(); createIndex("test", setting, mapping); - prepareIndex("test").setId(Integer.toString(1)) - .setSource(jsonBuilder().startObject().field("field", queryString).endObject()) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(1)) + .setSource(jsonBuilder().startObject().field("field", queryString).endObject()); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } client().admin().indices().prepareRefresh().get(); TermVectorsRequestBuilder resp = client().prepareTermVectors("test", Integer.toString(1)) .setPayloads(true) diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index 7ee4d2d6bba9b..d0b7092ebad5d 100644 --- a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -147,6 +147,7 @@ public void testFromXContent() throws Exception { assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); params = script.getParams(); assertThat(params, equalTo(emptyMap())); + request.decRef(); // script with params request = new UpdateRequest("test", "1"); @@ -174,6 +175,7 @@ public void testFromXContent() throws Exception { assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); assertThat(params.get("param1").toString(), equalTo("value1")); + request.decRef(); request = new UpdateRequest("test", "1"); try ( @@ -200,6 +202,7 @@ public void testFromXContent() throws Exception { assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); assertThat(params.get("param1").toString(), equalTo("value1")); + request.decRef(); // script with params and upsert request = new UpdateRequest("test", "1"); @@ -240,6 +243,7 @@ public void testFromXContent() throws Exception { ).v2(); assertThat(upsertDoc.get("field1").toString(), equalTo("value1")); assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2")); + request.decRef(); request = new UpdateRequest("test", "1"); try ( @@ -275,6 +279,7 @@ public void testFromXContent() throws Exception { upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true, request.upsertRequest().getContentType()).v2(); assertThat(upsertDoc.get("field1").toString(), equalTo("value1")); assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2")); + request.decRef(); // script with doc request = new UpdateRequest("test", "1"); @@ -296,6 +301,7 @@ public void testFromXContent() throws Exception { Map doc = request.doc().sourceAsMap(); assertThat(doc.get("field1").toString(), equalTo("value1")); assertThat(((Map) doc.get("compound")).get("field2").toString(), equalTo("value2")); + request.decRef(); } public void testUnknownFieldParsing() throws Exception { @@ -308,6 +314,7 @@ public void testUnknownFieldParsing() throws Exception { XContentParseException ex = expectThrows(XContentParseException.class, () -> request.fromXContent(contentParser)); assertEquals("[1:2] [UpdateRequest] unknown field [unknown_field]", ex.getMessage()); } + request.decRef(); UpdateRequest request2 = new UpdateRequest("test", "1"); try ( @@ -324,6 +331,7 @@ public void testUnknownFieldParsing() throws Exception { XContentParseException ex = expectThrows(XContentParseException.class, () -> request2.fromXContent(unknownObject)); assertEquals("[1:76] [UpdateRequest] unknown field [params]", ex.getMessage()); } + request2.decRef(); } public void testFetchSourceParsing() throws Exception { @@ -364,6 +372,7 @@ public void testFetchSourceParsing() throws Exception { assertThat(request.fetchSource().excludes().length, equalTo(1)); assertThat(request.fetchSource().includes()[0], equalTo("path.inner.*")); assertThat(request.fetchSource().excludes()[0], equalTo("another.inner.*")); + request.decRef(); } public void testNowInScript() throws IOException { @@ -382,13 +391,17 @@ public void testNowInScript() throws IOException { assertThat(action, instanceOf(IndexRequest.class)); IndexRequest indexAction = (IndexRequest) action; assertEquals(nowInMillis, indexAction.sourceAsMap().get("update_timestamp")); + indexAction.decRef(); + updateRequest.decRef(); } + indexRequest.decRef(); } public void testIndexTimeout() { final GetResult getResult = new GetResult("test", "1", 0, 1, 0, true, new BytesArray("{\"f\":\"v\"}"), null, null); final UpdateRequest updateRequest = new UpdateRequest("test", "1").script(mockInlineScript("return")).timeout(randomTimeValue()); runTimeoutTest(getResult, updateRequest); + updateRequest.decRef(); } public void testDeleteTimeout() { @@ -396,6 +409,7 @@ public void testDeleteTimeout() { final UpdateRequest updateRequest = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = 'delete'")) .timeout(randomTimeValue()); runTimeoutTest(getResult, updateRequest); + updateRequest.decRef(); } public void testUpsertTimeout() throws IOException { @@ -413,6 +427,8 @@ public void testUpsertTimeout() throws IOException { .script(mockInlineScript("return")) .timeout(randomTimeValue()); runTimeoutTest(getResult, updateRequest); + upsert.decRef(); + updateRequest.decRef(); } private void runTimeoutTest(final GetResult getResult, final UpdateRequest updateRequest) { @@ -426,6 +442,9 @@ private void runTimeoutTest(final GetResult getResult, final UpdateRequest updat assertThat(action, instanceOf(ReplicationRequest.class)); final ReplicationRequest request = (ReplicationRequest) action; assertThat(request.timeout(), equalTo(updateRequest.timeout())); + if (action != updateRequest.upsertRequest()) { + request.decRef(); + } } public void testToAndFromXContent() throws IOException { @@ -435,7 +454,9 @@ public void testToAndFromXContent() throws IOException { if (randomBoolean()) { XContentType xContentType = randomFrom(XContentType.values()); BytesReference source = RandomObjects.randomSource(random(), xContentType); - updateRequest.doc(new IndexRequest().source(source, xContentType)); + IndexRequest indexRequest = new IndexRequest().source(source, xContentType); + updateRequest.doc(indexRequest); + indexRequest.decRef(); updateRequest.docAsUpsert(randomBoolean()); } else { ScriptType scriptType = randomFrom(ScriptType.values()); @@ -452,7 +473,9 @@ public void testToAndFromXContent() throws IOException { if (randomBoolean()) { XContentType xContentType = randomFrom(XContentType.values()); BytesReference source = RandomObjects.randomSource(random(), xContentType); - updateRequest.upsert(new IndexRequest().source(source, xContentType)); + IndexRequest indexRequest = new IndexRequest().source(source, xContentType); + updateRequest.upsert(indexRequest); + indexRequest.decRef(); } if (randomBoolean()) { if (randomBoolean()) { @@ -475,6 +498,7 @@ public void testToAndFromXContent() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); boolean humanReadable = randomBoolean(); BytesReference originalBytes = toShuffledXContent(updateRequest, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); + updateRequest.decRef(); if (randomBoolean()) { try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) { @@ -496,6 +520,7 @@ public void testToAndFromXContent() throws IOException { BytesReference finalBytes = toXContent(parsedUpdateRequest, xContentType, humanReadable); assertToXContentEquivalent(originalBytes, finalBytes, xContentType); + parsedUpdateRequest.decRef(); } public void testToValidateUpsertRequestAndCAS() { @@ -503,18 +528,24 @@ public void testToValidateUpsertRequestAndCAS() { updateRequest.setIfSeqNo(1L); updateRequest.setIfPrimaryTerm(1L); updateRequest.doc("{}", XContentType.JSON); - updateRequest.upsert(new IndexRequest("index").id("id")); + IndexRequest indexRequest = new IndexRequest("index").id("id"); + updateRequest.upsert(indexRequest); assertThat( updateRequest.validate().validationErrors(), contains("upsert requests don't support `if_seq_no` and `if_primary_term`") ); + indexRequest.decRef(); + updateRequest.decRef(); } public void testToValidateUpsertRequestWithVersion() { UpdateRequest updateRequest = new UpdateRequest("index", "id"); updateRequest.doc("{}", XContentType.JSON); - updateRequest.upsert(new IndexRequest("index").id("1").version(1L)); + IndexRequest indexRequest = new IndexRequest("index").id("1").version(1L); + updateRequest.upsert(indexRequest); assertThat(updateRequest.validate().validationErrors(), contains("can't provide version in upsert request")); + indexRequest.decRef(); + updateRequest.decRef(); } public void testUpdatingRejectsLongIds() { @@ -522,18 +553,21 @@ public void testUpdatingRejectsLongIds() { UpdateRequest request = new UpdateRequest("index", id); request.doc("{}", XContentType.JSON); ActionRequestValidationException validate = request.validate(); + request.decRef(); assertNull(validate); id = randomAlphaOfLength(512); request = new UpdateRequest("index", id); request.doc("{}", XContentType.JSON); validate = request.validate(); + request.decRef(); assertNull(validate); id = randomAlphaOfLength(513); request = new UpdateRequest("index", id); request.doc("{}", XContentType.JSON); validate = request.validate(); + request.decRef(); assertThat(validate, notNullValue()); assertThat(validate.getMessage(), containsString("id [" + id + "] is too long, must be no longer than 512 bytes but was: 513")); } @@ -545,6 +579,7 @@ public void testValidate() { ActionRequestValidationException validate = request.validate(); assertThat(validate, nullValue()); + request.decRef(); } { UpdateRequest request = new UpdateRequest("index", null); @@ -553,6 +588,7 @@ public void testValidate() { assertThat(validate, not(nullValue())); assertThat(validate.validationErrors(), hasItems("id is missing")); + request.decRef(); } } @@ -580,6 +616,7 @@ public void testRoutingExtraction() throws Exception { // Use the get result parent and routing assertThat(UpdateHelper.calculateRouting(getResult, indexRequest), equalTo("routing1")); + indexRequest.decRef(); } public void testNoopDetection() throws Exception { @@ -601,6 +638,8 @@ public void testNoopDetection() throws Exception { assertThat(result.action(), instanceOf(IndexRequest.class)); assertThat(result.getResponseResult(), equalTo(DocWriteResponse.Result.UPDATED)); assertThat(result.updatedSourceAsMap().get("body").toString(), equalTo("foo")); + request.decRef(); + ((IndexRequest) result.action()).decRef(); try (var parser = createParser(JsonXContent.jsonXContent, new BytesArray("{\"doc\": {\"body\": \"bar\"}}"))) { // Change the request to be a different doc @@ -610,6 +649,8 @@ public void testNoopDetection() throws Exception { assertThat(result.action(), instanceOf(IndexRequest.class)); assertThat(result.getResponseResult(), equalTo(DocWriteResponse.Result.UPDATED)); assertThat(result.updatedSourceAsMap().get("body").toString(), equalTo("bar")); + request.decRef(); + ((IndexRequest) result.action()).decRef(); } } @@ -630,11 +671,14 @@ public void testUpdateScript() throws Exception { assertThat(result.action(), instanceOf(IndexRequest.class)); assertThat(result.getResponseResult(), equalTo(DocWriteResponse.Result.UPDATED)); assertThat(result.updatedSourceAsMap().get("body").toString(), equalTo("foo")); + request.decRef(); + ((IndexRequest) result.action()).decRef(); // Now where the script changes the op to "delete" request = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = 'delete'")); result = updateHelper.prepareUpdateScriptRequest(shardId, request, getResult, ESTestCase::randomNonNegativeLong); + request.decRef(); assertThat(result.action(), instanceOf(DeleteRequest.class)); assertThat(result.getResponseResult(), equalTo(DocWriteResponse.Result.DELETED)); @@ -648,6 +692,7 @@ public void testUpdateScript() throws Exception { } result = updateHelper.prepareUpdateScriptRequest(shardId, request, getResult, ESTestCase::randomNonNegativeLong); + request.decRef(); assertThat(result.action(), instanceOf(UpdateResponse.class)); assertThat(result.getResponseResult(), equalTo(DocWriteResponse.Result.NOOP)); @@ -658,11 +703,13 @@ public void testToString() throws IOException { assertThat(request.toString(), equalTo(""" update {[test][1], doc_as_upsert[false], script[Script{type=inline, lang='mock', idOrCode='ctx._source.body = "foo"', \ options={}, params={}}], scripted_upsert[false], detect_noop[true]}""")); + request.decRef(); try (var parser = createParser(JsonXContent.jsonXContent, new BytesArray("{\"doc\": {\"body\": \"bar\"}}"))) { request = new UpdateRequest("test", "1").fromXContent(parser); assertThat(request.toString(), equalTo(""" update {[test][1], doc_as_upsert[false], doc[index {[null][null], source[{"body":"bar"}]}], \ scripted_upsert[false], detect_noop[true]}""")); + request.decRef(); } } } diff --git a/server/src/test/java/org/elasticsearch/client/internal/AbstractClientHeadersTestCase.java b/server/src/test/java/org/elasticsearch/client/internal/AbstractClientHeadersTestCase.java index 97c52ef2edc37..fbc8429bb09c2 100644 --- a/server/src/test/java/org/elasticsearch/client/internal/AbstractClientHeadersTestCase.java +++ b/server/src/test/java/org/elasticsearch/client/internal/AbstractClientHeadersTestCase.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.delete.TransportDeleteAction; import org.elasticsearch.action.get.TransportGetAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.TransportIndexAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.common.settings.Settings; @@ -104,10 +105,12 @@ public void testActions() { .cluster() .prepareDeleteStoredScript("id") .execute(new AssertingActionListener<>(TransportDeleteStoredScriptAction.TYPE.name(), client.threadPool())); - client.prepareIndex("idx") - .setId("id") - .setSource("source", XContentType.JSON) - .execute(new AssertingActionListener<>(TransportIndexAction.NAME, client.threadPool())); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex("idx").setId("id").setSource("source", XContentType.JSON); + try { + indexRequestBuilder.execute(new AssertingActionListener<>(TransportIndexAction.NAME, client.threadPool())); + } finally { + indexRequestBuilder.request().decRef(); + } // choosing arbitrary cluster admin actions to test client.admin().cluster().prepareClusterStats().execute(new AssertingActionListener<>(ClusterStatsAction.NAME, client.threadPool())); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 99dc1c84ba15b..60a82a1c0d442 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -1896,11 +1897,9 @@ public void testConcreteWriteIndexWithNoWriteIndexWithSingleIndex() { ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); - DocWriteRequest request = randomFrom( - new IndexRequest("test-alias"), - new UpdateRequest("test-alias", "_id"), - new DeleteRequest("test-alias") - ); + IndexRequest indexRequest = new IndexRequest("test-alias"); + UpdateRequest updateRequest = new UpdateRequest("test-alias", "_id"); + DocWriteRequest request = randomFrom(indexRequest, updateRequest, new DeleteRequest("test-alias")); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> indexNameExpressionResolver.concreteWriteIndex(state, request.indicesOptions(), request.indices()[0], false, false) @@ -1913,6 +1912,8 @@ public void testConcreteWriteIndexWithNoWriteIndexWithSingleIndex() { + " indices without one being designated as a write index" ) ); + indexRequest.decRef(); + updateRequest.decRef(); } public void testConcreteWriteIndexWithNoWriteIndexWithMultipleIndices() { @@ -1932,11 +1933,9 @@ public void testConcreteWriteIndexWithNoWriteIndexWithMultipleIndices() { ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); - DocWriteRequest request = randomFrom( - new IndexRequest("test-alias"), - new UpdateRequest("test-alias", "_id"), - new DeleteRequest("test-alias") - ); + IndexRequest indexRequest = new IndexRequest("test-alias"); + UpdateRequest updateRequest = new UpdateRequest("test-alias", "_id"); + DocWriteRequest request = randomFrom(indexRequest, updateRequest, new DeleteRequest("test-alias")); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> indexNameExpressionResolver.concreteWriteIndex(state, request.indicesOptions(), request.indices()[0], false, false) @@ -1949,6 +1948,8 @@ public void testConcreteWriteIndexWithNoWriteIndexWithMultipleIndices() { + " indices without one being designated as a write index" ) ); + indexRequest.decRef(); + updateRequest.decRef(); } public void testAliasResolutionNotAllowingMultipleIndices() { @@ -3187,17 +3188,27 @@ public void testResolveWriteIndexAbstraction() { assertThat(infe.toString(), containsString("logs-foobar")); assertThat(infe.getMetadataKeys().contains(IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY), is(true)); } + if (request instanceof RefCounted refCountedRequest) { + refCountedRequest.decRef(); + } } for (DocWriteRequest request : docWriteRequestsForName.apply("my-index")) { IndexAbstraction result = indexNameExpressionResolver.resolveWriteIndexAbstraction(finalState, request); assertThat(result.getName(), equalTo("my-index")); assertThat(result.getType(), equalTo(IndexAbstraction.Type.CONCRETE_INDEX)); + if (request instanceof RefCounted refCountedRequest) { + refCountedRequest.decRef(); + } } for (DocWriteRequest request : docWriteRequestsForName.apply("my-alias")) { IndexAbstraction result = indexNameExpressionResolver.resolveWriteIndexAbstraction(finalState, request); assertThat(result.getName(), equalTo("my-alias")); assertThat(result.getType(), equalTo(IndexAbstraction.Type.ALIAS)); + if (request instanceof RefCounted refCountedRequest) { + refCountedRequest.decRef(); + } } + } public void testResolveWriteIndexAbstractionNoWriteIndexForAlias() { @@ -3214,7 +3225,7 @@ public void testResolveWriteIndexAbstractionNoWriteIndexForAlias() { ) .build(); - DocWriteRequest request = new IndexRequest("my-alias"); + IndexRequest request = new IndexRequest("my-alias"); var e = expectThrows( IllegalArgumentException.class, () -> indexNameExpressionResolver.resolveWriteIndexAbstraction(state2, request) @@ -3226,6 +3237,7 @@ public void testResolveWriteIndexAbstractionNoWriteIndexForAlias() { + " or the alias points to multiple indices without one being designated as a write index" ) ); + request.decRef(); } public void testResolveWriteIndexAbstractionMissing() { @@ -3233,8 +3245,9 @@ public void testResolveWriteIndexAbstractionMissing() { List.of(new Tuple<>("logs-foobar", 1)), List.of("my-index") ); - DocWriteRequest request = new IndexRequest("logs-my-index"); + IndexRequest request = new IndexRequest("logs-my-index"); expectThrows(IndexNotFoundException.class, () -> indexNameExpressionResolver.resolveWriteIndexAbstraction(state, request)); + request.decRef(); } public void testResolveWriteIndexAbstractionMultipleMatches() { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 8af74e03f8605..52bc55099cd5c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -49,6 +49,7 @@ public void testSimpleRoutingRejectsEmptyId() { IndexRequest req = new IndexRequest().id(""); Exception e = expectThrows(IllegalArgumentException.class, () -> indexRouting.process(req)); assertThat(e.getMessage(), equalTo("if _id is specified it must not be empty")); + req.decRef(); } public void testSimpleRoutingAcceptsId() { @@ -60,6 +61,7 @@ public void testSimpleRoutingAcceptsId() { indexRouting.process(req); assertThat(req.id(), equalTo(id)); assertThat(req.getAutoGeneratedTimestamp(), equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP)); + req.decRef(); } public void testSimpleRoutingAssignedRandomId() { @@ -70,6 +72,7 @@ public void testSimpleRoutingAssignedRandomId() { indexRouting.process(req); assertThat(req.id(), not(nullValue())); assertThat(req.getAutoGeneratedTimestamp(), not(equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP))); + req.decRef(); } public void testGenerateShardId() { @@ -471,6 +474,7 @@ public void testRoutingAllowsId() { IndexRequest req = new IndexRequest().id(id); indexRouting.process(req); assertThat(req.id(), equalTo(id)); + req.decRef(); } /** @@ -484,6 +488,7 @@ public void testRoutingPathLeavesIdNull() { IndexRequest req = new IndexRequest(); indexRouting.process(req); assertThat(req.id(), nullValue()); + req.decRef(); } public void testRoutingPathEmptySource() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java index d2304908a933b..8e72691466c2b 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -10,6 +10,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TopDocs; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; @@ -275,7 +276,9 @@ public void testRefreshActuallyWorks() throws Exception { assertEquals(1000, refreshTask.getInterval().millis()); assertTrue(indexService.getRefreshTask().mustReschedule()); IndexShard shard = indexService.getShard(0); - prepareIndex("test").setId("0").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("0").setSource("{\"foo\": \"bar\"}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); // now disable the refresh indicesAdmin().prepareUpdateSettings("test") .setSettings(Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)) @@ -294,7 +297,9 @@ public void testRefreshActuallyWorks() throws Exception { }); assertFalse(refreshTask.isClosed()); // refresh every millisecond - prepareIndex("test").setId("1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); + indexRequestBuilder = prepareIndex("test").setId("1").setSource("{\"foo\": \"bar\"}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); indicesAdmin().prepareUpdateSettings("test") .setSettings(Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1ms")) .get(); @@ -306,7 +311,9 @@ public void testRefreshActuallyWorks() throws Exception { assertEquals(2, search.totalHits.value); } }); - prepareIndex("test").setId("2").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); + indexRequestBuilder = prepareIndex("test").setId("2").setSource("{\"foo\": \"bar\"}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertBusy(() -> { // this one becomes visible due to the scheduled refresh try (Engine.Searcher searcher = shard.acquireSearcher("test")) { @@ -324,7 +331,9 @@ public void testAsyncFsyncActuallyWorks() throws Exception { IndexService indexService = createIndex("test", settings); ensureGreen("test"); assertTrue(indexService.getRefreshTask().mustReschedule()); - prepareIndex("test").setId("1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("{\"foo\": \"bar\"}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); IndexShard shard = indexService.getShard(0); assertBusy(() -> assertFalse(shard.isSyncNeeded())); } @@ -344,7 +353,9 @@ public void testRescheduleAsyncFsync() throws Exception { assertNotNull(indexService.getFsyncTask()); assertTrue(indexService.getFsyncTask().mustReschedule()); - prepareIndex("test").setId("1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("{\"foo\": \"bar\"}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNotNull(indexService.getFsyncTask()); final IndexShard shard = indexService.getShard(0); assertBusy(() -> assertFalse(shard.isSyncNeeded())); @@ -367,7 +378,9 @@ public void testAsyncTranslogTrimActuallyWorks() throws Exception { IndexService indexService = createIndex("test", settings); ensureGreen("test"); assertTrue(indexService.getTrimTranslogTask().mustReschedule()); - prepareIndex("test").setId("1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("{\"foo\": \"bar\"}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); indicesAdmin().prepareFlush("test").get(); IndexShard shard = indexService.getShard(0); assertBusy(() -> assertThat(IndexShardTestCase.getTranslog(shard).totalOperations(), equalTo(0))); @@ -384,7 +397,10 @@ public void testAsyncTranslogTrimTaskOnClosedIndex() throws Exception { int translogOps = 0; final int numDocs = scaledRandomIntBetween(10, 100); for (int i = 0; i < numDocs; i++) { - prepareIndex(indexName).setId(String.valueOf(i)).setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId(String.valueOf(i)) + .setSource("{\"foo\": \"bar\"}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); translogOps++; if (randomBoolean()) { indicesAdmin().prepareFlush(indexName).get(); diff --git a/server/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java b/server/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java index 6577148d78c7b..9736ac393724d 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.fieldstats; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.index.query.QueryBuilders; @@ -93,7 +94,12 @@ private void refreshIndex() { } private void indexDocument(String id, String sValue) { - DocWriteResponse response = prepareIndex("index").setId(id).setSource("s", sValue).get(); - assertThat(response.status(), anyOf(equalTo(RestStatus.OK), equalTo(RestStatus.CREATED))); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId(id).setSource("s", sValue); + try { + DocWriteResponse response = indexRequestBuilder.get(); + assertThat(response.status(), anyOf(equalTo(RestStatus.OK), equalTo(RestStatus.CREATED))); + } finally { + indexRequestBuilder.request().decRef(); + } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSearchTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSearchTests.java index 39a67bb2229bc..5e08dcbeae73b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSearchTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSearchTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; @@ -75,7 +76,7 @@ public void setUpIndex() throws IOException { } public void testMatchQuery() throws Exception { - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource( XContentFactory.jsonBuilder() @@ -85,8 +86,9 @@ public void testMatchQuery() throws Exception { .field("origin", "https://www.elastic.co") .endObject() .endObject() - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCount(client().prepareSearch().setQuery(matchQuery("headers", "application/json")), 1L); @@ -96,7 +98,7 @@ public void testMatchQuery() throws Exception { } public void testMultiMatchQuery() throws Exception { - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource( XContentFactory.jsonBuilder() @@ -106,8 +108,9 @@ public void testMultiMatchQuery() throws Exception { .field("origin", "https://www.elastic.co") .endObject() .endObject() - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCount(client().prepareSearch().setQuery(multiMatchQuery("application/json", "headers")), 1L); assertHitCount(client().prepareSearch().setQuery(multiMatchQuery("application/json text/plain", "headers.content-type")), 1L); @@ -116,7 +119,7 @@ public void testMultiMatchQuery() throws Exception { } public void testQueryStringQuery() throws Exception { - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource( XContentFactory.jsonBuilder() @@ -126,8 +129,9 @@ public void testQueryStringQuery() throws Exception { .field("field2", "2.718") .endObject() .endObject() - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCountAndNoFailures(client().prepareSearch("test").setQuery(queryStringQuery("flattened.field1:value")), 1); assertHitCountAndNoFailures( @@ -138,7 +142,7 @@ public void testQueryStringQuery() throws Exception { } public void testSimpleQueryStringQuery() throws Exception { - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource( XContentFactory.jsonBuilder() @@ -148,8 +152,9 @@ public void testSimpleQueryStringQuery() throws Exception { .field("field2", "2.718") .endObject() .endObject() - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCountAndNoFailures(client().prepareSearch("test").setQuery(simpleQueryStringQuery("value").field("flattened.field1")), 1); assertHitCountAndNoFailures(client().prepareSearch("test").setQuery(simpleQueryStringQuery("+value +2.718").field("flattened")), 1); @@ -157,7 +162,7 @@ public void testSimpleQueryStringQuery() throws Exception { } public void testExists() throws Exception { - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource( XContentFactory.jsonBuilder() @@ -166,8 +171,9 @@ public void testExists() throws Exception { .field("content-type", "application/json") .endObject() .endObject() - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCount(client().prepareSearch().setQuery(existsQuery("headers")), 1L); assertHitCount(client().prepareSearch().setQuery(existsQuery("headers.content-type")), 1L); @@ -183,22 +189,24 @@ public void testCardinalityAggregation() throws IOException { // Add a random number of documents containing a flattened field, plus // a small number of dummy documents. for (int i = 0; i < numDocs; ++i) { - bulkRequest.add( - client().prepareIndex() - .setSource( - XContentFactory.jsonBuilder() - .startObject() - .startObject("flattened") - .field("first", i) - .field("second", i / 2) - .endObject() - .endObject() - ) - ); + IndexRequestBuilder indexRequestBuilder = client().prepareIndex() + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject("flattened") + .field("first", i) + .field("second", i / 2) + .endObject() + .endObject() + ); + bulkRequest.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } for (int i = 0; i < 10; i++) { - bulkRequest.add(prepareIndex("test").setSource("other_field", "1")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource("other_field", "1"); + bulkRequest.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } BulkResponse bulkResponse = bulkRequest.get(); @@ -249,18 +257,18 @@ private void assertCardinality(Cardinality count, long value, int precisionThres public void testTermsAggregation() throws IOException { try (BulkRequestBuilder bulkRequest = client().prepareBulk("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE)) { for (int i = 0; i < 5; i++) { - bulkRequest.add( - client().prepareIndex() - .setSource( - XContentFactory.jsonBuilder() - .startObject() - .startObject("labels") - .field("priority", "urgent") - .field("release", "v1.2." + i) - .endObject() - .endObject() - ) - ); + IndexRequestBuilder indexRequestBuilder = client().prepareIndex() + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject("labels") + .field("priority", "urgent") + .field("release", "v1.2." + i) + .endObject() + .endObject() + ); + bulkRequest.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } BulkResponse bulkResponse = bulkRequest.get(); @@ -329,7 +337,7 @@ private TermsAggregationBuilder createTermsAgg(String field) { } public void testLoadDocValuesFields() throws Exception { - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource( XContentFactory.jsonBuilder() @@ -339,8 +347,9 @@ public void testLoadDocValuesFields() throws Exception { .field("other_key", "other_value") .endObject() .endObject() - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNoFailuresAndResponse( client().prepareSearch("test").addDocValueField("flattened").addDocValueField("flattened.key"), @@ -361,7 +370,7 @@ public void testLoadDocValuesFields() throws Exception { } public void testFieldSort() throws Exception { - prepareIndex("test").setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource( XContentFactory.jsonBuilder() @@ -371,10 +380,11 @@ public void testFieldSort() throws Exception { .field("other_key", "D") .endObject() .endObject() - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex("test").setId("2") + indexRequestBuilder = prepareIndex("test").setId("2") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource( XContentFactory.jsonBuilder() @@ -384,13 +394,17 @@ public void testFieldSort() throws Exception { .field("other_key", "C") .endObject() .endObject() - ) - .get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex("test").setId("3") + indexRequestBuilder = prepareIndex("test").setId("3") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .setSource(XContentFactory.jsonBuilder().startObject().startObject("flattened").field("other_key", "E").endObject().endObject()) - .get(); + .setSource( + XContentFactory.jsonBuilder().startObject().startObject("flattened").field("other_key", "E").endObject().endObject() + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNoFailuresAndResponse(client().prepareSearch("test").addSort("flattened", SortOrder.DESC), response -> { assertHitCount(response, 3); @@ -416,7 +430,11 @@ public void testSourceFiltering() { headers.put("origin", "https://www.elastic.co"); Map source = Collections.singletonMap("headers", headers); - prepareIndex("test").setId("1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).setSource(source).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1") + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .setSource(source); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertResponse( client().prepareSearch("test").setFetchSource(true), diff --git a/server/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java b/server/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java index 1cf8436048a27..b79ead91d2b69 100644 --- a/server/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java @@ -72,6 +72,8 @@ public void testForSlice() { assertEquals(slicingTask, forSliced.getParentTask()); extraForSliceAssertions(original, forSliced); + forSliced.decRef(); + original.decRef(); } protected abstract R newRequest(); diff --git a/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java index 337b6effc3252..9aa8aedb7897a 100644 --- a/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java @@ -159,6 +159,7 @@ public void testReindexFromRemoteDoesNotSupportSearchQuery() { "Validation Failed: 1: reindex from remote sources should use RemoteInfo's query instead of source's query;", e.getMessage() ); + reindex.decRef(); } public void testReindexFromRemoteDoesNotSupportSlices() { @@ -183,6 +184,8 @@ public void testReindexFromRemoteDoesNotSupportSlices() { "Validation Failed: 1: reindex from remote sources doesn't support slices > 1 but was [" + reindex.getSlices() + "];", e.getMessage() ); + reindex.decRef(); + } public void testNoSliceBuilderSetWithSlicedRequest() { @@ -191,6 +194,7 @@ public void testNoSliceBuilderSetWithSlicedRequest() { reindex.setSlices(between(2, Integer.MAX_VALUE)); ActionRequestValidationException e = reindex.validate(); assertEquals("Validation Failed: 1: can't specify both manual and automatic slicing at the same time;", e.getMessage()); + reindex.decRef(); } @Override @@ -352,6 +356,7 @@ public void testReindexFromRemoteRequestParsing() throws IOException { ReindexRequest r = ReindexRequest.fromXContent(p); assertEquals("localhost", r.getRemoteInfo().getHost()); assertArrayEquals(new String[] { "source" }, r.getSearchRequest().indices()); + r.decRef(); } } @@ -368,11 +373,13 @@ private RemoteInfo buildRemoteInfoHostTestCase(String hostInRest) throws IOExcep public void testCommaSeparatedSourceIndices() throws IOException { ReindexRequest r = parseRequestWithSourceIndices("a,b"); assertArrayEquals(new String[] { "a", "b" }, r.getSearchRequest().indices()); + r.decRef(); } public void testArraySourceIndices() throws IOException { ReindexRequest r = parseRequestWithSourceIndices(new String[] { "a", "b" }); assertArrayEquals(new String[] { "a", "b" }, r.getSearchRequest().indices()); + r.decRef(); } public void testEmptyStringSourceIndices() throws IOException { @@ -381,6 +388,7 @@ public void testEmptyStringSourceIndices() throws IOException { ActionRequestValidationException validationException = r.validate(); assertNotNull(validationException); assertEquals(List.of("use _all if you really want to copy from all existing indexes"), validationException.validationErrors()); + r.decRef(); } private ReindexRequest parseRequestWithSourceIndices(Object sourceIndices) throws IOException { @@ -406,4 +414,9 @@ private ReindexRequest parseRequestWithSourceIndices(Object sourceIndices) throw return ReindexRequest.fromXContent(p); } } + + @Override + protected void dispose(ReindexRequest instance) { + instance.decRef(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java index 4bde3d5688d06..86a3758f23ad2 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java @@ -154,6 +154,8 @@ public void testRetryAppendOnlyAfterRecovering() throws Exception { shards.index(originalRequest); // original append-only arrives after recovery completed shards.assertAllEqual(1); assertThat(replica.getMaxSeenAutoIdTimestamp(), equalTo(originalRequest.getAutoGeneratedTimestamp())); + originalRequest.decRef(); + retryRequest.decRef(); } } @@ -186,6 +188,7 @@ public IndexResult index(Index op) throws IOException { } catch (Exception e) { throw new AssertionError(e); } + indexRequest.decRef(); }); thread.start(); IndexShard replica = shards.addReplica(); @@ -212,6 +215,7 @@ public void testInheritMaxValidAutoIDTimestampOnRecovery() throws Exception { final IndexRequest indexRequest = new IndexRequest(index.getName()).source("{}", XContentType.JSON); indexRequest.onRetry(); // force an update of the timestamp final BulkItemResponse response = shards.index(indexRequest); + indexRequest.decRef(); assertEquals(DocWriteResponse.Result.CREATED, response.getResponse().getResult()); if (randomBoolean()) { // lets check if that also happens if no translog record is replicated shards.flush(); @@ -293,11 +297,14 @@ public void testConflictingOpsOnReplica() throws Exception { for (int i = 1; i < replicas.size(); i++) { indexOnReplica(replicationRequest, shards, replicas.get(i)); } + indexRequest.decRef(); + replicationRequest.decRef(); logger.info("--> promoting replica to primary " + replica1.routingEntry()); shards.promoteReplicaToPrimary(replica1).get(); indexRequest = new IndexRequest(index.getName()).id("1").source("{ \"f\": \"2\"}", XContentType.JSON); shards.index(indexRequest); + indexRequest.decRef(); shards.refresh("test"); for (IndexShard shard : shards) { try (Engine.Searcher searcher = shard.acquireSearcher("test")) { @@ -326,6 +333,7 @@ public void testReplicaTermIncrementWithConcurrentPrimaryPromotion() throws Exce IndexRequest indexRequest = new IndexRequest(index.getName()).id("1").source("{ \"f\": \"1\"}", XContentType.JSON); BulkShardRequest replicationRequest = indexOnPrimary(indexRequest, replica1); + indexRequest.decRef(); CyclicBarrier barrier = new CyclicBarrier(2); Thread t1 = new Thread(() -> { @@ -356,6 +364,7 @@ public void testReplicaTermIncrementWithConcurrentPrimaryPromotion() throws Exce t2.join(); assertEquals(newReplica1Term + 1, replica2.getPendingPrimaryTerm()); + replicationRequest.decRef(); } } @@ -367,6 +376,7 @@ public void testReplicaOperationWithConcurrentPrimaryPromotion() throws Exceptio long primaryPrimaryTerm = shards.getPrimary().getPendingPrimaryTerm(); IndexRequest indexRequest = new IndexRequest(index.getName()).id("1").source("{ \"f\": \"1\"}", XContentType.JSON); BulkShardRequest replicationRequest = indexOnPrimary(indexRequest, shards.getPrimary()); + indexRequest.decRef(); List replicas = shards.getReplicas(); IndexShard replica = replicas.get(0); @@ -409,6 +419,7 @@ public void testReplicaOperationWithConcurrentPrimaryPromotion() throws Exceptio assertThat(op.primaryTerm(), equalTo(primaryPrimaryTerm)); } } + replicationRequest.decRef(); } } @@ -445,9 +456,11 @@ protected EngineFactory getEngineFactory(ShardRouting routing) { shards.startPrimary(); long primaryTerm = shards.getPrimary().getPendingPrimaryTerm(); List expectedTranslogOps = new ArrayList<>(); - BulkItemResponse indexResp = shards.index(new IndexRequest(index.getName()).id("1").source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id("1").source("{}", XContentType.JSON); + BulkItemResponse indexResp = shards.index(indexRequest); assertThat(indexResp.isFailed(), equalTo(true)); assertThat(indexResp.getFailure().getCause(), equalTo(indexException)); + indexRequest.decRef(); expectedTranslogOps.add(new Translog.NoOp(0, primaryTerm, indexException.toString())); try (Translog.Snapshot snapshot = getTranslog(shards.getPrimary()).newSnapshot()) { assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps)); @@ -482,7 +495,9 @@ protected EngineFactory getEngineFactory(ShardRouting routing) { } } // the failure replicated directly from the replication channel. - indexResp = shards.index(new IndexRequest(index.getName()).id("any").source("{}", XContentType.JSON)); + indexRequest = new IndexRequest(index.getName()).id("any").source("{}", XContentType.JSON); + indexResp = shards.index(indexRequest); + indexRequest.decRef(); assertThat(indexResp.getFailure().getCause(), equalTo(indexException)); Translog.NoOp noop2 = new Translog.NoOp(1, primaryTerm, indexException.toString()); expectedTranslogOps.add(noop2); @@ -518,7 +533,9 @@ protected EngineFactory getEngineFactory(ShardRouting routing) { public void testRequestFailureReplication() throws Exception { try (ReplicationGroup shards = createGroup(0)) { shards.startAll(); - BulkItemResponse response = shards.index(new IndexRequest(index.getName()).id("1").source("{}", XContentType.JSON).version(2)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id("1").source("{}", XContentType.JSON).version(2); + BulkItemResponse response = shards.index(indexRequest); + indexRequest.decRef(); assertTrue(response.isFailed()); assertThat(response.getFailure().getCause(), instanceOf(VersionConflictEngineException.class)); shards.assertAllEqual(0); @@ -536,7 +553,9 @@ public void testRequestFailureReplication() throws Exception { shards.addReplica(); } shards.startReplicas(nReplica); - response = shards.index(new IndexRequest(index.getName()).id("1").source("{}", XContentType.JSON).version(2)); + indexRequest = new IndexRequest(index.getName()).id("1").source("{}", XContentType.JSON).version(2); + response = shards.index(indexRequest); + indexRequest.decRef(); assertTrue(response.isFailed()); assertThat(response.getFailure().getCause(), instanceOf(VersionConflictEngineException.class)); shards.assertAllEqual(0); @@ -562,7 +581,9 @@ public void testSeqNoCollision() throws Exception { logger.info("--> Isolate replica1"); IndexRequest indexDoc1 = new IndexRequest(index.getName()).id("d1").source("{}", XContentType.JSON); BulkShardRequest replicationRequest = indexOnPrimary(indexDoc1, shards.getPrimary()); + indexDoc1.decRef(); indexOnReplica(replicationRequest, shards, replica2); + replicationRequest.decRef(); final Translog.Operation op1; final List initOperations = new ArrayList<>(initDocs); @@ -580,7 +601,9 @@ public void testSeqNoCollision() throws Exception { } logger.info("--> Promote replica1 as the primary"); shards.promoteReplicaToPrimary(replica1).get(); // wait until resync completed. - shards.index(new IndexRequest(index.getName()).id("d2").source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id("d2").source("{}", XContentType.JSON); + shards.index(indexRequest); + indexRequest.decRef(); final Translog.Operation op2; try (Translog.Snapshot snapshot = getTranslog(replica2).newSnapshot()) { assertThat(snapshot.totalOperations(), equalTo(1)); @@ -629,18 +652,19 @@ public void testLateDeliveryAfterGCTriggeredOnReplica() throws Exception { // I think we can just set this to something very small (10ms?) and also set ThreadPool#ESTIMATED_TIME_INTERVAL_SETTING to 0? updateGCDeleteCycle(replica, gcInterval); - final BulkShardRequest indexRequest = indexOnPrimary( - new IndexRequest(index.getName()).id("d1").source("{}", XContentType.JSON), - primary - ); + IndexRequest singleIndexRequest = new IndexRequest(index.getName()).id("d1").source("{}", XContentType.JSON); + final BulkShardRequest indexRequest = indexOnPrimary(singleIndexRequest, primary); + singleIndexRequest.decRef(); final BulkShardRequest deleteRequest = deleteOnPrimary(new DeleteRequest(index.getName()).id("d1"), primary); deleteOnReplica(deleteRequest, shards, replica); // delete arrives on replica first. + deleteRequest.decRef(); final long deleteTimestamp = threadPool.relativeTimeInMillis(); replica.refresh("test"); assertBusy(() -> assertThat(threadPool.relativeTimeInMillis() - deleteTimestamp, greaterThan(gcInterval.millis()))); getEngine(replica).maybePruneDeletes(); indexOnReplica(indexRequest, shards, replica); // index arrives on replica lately. shards.assertAllEqual(0); + indexRequest.decRef(); } } @@ -666,15 +690,16 @@ public void testOutOfOrderDeliveryForAppendOnlyOperations() throws Exception { final IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); // Append-only request - without id - final BulkShardRequest indexRequest = indexOnPrimary( - new IndexRequest(index.getName()).source("{}", XContentType.JSON), - primary - ); + IndexRequest singleIndexRequest = new IndexRequest(index.getName()).source("{}", XContentType.JSON); + final BulkShardRequest indexRequest = indexOnPrimary(singleIndexRequest, primary); + singleIndexRequest.decRef(); final String docId = Iterables.get(getShardDocUIDs(primary), 0); final BulkShardRequest deleteRequest = deleteOnPrimary(new DeleteRequest(index.getName()).id(docId), primary); deleteOnReplica(deleteRequest, shards, replica); + deleteRequest.decRef(); indexOnReplica(indexRequest, shards, replica); shards.assertAllEqual(0); + indexRequest.decRef(); } } @@ -687,10 +712,12 @@ public void testIndexingOptimizationUsingSequenceNumbers() throws Exception { for (int i = 0; i < numDocs; i++) { String id = Integer.toString(randomIntBetween(1, 100)); if (randomBoolean()) { - group.index(new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON); + group.index(indexRequest); if (liveDocs.add(id) == false) { versionLookups++; } + indexRequest.decRef(); } else { group.delete(new DeleteRequest(index.getName()).id(id)); liveDocs.remove(id); diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java index 798f52cfbdc19..4b0bccc49cf42 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java @@ -115,7 +115,9 @@ public void testRecoveryToReplicaThatReceivedExtraDocument() throws Exception { shards.startAll(); final int docs = randomIntBetween(0, 16); for (int i = 0; i < docs; i++) { - shards.index(new IndexRequest("index").id(Integer.toString(i)).source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest("index").id(Integer.toString(i)).source("{}", XContentType.JSON); + shards.index(indexRequest); + indexRequest.decRef(); } shards.flush(); @@ -184,7 +186,9 @@ public void testRecoveryAfterPrimaryPromotion() throws Exception { for (int i = 0; i < rollbackDocs; i++) { final IndexRequest indexRequest = new IndexRequest(index.getName()).id("rollback_" + i).source("{}", XContentType.JSON); final BulkShardRequest bulkShardRequest = indexOnPrimary(indexRequest, oldPrimary); + indexRequest.decRef(); indexOnReplica(bulkShardRequest, shards, replica); + bulkShardRequest.decRef(); } if (randomBoolean()) { oldPrimary.flush(new FlushRequest(index.getName())); @@ -300,7 +304,9 @@ public void testReplicaRollbackStaleDocumentsInPeerRecovery() throws Exception { for (int i = 0; i < staleDocs; i++) { final IndexRequest indexRequest = new IndexRequest(index.getName()).id("stale_" + i).source("{}", XContentType.JSON); final BulkShardRequest bulkShardRequest = indexOnPrimary(indexRequest, oldPrimary); + indexRequest.decRef(); indexOnReplica(bulkShardRequest, shards, replica); + bulkShardRequest.decRef(); } shards.flush(); shards.promoteReplicaToPrimary(newPrimary).get(); @@ -338,6 +344,7 @@ public void testResyncAfterPrimaryPromotion() throws Exception { final IndexRequest indexRequest = new IndexRequest(index.getName()).id("initial_doc_" + i) .source("{ \"f\": \"normal\"}", XContentType.JSON); shards.index(indexRequest); + indexRequest.decRef(); } boolean syncedGlobalCheckPoint = randomBoolean(); @@ -356,7 +363,9 @@ public void testResyncAfterPrimaryPromotion() throws Exception { final IndexRequest indexRequest = new IndexRequest(index.getName()).id("extra_doc_" + i) .source("{ \"f\": \"normal\"}", XContentType.JSON); final BulkShardRequest bulkShardRequest = indexOnPrimary(indexRequest, oldPrimary); + indexRequest.decRef(); indexOnReplica(bulkShardRequest, shards, newPrimary); + bulkShardRequest.decRef(); } final int extraDocsToBeTrimmed = randomIntBetween(0, 10); @@ -365,8 +374,10 @@ public void testResyncAfterPrimaryPromotion() throws Exception { final IndexRequest indexRequest = new IndexRequest(index.getName()).id("extra_trimmed_" + i) .source("{ \"f\": \"trimmed\"}", XContentType.JSON); final BulkShardRequest bulkShardRequest = indexOnPrimary(indexRequest, oldPrimary); + indexRequest.decRef(); // have to replicate to another replica != newPrimary one - the subject to trim indexOnReplica(bulkShardRequest, shards, justReplica); + bulkShardRequest.decRef(); } logger.info("--> resyncing replicas seqno_stats primary {} replica {}", oldPrimary.seqNoStats(), newPrimary.seqNoStats()); @@ -430,7 +441,9 @@ protected EngineFactory getEngineFactory(ShardRouting routing) { final String id = "pending_" + i; threadPool.generic().submit(() -> { try { - shards.index(new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON); + shards.index(indexRequest); + indexRequest.decRef(); } catch (Exception e) { throw new AssertionError(e); } finally { @@ -521,7 +534,9 @@ public void indexTranslogOperations( replicaEngineFactory.latchIndexers(1); threadPool.generic().submit(() -> { try { - shards.index(new IndexRequest(index.getName()).id("pending").source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id("pending").source("{}", XContentType.JSON); + shards.index(indexRequest); + indexRequest.decRef(); } catch (final Exception e) { throw new RuntimeException(e); } finally { @@ -533,7 +548,9 @@ public void indexTranslogOperations( replicaEngineFactory.awaitIndexersLatch(); // unblock indexing for the next doc replicaEngineFactory.allowIndexing(); - shards.index(new IndexRequest(index.getName()).id("completed").source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id("completed").source("{}", XContentType.JSON); + shards.index(indexRequest); + indexRequest.decRef(); pendingDocActiveWithExtraDocIndexed.countDown(); } catch (final Exception e) { throw new AssertionError(e); @@ -569,7 +586,9 @@ public void indexTranslogOperations( // wait for the translog phase to complete and the recovery to block global checkpoint advancement assertBusy(() -> assertTrue(shards.getPrimary().pendingInSync())); { - shards.index(new IndexRequest(index.getName()).id("last").source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id("last").source("{}", XContentType.JSON); + shards.index(indexRequest); + indexRequest.decRef(); final long expectedDocs = docs + 3L; assertThat(shards.getPrimary().getLocalCheckpoint(), equalTo(expectedDocs - 1)); // recovery is now in the process of being completed, therefore the global checkpoint can not have advanced on the primary @@ -623,6 +642,8 @@ public void testTransferMaxSeenAutoIdTimestampOnResync() throws Exception { indexOnReplica(bulkShardRequest, shards, replica2); maxTimestampOnReplica2 = Math.max(maxTimestampOnReplica2, indexRequest.getAutoGeneratedTimestamp()); } + indexRequest.decRef(); + bulkShardRequest.decRef(); } assertThat(replica1.getMaxSeenAutoIdTimestamp(), equalTo(maxTimestampOnReplica1)); assertThat(replica2.getMaxSeenAutoIdTimestamp(), equalTo(maxTimestampOnReplica2)); @@ -630,6 +651,7 @@ public void testTransferMaxSeenAutoIdTimestampOnResync() throws Exception { assertThat(replica2.getMaxSeenAutoIdTimestamp(), equalTo(maxTimestampOnReplica1)); for (IndexRequest request : replicationRequests) { shards.index(request); // deliver via normal replication + request.decRef(); } for (IndexShard shard : shards) { assertThat(shard.getMaxSeenAutoIdTimestamp(), equalTo(Math.max(maxTimestampOnReplica1, maxTimestampOnReplica2))); @@ -662,10 +684,14 @@ public void testAddNewReplicas() throws Exception { int nextId = docId.incrementAndGet(); if (appendOnly) { String id = randomBoolean() ? Integer.toString(nextId) : null; - shards.index(new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON); + shards.index(indexRequest); + indexRequest.decRef(); } else if (frequently()) { String id = Integer.toString(frequently() ? nextId : between(0, nextId)); - shards.index(new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON)); + IndexRequest indexRequest = new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON); + shards.index(indexRequest); + indexRequest.decRef(); } else { String id = Integer.toString(between(0, nextId)); shards.delete(new DeleteRequest(index.getName()).id(id)); @@ -705,6 +731,7 @@ public void testRollbackOnPromotion() throws Exception { String id = "extra-" + i; IndexRequest primaryRequest = new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON); BulkShardRequest replicationRequest = indexOnPrimary(primaryRequest, shards.getPrimary()); + primaryRequest.decRef(); for (IndexShard replica : shards.getReplicas()) { if (randomBoolean()) { indexOnReplica(replicationRequest, shards, replica); @@ -713,6 +740,7 @@ public void testRollbackOnPromotion() throws Exception { } } } + replicationRequest.decRef(); if (randomBoolean()) { shards.syncGlobalCheckpoint(); } diff --git a/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java b/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java index 8967390dc4fe6..9b6c2578d29fe 100644 --- a/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.common.settings.Settings; @@ -48,7 +49,9 @@ public void testTook() throws Exception { createIndex("test", Settings.EMPTY, mapping); ensureGreen(); - prepareIndex("test").setId("0").setSource("field", "foo bar").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("0").setSource("field", "foo bar").setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService(resolveIndex("test")); @@ -82,7 +85,10 @@ public void testDocFreqs() throws IOException { int max = between(3, 10); try (BulkRequestBuilder bulk = client().prepareBulk()) { for (int i = 0; i < max; i++) { - bulk.add(prepareIndex("test").setId(Integer.toString(i)).setSource("text", "the quick brown fox jumped over the lazy dog")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)) + .setSource("text", "the quick brown fox jumped over the lazy dog"); + bulk.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } bulk.get(); } @@ -122,7 +128,10 @@ public void testWithIndexedPhrases() throws IOException { int max = between(3, 10); try (BulkRequestBuilder bulk = client().prepareBulk()) { for (int i = 0; i < max; i++) { - bulk.add(prepareIndex("test").setId(Integer.toString(i)).setSource("text", "the quick brown fox jumped over the lazy dog")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)) + .setSource("text", "the quick brown fox jumped over the lazy dog"); + bulk.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } bulk.get(); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java index 8fbd9e881f698..7a0f62135b4c8 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.common.bytes.BytesArray; @@ -136,7 +137,12 @@ public void testCloseWhileOngoingRequest() throws Exception { assertEquals(1, indicesService.indicesRefCount.refCount()); assertAcked(node.client().admin().indices().prepareCreate("test").setSettings(indexSettings(1, 0))); - node.client().prepareIndex("test").setId("1").setSource(Collections.emptyMap()).get(); + IndexRequestBuilder indexRequestBuilder = node.client().prepareIndex("test").setId("1").setSource(Collections.emptyMap()); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } ElasticsearchAssertions.assertAllSuccessful(node.client().admin().indices().prepareRefresh("test").get()); assertEquals(2, indicesService.indicesRefCount.refCount()); @@ -165,7 +171,15 @@ public void testCloseAfterRequestHasUsedQueryCache() throws Exception { .prepareCreate("test") .setSettings(indexSettings(1, 0).put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)) ); - node.client().prepareIndex("test").setId("1").setSource(Collections.singletonMap("foo", 3L)).get(); + IndexRequestBuilder indexRequestBuilder = node.client() + .prepareIndex("test") + .setId("1") + .setSource(Collections.singletonMap("foo", 3L)); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } ElasticsearchAssertions.assertAllSuccessful(node.client().admin().indices().prepareRefresh("test").get()); assertEquals(2, indicesService.indicesRefCount.refCount()); @@ -203,7 +217,15 @@ public void testCloseWhileOngoingRequestUsesQueryCache() throws Exception { .prepareCreate("test") .setSettings(indexSettings(1, 0).put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)) ); - node.client().prepareIndex("test").setId("1").setSource(Collections.singletonMap("foo", 3L)).get(); + IndexRequestBuilder indexRequestBuilder = node.client() + .prepareIndex("test") + .setId("1") + .setSource(Collections.singletonMap("foo", 3L)); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } ElasticsearchAssertions.assertAllSuccessful(node.client().admin().indices().prepareRefresh("test").get()); assertEquals(2, indicesService.indicesRefCount.refCount()); @@ -240,7 +262,15 @@ public void testCloseWhileOngoingRequestUsesRequestCache() throws Exception { .prepareCreate("test") .setSettings(indexSettings(1, 0).put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)) ); - node.client().prepareIndex("test").setId("1").setSource(Collections.singletonMap("foo", 3L)).get(); + IndexRequestBuilder indexRequestBuilder = node.client() + .prepareIndex("test") + .setId("1") + .setSource(Collections.singletonMap("foo", 3L)); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } ElasticsearchAssertions.assertAllSuccessful(node.client().admin().indices().prepareRefresh("test").get()); assertEquals(2, indicesService.indicesRefCount.refCount()); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index 846625fc4f790..01478218c8aba 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; @@ -268,7 +269,9 @@ public void testDeleteIndexStore() throws Exception { assertNull(meta.index("test")); test = createIndex("test"); - prepareIndex("test").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); client().admin().indices().prepareFlush("test").get(); assertHitCount(client().prepareSearch("test"), 1); IndexMetadata secondMetadata = clusterService.state().metadata().index("test"); diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java index 2cbe1202520df..44bd5f81cfcee 100644 --- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.indices.breaker; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.settings.Settings; @@ -122,10 +123,12 @@ public void testCircuitBreakerTripCountMetric() { .get() .isAcknowledged() ); - assertEquals( - RestStatus.OK.getStatus(), - client().prepareIndex("test").setWaitForActiveShards(1).setSource("field", "value").get().status().getStatus() - ); + IndexRequestBuilder indexRequestBuilder = client().prepareIndex("test").setWaitForActiveShards(1).setSource("field", "value"); + try { + assertEquals(RestStatus.OK.getStatus(), indexRequestBuilder.get().status().getStatus()); + } finally { + indexRequestBuilder.request().decRef(); + } } catch (CircuitBreakingException cbex) { final List dataNodeMeasurements = getMeasurements(dataNodeName); final List masterNodeMeasurements = getMeasurements(masterNodeName); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java index 20e85c9c6fed8..53f6f33c1c50b 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java @@ -463,12 +463,14 @@ public void testRecoveryTrimsLocalTranslog() throws Exception { for (int i = 0; i < inflightDocs; i++) { final IndexRequest indexRequest = new IndexRequest(index.getName()).id("extra_" + i).source("{}", XContentType.JSON); final BulkShardRequest bulkShardRequest = indexOnPrimary(indexRequest, oldPrimary); + indexRequest.decRef(); for (IndexShard replica : randomSubsetOf(shards.getReplicas())) { indexOnReplica(bulkShardRequest, shards, replica); } if (rarely()) { shards.flush(); } + bulkShardRequest.decRef(); } shards.syncGlobalCheckpoint(); shards.promoteReplicaToPrimary(randomFrom(shards.getReplicas())).get(); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index b7501aee693c7..06ab46797d20d 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -212,6 +212,7 @@ public void testExecuteIndexPipelineDoesNotExist() { assertTrue(failure.get()); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); + indexRequest.decRef(); } public void testUpdatePipelines() { @@ -1096,6 +1097,8 @@ public String getType() { bulkRequest.add(indexRequest1); IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(Map.of()).setPipeline(id).setFinalPipeline("_none"); bulkRequest.add(indexRequest2); + indexRequest1.decRef(); + indexRequest2.decRef(); final BiConsumer failureHandler = (slot, e) -> { assertThat(e.getCause(), instanceOf(IllegalStateException.class)); @@ -1152,6 +1155,9 @@ public void testExecuteBulkPipelineDoesNotExist() { .setPipeline("does_not_exist") .setFinalPipeline("_none"); bulkRequest.add(indexRequest3); + indexRequest1.decRef(); + indexRequest2.decRef(); + indexRequest3.decRef(); @SuppressWarnings("unchecked") BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") @@ -1215,8 +1221,10 @@ public void close() { UpdateRequest updateRequest = new UpdateRequest("_index", "_id1").upsert("{}", "{}"); updateRequest.upsertRequest().setPipeline("_id"); bulkRequest.add(updateRequest); + updateRequest.decRef(); IndexRequest indexRequest = new IndexRequest("_index").id("_id1").source(Map.of()).setPipeline("_id1"); bulkRequest.add(indexRequest); + indexRequest.decRef(); @SuppressWarnings("unchecked") BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") @@ -1258,6 +1266,7 @@ public void testExecuteSuccess() { ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); verify(failureHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); + indexRequest.decRef(); } public void testDynamicTemplates() throws Exception { @@ -1291,6 +1300,7 @@ public void testDynamicTemplates() throws Exception { ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); latch.await(); assertThat(indexRequest.getDynamicTemplates(), equalTo(Map.of("foo", "bar", "foo.bar", "baz"))); + indexRequest.decRef(); } public void testExecuteEmptyPipeline() throws Exception { @@ -1312,6 +1322,7 @@ public void testExecuteEmptyPipeline() throws Exception { ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); verify(failureHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); + indexRequest.decRef(); } public void testExecutePropagateAllMetadataUpdates() throws Exception { @@ -1374,6 +1385,7 @@ public void testExecutePropagateAllMetadataUpdates() throws Exception { assertThat(indexRequest.versionType(), equalTo(VersionType.fromString(versionType))); assertThat(indexRequest.ifSeqNo(), equalTo(ifSeqNo)); assertThat(indexRequest.ifPrimaryTerm(), equalTo(ifPrimaryTerm)); + indexRequest.decRef(); } public void testExecuteFailure() throws Exception { @@ -1416,6 +1428,7 @@ public void testExecuteFailure() throws Exception { verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); verify(failureHandler, times(1)).accept(eq(0), any(RuntimeException.class)); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); + indexRequest.decRef(); } public void testExecuteSuccessWithOnFailure() throws Exception { @@ -1464,6 +1477,7 @@ public void testExecuteSuccessWithOnFailure() throws Exception { ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); verify(failureHandler, never()).accept(eq(0), any(IngestProcessorException.class)); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); + indexRequest.decRef(); } public void testExecuteFailureWithNestedOnFailure() throws Exception { @@ -1508,6 +1522,7 @@ public void testExecuteFailureWithNestedOnFailure() throws Exception { verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); verify(failureHandler, times(1)).accept(eq(0), any(RuntimeException.class)); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); + indexRequest.decRef(); } public void testBulkRequestExecutionWithFailures() throws Exception { @@ -1517,20 +1532,23 @@ public void testBulkRequestExecutionWithFailures() throws Exception { int numRequest = scaledRandomIntBetween(8, 64); int numIndexRequests = 0; for (int i = 0; i < numRequest; i++) { - DocWriteRequest request; if (randomBoolean()) { if (randomBoolean()) { - request = new DeleteRequest("_index", "_id"); + DeleteRequest request = new DeleteRequest("_index", "_id"); + bulkRequest.add(request); } else { - request = new UpdateRequest("_index", "_id"); + UpdateRequest request = new UpdateRequest("_index", "_id"); + bulkRequest.add(request); + request.decRef(); } + } else { IndexRequest indexRequest = new IndexRequest("_index").id("_id").setPipeline(pipelineId).setFinalPipeline("_none"); indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); - request = indexRequest; numIndexRequests++; + bulkRequest.add(indexRequest); + indexRequest.decRef(); } - bulkRequest.add(request); } CompoundProcessor processor = mock(CompoundProcessor.class); @@ -1591,6 +1609,7 @@ public void testBulkRequestExecution() throws Exception { executedPipelinesExpected.add(shouldListExecutedPiplines); indexRequest.setListExecutedPipelines(shouldListExecutedPiplines); bulkRequest.add(indexRequest); + indexRequest.decRef(); } final Processor processor = mock(Processor.class); @@ -1732,6 +1751,7 @@ public String execute() { indexRequest.setPipeline("_id1").setFinalPipeline("_id2"); indexRequest.source(randomAlphaOfLength(10), randomAlphaOfLength(10)); ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, (integer, e) -> {}, (thread, e) -> {}, Names.WRITE); + indexRequest.decRef(); { final IngestStats ingestStats = ingestService.stats(); @@ -1876,6 +1896,7 @@ public void testStats() throws Exception { assertProcessorStats(0, afterForthRequestStats, "_id1", 1, 1, 0); // not carried forward since type changed assertProcessorStats(1, afterForthRequestStats, "_id1", 2, 0, 0); // carried forward and added from old stats assertProcessorStats(0, afterForthRequestStats, "_id2", 1, 0, 0); + indexRequest.decRef(); } public void testStatName() { @@ -1949,6 +1970,9 @@ public String getDescription() { .setFinalPipeline("_none"); bulkRequest.add(indexRequest2); + indexRequest1.decRef(); + indexRequest2.decRef(); + @SuppressWarnings("unchecked") final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") @@ -2045,6 +2069,7 @@ public void testCBORParsing() throws Exception { .setFinalPipeline("_none"); ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, (integer, e) -> {}, (thread, e) -> {}, Names.WRITE); + indexRequest.decRef(); } assertThat(reference.get(), is(instanceOf(byte[].class))); @@ -2115,6 +2140,14 @@ public void testSetsRawTimestamp() { bulkRequest.add(indexRequest6); bulkRequest.add(indexRequest7); bulkRequest.add(indexRequest8); + indexRequest1.decRef(); + indexRequest2.decRef(); + indexRequest3.decRef(); + indexRequest4.decRef(); + indexRequest5.decRef(); + indexRequest6.decRef(); + indexRequest7.decRef(); + indexRequest8.decRef(); ingestService.executeBulkRequest(8, bulkRequest.requests(), indexReq -> {}, (integer, e) -> {}, (thread, e) -> {}, Names.WRITE); assertThat(indexRequest1.getRawTimestamp(), nullValue()); @@ -2156,6 +2189,7 @@ public void testHasPipeline() { indexRequest.setPipeline("some-pipeline").setFinalPipeline("some-final-pipeline"); assertTrue(hasPipeline(indexRequest)); + indexRequest.decRef(); } public void testResolveRequiredOrDefaultPipelineDefaultPipeline() { @@ -2172,6 +2206,7 @@ public void testResolveRequiredOrDefaultPipelineDefaultPipeline() { assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("default-pipeline")); + indexRequest.decRef(); // alias name matches with IDM: indexRequest = new IndexRequest("alias"); @@ -2179,6 +2214,7 @@ public void testResolveRequiredOrDefaultPipelineDefaultPipeline() { assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("default-pipeline")); + indexRequest.decRef(); // index name matches with ITMD: IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder("name1") @@ -2190,6 +2226,7 @@ public void testResolveRequiredOrDefaultPipelineDefaultPipeline() { assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("default-pipeline")); + indexRequest.decRef(); } public void testResolveFinalPipeline() { @@ -2207,6 +2244,7 @@ public void testResolveFinalPipeline() { assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline")); + indexRequest.decRef(); // alias name matches with IDM: indexRequest = new IndexRequest("alias"); @@ -2215,6 +2253,7 @@ public void testResolveFinalPipeline() { assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline")); + indexRequest.decRef(); // index name matches with ITMD: IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder("name1") @@ -2227,6 +2266,7 @@ public void testResolveFinalPipeline() { assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline")); + indexRequest.decRef(); } public void testResolveFinalPipelineWithDateMathExpression() { @@ -2245,6 +2285,7 @@ public void testResolveFinalPipelineWithDateMathExpression() { assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline")); + indexRequest.decRef(); } public void testResolveRequestOrDefaultPipelineAndFinalPipeline() { @@ -2256,6 +2297,7 @@ public void testResolveRequestOrDefaultPipelineAndFinalPipeline() { assertFalse(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo(NOOP_PIPELINE_NAME)); + indexRequest.decRef(); } // request pipeline: @@ -2266,6 +2308,7 @@ public void testResolveRequestOrDefaultPipelineAndFinalPipeline() { assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("request-pipeline")); + indexRequest.decRef(); } // request pipeline with default pipeline: @@ -2280,6 +2323,7 @@ public void testResolveRequestOrDefaultPipelineAndFinalPipeline() { assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("request-pipeline")); + indexRequest.decRef(); } // request pipeline with final pipeline: @@ -2295,6 +2339,7 @@ public void testResolveRequestOrDefaultPipelineAndFinalPipeline() { assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("request-pipeline")); assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline")); + indexRequest.decRef(); } } @@ -2494,6 +2539,7 @@ public void testResolvePipelinesWithNonePipeline() { assertFalse(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo(NOOP_PIPELINE_NAME)); + indexRequest.decRef(); } // _none default pipeline: @@ -2508,6 +2554,7 @@ public void testResolvePipelinesWithNonePipeline() { assertFalse(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo(NOOP_PIPELINE_NAME)); + indexRequest.decRef(); } // _none default pipeline with request pipeline: @@ -2522,6 +2569,7 @@ public void testResolvePipelinesWithNonePipeline() { assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("pipeline1")); + indexRequest.decRef(); } // _none request pipeline with default pipeline: @@ -2536,6 +2584,7 @@ public void testResolvePipelinesWithNonePipeline() { assertFalse(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo(NOOP_PIPELINE_NAME)); + indexRequest.decRef(); } // _none request pipeline with final pipeline: @@ -2551,6 +2600,7 @@ public void testResolvePipelinesWithNonePipeline() { assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo(NOOP_PIPELINE_NAME)); assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline")); + indexRequest.decRef(); } // _none final pipeline: @@ -2566,6 +2616,7 @@ public void testResolvePipelinesWithNonePipeline() { assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("pipeline1")); assertThat(indexRequest.getFinalPipeline(), equalTo(NOOP_PIPELINE_NAME)); + indexRequest.decRef(); } } diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index adfc333e9dc7e..d953ba9c79f09 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -104,7 +105,9 @@ public void testRetrieveSnapshots() { int numDocs = randomIntBetween(10, 20); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); - prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId(id).setSource("text", "sometext"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } indicesAdmin().prepareFlush(indexName).get(); diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index bfb62e6fed197..6a066edfebec3 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.ClosePointInTimeRequest; import org.elasticsearch.action.search.OpenPointInTimeRequest; @@ -241,7 +242,12 @@ protected Settings nodeSettings() { public void testClearOnClose() { createIndex("index"); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } assertResponse( client().prepareSearch("index").setSize(1).setScroll("1m"), searchResponse -> assertThat(searchResponse.getScrollId(), is(notNullValue())) @@ -255,7 +261,12 @@ public void testClearOnClose() { public void testClearOnStop() { createIndex("index"); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } assertResponse( client().prepareSearch("index").setSize(1).setScroll("1m"), searchResponse -> assertThat(searchResponse.getScrollId(), is(notNullValue())) @@ -269,7 +280,12 @@ public void testClearOnStop() { public void testClearIndexDelete() { createIndex("index"); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } assertResponse( client().prepareSearch("index").setSize(1).setScroll("1m"), searchResponse -> assertThat(searchResponse.getScrollId(), is(notNullValue())) @@ -284,7 +300,12 @@ public void testClearIndexDelete() { public void testCloseSearchContextOnRewriteException() { // if refresh happens while checking the exception, the subsequent reference count might not match, so we switch it off createIndex("index", Settings.builder().put("index.refresh_interval", -1).build()); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } SearchService service = getInstanceFromNode(SearchService.class); IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -303,7 +324,12 @@ public void testCloseSearchContextOnRewriteException() { public void testSearchWhileIndexDeleted() throws InterruptedException { createIndex("index"); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } SearchService service = getInstanceFromNode(SearchService.class); IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -319,28 +345,39 @@ public void testSearchWhileIndexDeleted() throws InterruptedException { randomBoolean(), ShardRoutingState.INITIALIZING ); - final Thread thread = new Thread(() -> { - startGun.countDown(); - while (running.get()) { - if (randomBoolean()) { - service.afterIndexRemoved(indexService.index(), indexService.getIndexSettings(), DELETED); - } else { - service.beforeIndexShardCreated(routing, indexService.getIndexSettings().getSettings()); - } - if (randomBoolean()) { - // here we trigger some refreshes to ensure the IR go out of scope such that we hit ACE if we access a search - // context in a non-sane way. - try { - semaphore.acquire(); - } catch (InterruptedException e) { - throw new AssertionError(e); + final Thread thread = new Thread() { + @Override + public void run() { + startGun.countDown(); + while (running.get()) { + if (randomBoolean()) { + service.afterIndexRemoved(indexService.index(), indexService.getIndexSettings(), DELETED); + } else { + service.beforeIndexShardCreated(routing, indexService.getIndexSettings().getSettings()); + } + if (randomBoolean()) { + // here we trigger some refreshes to ensure the IR go out of scope such that we hit ACE if we access a search + // context in a non-sane way. + try { + semaphore.acquire(); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + IndexRequestBuilder indexRequestBuilder = prepareIndex("index"); + try { + indexRequestBuilder.setSource("field", "value") + .setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); + indexRequestBuilder.execute(ActionListener.runBefore(ActionListener.running(semaphore::release), () -> { + indexRequestBuilder.request().decRef(); + })); + } catch (Exception e) { + indexRequestBuilder.request().decRef(); + throw e; + } } - prepareIndex("index").setSource("field", "value") - .setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())) - .execute(ActionListener.running(semaphore::release)); } } - }); + }; thread.start(); startGun.await(); try { @@ -418,7 +455,12 @@ public void testSearchWhileIndexDeleted() throws InterruptedException { public void testSearchWhileIndexDeletedDoesNotLeakSearchContext() throws ExecutionException, InterruptedException { createIndex("index"); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index")); @@ -473,7 +515,12 @@ public void testSearchWhileIndexDeletedDoesNotLeakSearchContext() throws Executi public void testBeforeShardLockDuringShardCreate() { IndexService indexService = createIndex("index", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build()); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } assertResponse( client().prepareSearch("index").setSize(1).setScroll("1m"), searchResponse -> assertThat(searchResponse.getScrollId(), is(notNullValue())) @@ -573,7 +620,14 @@ public void testTimeout() throws IOException { public void testMaxDocvalueFieldsSearch() throws IOException { final Settings settings = Settings.builder().put(IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.getKey(), 1).build(); createIndex("index", settings, null, "field1", "keyword", "field2", "keyword"); - prepareIndex("index").setId("1").setSource("field1", "value1", "field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1") + .setSource("field1", "value1", "field2", "value2") + .setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } final SearchService service = getInstanceFromNode(SearchService.class); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -627,7 +681,14 @@ public void testMaxDocvalueFieldsSearch() throws IOException { public void testDeduplicateDocValuesFields() throws Exception { createIndex("index", Settings.EMPTY, "_doc", "field1", "type=date", "field2", "type=date"); - prepareIndex("index").setId("1").setSource("field1", "2022-08-03", "field2", "2022-08-04").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1") + .setSource("field1", "2022-08-03", "field2", "2022-08-04") + .setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } SearchService service = getInstanceFromNode(SearchService.class); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index")); @@ -773,7 +834,12 @@ public void testIgnoreScriptfieldIfSizeZero() throws IOException { */ public void testMaxOpenScrollContexts() throws Exception { createIndex("index"); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } final SearchService service = getInstanceFromNode(SearchService.class); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -1010,8 +1076,13 @@ public void testCanMatch() throws Exception { ).canMatch() ); // the source can match and can be rewritten to a match_none, but not the alias filter - final DocWriteResponse response = prepareIndex("index").setSource("id", "1").get(); - assertEquals(RestStatus.CREATED, response.status()); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setSource("id", "1"); + try { + final DocWriteResponse response = indexRequestBuilder.get(); + assertEquals(RestStatus.CREATED, response.status()); + } finally { + indexRequestBuilder.request().decRef(); + } searchRequest.indices("alias").source(new SearchSourceBuilder().query(new TermQueryBuilder("id", "1"))); assertFalse( service.canMatch( @@ -1105,7 +1176,14 @@ public void testSetSearchThrottled() throws IOException { final SearchService service = getInstanceFromNode(SearchService.class); Index index = resolveIndex("throttled_threadpool_index"); assertTrue(service.getIndicesService().indexServiceSafe(index).getIndexSettings().isSearchThrottled()); - prepareIndex("throttled_threadpool_index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("throttled_threadpool_index").setId("1") + .setSource("field", "value") + .setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } assertSearchHits( client().prepareSearch("throttled_threadpool_index") .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED) @@ -1194,7 +1272,14 @@ public void testExpandSearchThrottled() { ) ).actionGet(); - prepareIndex("throttled_threadpool_index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("throttled_threadpool_index").setId("1") + .setSource("field", "value") + .setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } assertHitCount(client().prepareSearch(), 1L); assertHitCount(client().prepareSearch().setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED), 1L); } @@ -1207,7 +1292,14 @@ public void testExpandSearchFrozen() { new InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.Request(indexName, "index.frozen", "true") ).actionGet(); - prepareIndex(indexName).setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId("1") + .setSource("field", "value") + .setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } assertHitCount(client().prepareSearch(), 0L); assertHitCount(client().prepareSearch().setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED), 1L); assertWarnings(TransportSearchAction.FROZEN_INDICES_DEPRECATION_MESSAGE.replace("{}", indexName)); @@ -1419,7 +1511,12 @@ public void testDeleteIndexWhileSearch() throws Exception { createIndex("test"); int numDocs = randomIntBetween(1, 20); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setSource("f", "v").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource("f", "v"); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } } indicesAdmin().prepareRefresh("test").get(); AtomicBoolean stopped = new AtomicBoolean(false); @@ -1696,8 +1793,13 @@ public void testWaitOnRefresh() throws ExecutionException, InterruptedException searchRequest.setWaitForCheckpointsTimeout(TimeValue.timeValueSeconds(30)); searchRequest.setWaitForCheckpoints(Collections.singletonMap("index", new long[] { 0 })); - final DocWriteResponse response = prepareIndex("index").setSource("id", "1").get(); - assertEquals(RestStatus.CREATED, response.status()); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setSource("id", "1"); + try { + final DocWriteResponse response = indexRequestBuilder.get(); + assertEquals(RestStatus.CREATED, response.status()); + } finally { + indexRequestBuilder.request().decRef(); + } SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()); ShardSearchRequest request = new ShardSearchRequest( @@ -1731,8 +1833,13 @@ public void testWaitOnRefreshFailsWithRefreshesDisabled() { searchRequest.setWaitForCheckpointsTimeout(TimeValue.timeValueSeconds(30)); searchRequest.setWaitForCheckpoints(Collections.singletonMap("index", new long[] { 0 })); - final DocWriteResponse response = prepareIndex("index").setSource("id", "1").get(); - assertEquals(RestStatus.CREATED, response.status()); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setSource("id", "1"); + try { + final DocWriteResponse response = indexRequestBuilder.get(); + assertEquals(RestStatus.CREATED, response.status()); + } finally { + indexRequestBuilder.request().decRef(); + } SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); @@ -1767,8 +1874,13 @@ public void testWaitOnRefreshFailsIfCheckpointNotIndexed() { searchRequest.setWaitForCheckpointsTimeout(TimeValue.timeValueMillis(randomIntBetween(10, 100))); searchRequest.setWaitForCheckpoints(Collections.singletonMap("index", new long[] { 1 })); - final DocWriteResponse response = prepareIndex("index").setSource("id", "1").get(); - assertEquals(RestStatus.CREATED, response.status()); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setSource("id", "1"); + try { + final DocWriteResponse response = indexRequestBuilder.get(); + assertEquals(RestStatus.CREATED, response.status()); + } finally { + indexRequestBuilder.request().decRef(); + } SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); @@ -1804,8 +1916,13 @@ public void testWaitOnRefreshTimeout() { searchRequest.setWaitForCheckpointsTimeout(TimeValue.timeValueMillis(randomIntBetween(10, 100))); searchRequest.setWaitForCheckpoints(Collections.singletonMap("index", new long[] { 0 })); - final DocWriteResponse response = prepareIndex("index").setSource("id", "1").get(); - assertEquals(RestStatus.CREATED, response.status()); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setSource("id", "1"); + try { + final DocWriteResponse response = indexRequestBuilder.get(); + assertEquals(RestStatus.CREATED, response.status()); + } finally { + indexRequestBuilder.request().decRef(); + } SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); @@ -1832,7 +1949,12 @@ public void testMinimalSearchSourceInShardRequests() { createIndex("test"); int numDocs = between(0, 10); for (int i = 0; i < numDocs; i++) { - prepareIndex("test").setSource("id", Integer.toString(i)).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setSource("id", Integer.toString(i)); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } } indicesAdmin().prepareRefresh("test").get(); @@ -1864,7 +1986,12 @@ public void testMinimalSearchSourceInShardRequests() { public void testDfsQueryPhaseRewrite() { createIndex("index"); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } final SearchService service = getInstanceFromNode(SearchService.class); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index")); @@ -1961,7 +2088,12 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { executor.setMaximumPoolSize(configuredMaxPoolSize); // We set this explicitly to be independent of CPU cores. int numDocs = randomIntBetween(50, 100); for (int i = 0; i < numDocs; i++) { - prepareIndex("index").setId(String.valueOf(i)).setSource("field", "value").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId(String.valueOf(i)).setSource("field", "value"); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } if (i % 5 == 0) { indicesAdmin().prepareRefresh("index").get(); } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java index f638dd8e7c2b7..777b085ec9673 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.TelemetryMetrics; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; @@ -56,7 +57,7 @@ protected int maximumNumberOfReplicas() { public void testSearchTransportMetricsDfsQueryThenFetch() throws InterruptedException { var indexName = "test1"; createIndex(indexName); - indexRandom(true, false, prepareIndex(indexName).setId("1").setSource("body", "foo")); + indexRandomAndDecRefRequests(true, false, prepareIndex(indexName).setId("1").setSource("body", "foo")); assertSearchHitsWithoutFailures( prepareSearch(indexName).setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(simpleQueryStringQuery("foo")), @@ -72,7 +73,7 @@ public void testSearchTransportMetricsDfsQueryThenFetch() throws InterruptedExce public void testSearchTransportMetricsQueryThenFetch() throws InterruptedException { var indexName = "test2"; createIndex(indexName); - indexRandom(true, false, prepareIndex(indexName).setId("1").setSource("body", "foo")); + indexRandomAndDecRefRequests(true, false, prepareIndex(indexName).setId("1").setSource("body", "foo")); assertSearchHitsWithoutFailures( prepareSearch(indexName).setSearchType(SearchType.QUERY_THEN_FETCH).setQuery(simpleQueryStringQuery("foo")), @@ -87,7 +88,7 @@ public void testSearchTransportMetricsQueryThenFetch() throws InterruptedExcepti public void testSearchTransportMetricsScroll() throws InterruptedException { var indexName = "test3"; createIndex(indexName); - indexRandom( + indexRandomAndDecRefRequests( true, false, prepareIndex(indexName).setId("1").setSource("body", "foo"), @@ -135,4 +136,15 @@ private long getNumberOfMeasurements(String attributeValue) { ) .count(); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, boolean dummyDocuments, IndexRequestBuilder... builders) + throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java index cfee97891aa32..3a68ee0b6cdf9 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java @@ -77,6 +77,9 @@ protected void indexData() throws Exception { // total docs in shard "2" = 12 indexRandom(true, docs); + for (IndexRequestBuilder indexRequestBuilder : docs) { + indexRequestBuilder.request().decRef(); + } assertNoFailuresAndResponse(prepareSearch("idx").setRouting(routing1).setQuery(matchAllQuery()), resp -> { long totalOnOne = resp.getHits().getTotalHits().value; diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java index 779e0ad28433a..b14e37c9325b9 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.geo; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoJson; import org.elasticsearch.common.settings.Settings; @@ -73,10 +74,11 @@ public void testFieldAlias() throws IOException { ensureGreen(); Point point = GeometryTestUtils.randomPoint(false); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(point)).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCount(client().prepareSearch(defaultIndexName).setQuery(geoShapeQuery("alias", point)), 1); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 31df5fc9d9be8..6f4a0964c8b8f 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -303,18 +303,19 @@ public void testSuccessfulSnapshotAndRestore() { if (documents == 0) { afterIndexing.run(); } else { - try (BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { - for (int i = 0; i < documents; ++i) { - bulkRequest.add(new IndexRequest(index).source(Collections.singletonMap("foo", "bar" + i))); - } - final SubscribableListener bulkResponseStepListener = new SubscribableListener<>(); - client().bulk(bulkRequest, bulkResponseStepListener); - continueOrDie(bulkResponseStepListener, bulkResponse -> { - assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); - assertEquals(documents, bulkResponse.getItems().length); - afterIndexing.run(); - }); + BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < documents; ++i) { + IndexRequest indexRequest = new IndexRequest(index).source(Collections.singletonMap("foo", "bar" + i)); + bulkRequest.add(indexRequest); + indexRequest.decRef(); } + final SubscribableListener bulkResponseStepListener = new SubscribableListener<>(); + client().bulk(bulkRequest, ActionListener.releaseAfter(bulkResponseStepListener, bulkRequest)); + continueOrDie(bulkResponseStepListener, bulkResponse -> { + assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); + assertEquals(documents, bulkResponse.getItems().length); + afterIndexing.run(); + }); } }); @@ -800,18 +801,19 @@ private void indexNDocuments(int documents, String index, Runnable afterIndexing afterIndexing.run(); return; } - try (BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { - for (int i = 0; i < documents; ++i) { - bulkRequest.add(new IndexRequest(index).source(Collections.singletonMap("foo", "bar" + i))); - } - final SubscribableListener bulkResponseStepListener = new SubscribableListener<>(); - client().bulk(bulkRequest, bulkResponseStepListener); - continueOrDie(bulkResponseStepListener, bulkResponse -> { - assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); - assertEquals(documents, bulkResponse.getItems().length); - afterIndexing.run(); - }); + BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < documents; ++i) { + IndexRequest indexRequest = new IndexRequest(index).source(Collections.singletonMap("foo", "bar" + i)); + bulkRequest.add(indexRequest); + indexRequest.decRef(); } + final SubscribableListener bulkResponseStepListener = new SubscribableListener<>(); + client().bulk(bulkRequest, ActionListener.releaseAfter(bulkResponseStepListener, bulkRequest)); + continueOrDie(bulkResponseStepListener, bulkResponse -> { + assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); + assertEquals(documents, bulkResponse.getItems().length); + afterIndexing.run(); + }); } public void testConcurrentSnapshotDeleteAndDeleteIndex() throws IOException { @@ -1085,11 +1087,10 @@ public void testSuccessfulSnapshotWithConcurrentDynamicMappingUpdates() { final AtomicBoolean initiatedSnapshot = new AtomicBoolean(false); for (int i = 0; i < documents; ++i) { // Index a few documents with different field names so we trigger a dynamic mapping update for each of them - try ( - BulkRequest bulkRequest = new BulkRequest().add(new IndexRequest(index).source(Map.of("foo" + i, "bar"))) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - ) { - client().bulk(bulkRequest, assertNoFailureListener(bulkResponse -> { + IndexRequest indexRequest = new IndexRequest(index).source(Map.of("foo" + i, "bar")); + try { + BulkRequest bulkRequest = new BulkRequest().add(indexRequest).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client().bulk(bulkRequest, ActionListener.releaseAfter(assertNoFailureListener(bulkResponse -> { assertFalse("Failures in bulkresponse: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); if (initiatedSnapshot.compareAndSet(false, true)) { client().admin() @@ -1098,7 +1099,9 @@ public void testSuccessfulSnapshotWithConcurrentDynamicMappingUpdates() { .setWaitForCompletion(true) .execute(createSnapshotResponseStepListener); } - })); + }), bulkRequest)); + } finally { + indexRequest.decRef(); } } }); @@ -1193,18 +1196,19 @@ public void testRunConcurrentSnapshots() { .execute(snapshotListener) ); } - try (BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { - for (int i = 0; i < documents; ++i) { - bulkRequest.add(new IndexRequest(index).source(Collections.singletonMap("foo", "bar" + i))); - } - final SubscribableListener bulkResponseStepListener = new SubscribableListener<>(); - client().bulk(bulkRequest, bulkResponseStepListener); - continueOrDie(bulkResponseStepListener, bulkResponse -> { - assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); - assertEquals(documents, bulkResponse.getItems().length); - doneIndexing.set(true); - }); + BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < documents; ++i) { + IndexRequest indexRequest = new IndexRequest(index).source(Collections.singletonMap("foo", "bar" + i)); + bulkRequest.add(indexRequest); + indexRequest.decRef(); } + final SubscribableListener bulkResponseStepListener = new SubscribableListener<>(); + client().bulk(bulkRequest, ActionListener.releaseAfter(bulkResponseStepListener, bulkRequest)); + continueOrDie(bulkResponseStepListener, bulkResponse -> { + assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); + assertEquals(documents, bulkResponse.getItems().length); + doneIndexing.set(true); + }); }); final AtomicBoolean doneSnapshotting = new AtomicBoolean(false); diff --git a/server/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/server/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java index 0b84f14c56ecb..07b7e5cef71e0 100644 --- a/server/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java +++ b/server/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java @@ -88,5 +88,8 @@ public static void index01Docs(String type, String settings, ESIntegTestCase tes indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME).setId("6").setSource(TEXT_FIELD, gb, CLASS_FIELD, "0")); indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME).setId("7").setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); testCase.indexRandom(true, false, indexRequestBuilderList); + for (IndexRequestBuilder builder : indexRequestBuilderList) { + builder.request().decRef(); + } } } diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index bc5b15a4eba52..e8a99ffe856e4 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -421,8 +421,8 @@ private Response fetchMvLongs() throws IOException { private void initManyLongs() throws IOException { logger.info("loading many documents with longs"); - StringBuilder bulk = new StringBuilder(); for (int a = 0; a < 10; a++) { + StringBuilder bulk = new StringBuilder(); for (int b = 0; b < 10; b++) { for (int c = 0; c < 10; c++) { for (int d = 0; d < 10; d++) { @@ -438,7 +438,6 @@ private void initManyLongs() throws IOException { bulk("manylongs", bulk.toString()); bulk.setLength(0); } - initIndex("manylongs", bulk.toString()); } private void initSingleDocIndex() throws IOException { diff --git a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java index 6d49971df4f0d..c5c76dd1be279 100644 --- a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java +++ b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java @@ -112,7 +112,7 @@ public void testRetrieveSnapshots() throws Exception { int numDocs = randomIntBetween(10, 20); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); - prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); + indexDoc(indexName, id, "text", "sometext"); } indicesAdmin().prepareFlush(indexName).get(); diff --git a/test/external-modules/seek-tracking-directory/src/internalClusterTest/java/org/elasticsearch/test/seektracker/SeekTrackerPluginIT.java b/test/external-modules/seek-tracking-directory/src/internalClusterTest/java/org/elasticsearch/test/seektracker/SeekTrackerPluginIT.java index 7d1e4c4c3d0de..dc1034ecbea66 100644 --- a/test/external-modules/seek-tracking-directory/src/internalClusterTest/java/org/elasticsearch/test/seektracker/SeekTrackerPluginIT.java +++ b/test/external-modules/seek-tracking-directory/src/internalClusterTest/java/org/elasticsearch/test/seektracker/SeekTrackerPluginIT.java @@ -41,9 +41,13 @@ public void testSeekTrackerPlugin() throws InterruptedException { assertAcked(indicesAdmin().prepareCreate("index")); List docs = new ArrayList<>(); for (int i = 0; i < 100; i++) { - docs.add(prepareIndex("index").setSource("field", "term" + i % 5)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setSource("field", "term" + i % 5); + docs.add(indexRequestBuilder); } indexRandom(true, docs); + for (IndexRequestBuilder indexRequestBuilder : docs) { + indexRequestBuilder.request().decRef(); + } prepareSearch("index").setQuery(QueryBuilders.termQuery("field", "term2")).get().decRef(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index eea97d5bdc77f..f552739314763 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -231,11 +231,15 @@ public int indexDocs(final int numOfDoc) throws Exception { for (int doc = 0; doc < numOfDoc; doc++) { final IndexRequest indexRequest = new IndexRequest(index.getName()).id(Integer.toString(docId.incrementAndGet())) .source("{}", XContentType.JSON); - final BulkItemResponse response = index(indexRequest); - if (response.isFailed()) { - throw response.getFailure().getCause(); - } else { - assertEquals(DocWriteResponse.Result.CREATED, response.getResponse().getResult()); + try { + final BulkItemResponse response = index(indexRequest); + if (response.isFailed()) { + throw response.getFailure().getCause(); + } else { + assertEquals(DocWriteResponse.Result.CREATED, response.getResponse().getResult()); + } + } finally { + indexRequest.decRef(); } } return numOfDoc; @@ -244,11 +248,15 @@ public int indexDocs(final int numOfDoc) throws Exception { public int appendDocs(final int numOfDoc) throws Exception { for (int doc = 0; doc < numOfDoc; doc++) { final IndexRequest indexRequest = new IndexRequest(index.getName()).source("{}", XContentType.JSON); - final BulkItemResponse response = index(indexRequest); - if (response.isFailed()) { - throw response.getFailure().getCause(); - } else if (response.isFailed() == false) { - assertEquals(DocWriteResponse.Result.CREATED, response.getResponse().getResult()); + try { + final BulkItemResponse response = index(indexRequest); + if (response.isFailed()) { + throw response.getFailure().getCause(); + } else if (response.isFailed() == false) { + assertEquals(DocWriteResponse.Result.CREATED, response.getResponse().getResult()); + } + } finally { + indexRequest.decRef(); } } return numOfDoc; @@ -273,6 +281,8 @@ private BulkItemResponse executeWriteRequest(DocWriteRequest writeRequest, Wr try (BulkShardRequest request = new BulkShardRequest(shardId, refreshPolicy, items)) { new WriteReplicationAction(request, wrapBulkListener, this).execute(); return listener.get(); + } finally { + items[0].decRef(); } } @@ -917,16 +927,18 @@ Request extends ReplicatedWriteRequest & DocWriteRequest> Bulk IndexShard primary, Request request ) throws Exception { - try ( + BulkItemRequest bulkItemRequest = new BulkItemRequest(0, request); + try { BulkShardRequest bulkShardRequest = new BulkShardRequest( shardId, request.getRefreshPolicy(), - new BulkItemRequest[] { new BulkItemRequest(0, request) } - ) - ) { + new BulkItemRequest[] { bulkItemRequest } + ); final PlainActionFuture res = new PlainActionFuture<>(); executeShardBulkOnPrimary(primary, bulkShardRequest, res.map(TransportReplicationAction.PrimaryResult::replicaRequest)); return res.get(); + } finally { + bulkItemRequest.decRef(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java index a5ace3e357f90..9bafad3e1b54e 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java @@ -125,6 +125,9 @@ protected void checkTransientErrorsDuringRecoveryAreRetried(String recoveryActio requests.add(prepareIndex(indexName).setSource("{}", XContentType.JSON)); } indexRandom(true, requests); + for (IndexRequestBuilder indexRequestBuilder : requests) { + indexRequestBuilder.request().decRef(); + } flush(indexName); requests.clear(); @@ -132,6 +135,9 @@ protected void checkTransientErrorsDuringRecoveryAreRetried(String recoveryActio requests.add(prepareIndex(indexName).setSource("{}", XContentType.JSON)); } indexRandom(true, requests); + for (IndexRequestBuilder indexRequestBuilder : requests) { + indexRequestBuilder.request().decRef(); + } ensureSearchable(indexName); ClusterStateResponse stateResponse = clusterAdmin().prepareState().get(); @@ -228,6 +234,9 @@ public void checkDisconnectsWhileRecovering(String recoveryActionToBlock) throws requests.add(prepareIndex(indexName).setSource("{}", XContentType.JSON)); } indexRandom(true, requests); + for (IndexRequestBuilder indexRequestBuilder : requests) { + indexRequestBuilder.request().decRef(); + } ensureSearchable(indexName); ClusterStateResponse stateResponse = clusterAdmin().prepareState().get(); @@ -329,6 +338,9 @@ public void checkDisconnectsDuringRecovery(boolean useSnapshotBasedRecoveries) t requests.add(prepareIndex(indexName).setSource("{}", XContentType.JSON)); } indexRandom(true, requests); + for (IndexRequestBuilder indexRequestBuilder : requests) { + indexRequestBuilder.request().decRef(); + } ensureSearchable(indexName); assertHitCount(prepareSearch(indexName), numDocs); diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java index 3d4dea430a9b5..156e963fcedb6 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java @@ -8,8 +8,10 @@ package org.elasticsearch.repositories; import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobPath; @@ -94,9 +96,9 @@ public void testCreateSnapshot() { logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { - prepareIndex("test-idx-1").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); - prepareIndex("test-idx-2").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); - prepareIndex("test-idx-3").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); + indexDoc("test-idx-1", Integer.toString(i), "foo", "bar" + i); + indexDoc("test-idx-2", Integer.toString(i), "foo", "bar" + i); + indexDoc("test-idx-3", Integer.toString(i), "foo", "bar" + i); } client().admin().indices().prepareRefresh().get(); @@ -167,9 +169,9 @@ public void testCleanup() throws Exception { logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { - prepareIndex("test-idx-1").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); - prepareIndex("test-idx-2").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); - prepareIndex("test-idx-3").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); + indexDoc("test-idx-1", Integer.toString(i), "foo", "bar" + i); + indexDoc("test-idx-2", Integer.toString(i), "foo", "bar" + i); + indexDoc("test-idx-3", Integer.toString(i), "foo", "bar" + i); } client().admin().indices().prepareRefresh().get(); @@ -220,7 +222,11 @@ public void testIndexLatest() throws Exception { createIndex("test-idx-1"); for (int i = 0; i < 100; i++) { - client().prepareIndex("test-idx-1").setId(Integer.toString(i)).setSource("foo", "bar" + i).get(); + IndexRequestBuilder indexRequestBuilder = client().prepareIndex("test-idx-1") + .setId(Integer.toString(i)) + .setSource("foo", "bar" + i); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } final var repository = getRepository(); @@ -314,4 +320,13 @@ private Set listChildren(BlobPath path) { protected BlobStoreRepository getRepository() { return (BlobStoreRepository) getInstanceFromNode(RepositoriesService.class).repository(TEST_REPO_NAME); } + + protected final DocWriteResponse indexDoc(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java index b1765218ff7f2..7a9ab4b47aeca 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java @@ -627,6 +627,9 @@ protected void addRandomDocuments(String name, int numDocs) throws InterruptedEx .setSource("field", "value"); } indexRandom(true, indexRequestBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } } private String[] generateRandomNames(int num) { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java index 7fc1826952477..deb8cf39c8dd5 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -219,6 +219,9 @@ public void setupSuiteScopeCluster() throws Exception { assertAcked(prepareCreate(IDX_ZERO_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=" + fieldTypeName())); indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); // Added to debug a test failure where the terms aggregation seems to be reporting two documents with the same diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java index a41c60cba4a4a..8c12f35aa0fb6 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java @@ -46,6 +46,9 @@ public void setupSuiteScopeCluster() throws Exception { maxValue = numDocs; maxValues = numDocs + 2; indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } // creating an index to test the empty buckets functionality. The way it works is by indexing // two docs {value: 0} and {value : 2}, then building a histogram agg with interval 1 and with empty @@ -60,6 +63,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } ensureSearchable(); } diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/BasePointShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/BasePointShapeQueryTestCase.java index 52d2f3f53a43e..e73f52e827a14 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/geo/BasePointShapeQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/BasePointShapeQueryTestCase.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoJson; @@ -85,10 +86,11 @@ public void testNullShape() throws Exception { createMapping(defaultIndexName, defaultFieldName); ensureGreen(); - prepareIndex(defaultIndexName).setId("aNullshape") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("aNullshape") .setSource("{\"geo\": null}", XContentType.JSON) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); GetResponse result = client().prepareGet(defaultIndexName, "aNullshape").get(); assertThat(result.getField("location"), nullValue()); }; @@ -97,15 +99,17 @@ public void testIndexPointsFilterRectangle() throws Exception { createMapping(defaultIndexName, defaultFieldName); ensureGreen(); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(jsonBuilder().startObject().field("name", "Document 1").field(defaultFieldName, "POINT(-30 -30)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex(defaultIndexName).setId("2") + indexRequestBuilder = prepareIndex(defaultIndexName).setId("2") .setSource(jsonBuilder().startObject().field("name", "Document 2").field(defaultFieldName, "POINT(-45 -50)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Geometry geometry = new Rectangle(-45, 45, 45, -45); assertNoFailuresAndResponse( @@ -132,15 +136,17 @@ public void testIndexPointsCircle() throws Exception { createMapping(defaultIndexName, defaultFieldName); ensureGreen(); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(jsonBuilder().startObject().field("name", "Document 1").field(defaultFieldName, "POINT(-30 -30)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex(defaultIndexName).setId("2") + indexRequestBuilder = prepareIndex(defaultIndexName).setId("2") .setSource(jsonBuilder().startObject().field("name", "Document 2").field(defaultFieldName, "POINT(-45 -50)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Geometry geometry = new Circle(-30, -30, 100); @@ -161,15 +167,17 @@ public void testIndexPointsPolygon() throws Exception { createMapping(defaultIndexName, defaultFieldName); ensureGreen(); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-30 -30)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex(defaultIndexName).setId("2") + indexRequestBuilder = prepareIndex(defaultIndexName).setId("2") .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-45 -50)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Polygon polygon = new Polygon(new LinearRing(new double[] { -35, -35, -25, -25, -35 }, new double[] { -35, -25, -25, -35, -35 })); @@ -188,20 +196,23 @@ public void testIndexPointsMultiPolygon() throws Exception { createMapping(defaultIndexName, defaultFieldName); ensureGreen(); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(jsonBuilder().startObject().field("name", "Document 1").field(defaultFieldName, "POINT(-30 -30)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex(defaultIndexName).setId("2") + indexRequestBuilder = prepareIndex(defaultIndexName).setId("2") .setSource(jsonBuilder().startObject().field("name", "Document 2").field(defaultFieldName, "POINT(-40 -40)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex(defaultIndexName).setId("3") + indexRequestBuilder = prepareIndex(defaultIndexName).setId("3") .setSource(jsonBuilder().startObject().field("name", "Document 3").field(defaultFieldName, "POINT(-50 -50)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Polygon encloseDocument1Cb = new Polygon( new LinearRing(new double[] { -35, -35, -25, -25, -35 }, new double[] { -35, -25, -25, -35, -35 }) @@ -256,15 +267,17 @@ public void testIndexPointsRectangle() throws Exception { createMapping(defaultIndexName, defaultFieldName); ensureGreen(); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(jsonBuilder().startObject().field("name", "Document 1").field(defaultFieldName, "POINT(-30 -30)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex(defaultIndexName).setId("2") + indexRequestBuilder = prepareIndex(defaultIndexName).setId("2") .setSource(jsonBuilder().startObject().field("name", "Document 2").field(defaultFieldName, "POINT(-45 -50)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Rectangle rectangle = new Rectangle(-50, -40, -45, -55); @@ -283,15 +296,17 @@ public void testIndexPointsIndexedRectangle() throws Exception { createMapping(defaultIndexName, defaultFieldName); ensureGreen(); - prepareIndex(defaultIndexName).setId("point1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("point1") .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-30 -30)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex(defaultIndexName).setId("point2") + indexRequestBuilder = prepareIndex(defaultIndexName).setId("point2") .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-45 -50)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); String indexedShapeIndex = "indexed_query_shapes"; String indexedShapePath = "shape"; @@ -308,15 +323,17 @@ public void testIndexPointsIndexedRectangle() throws Exception { client().admin().indices().prepareCreate(indexedShapeIndex).setMapping(queryShapesMapping).get(); ensureGreen(); - prepareIndex(indexedShapeIndex).setId("shape1") + indexRequestBuilder = prepareIndex(indexedShapeIndex).setId("shape1") .setSource(jsonBuilder().startObject().field(indexedShapePath, "BBOX(-50, -40, -45, -55)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); - prepareIndex(indexedShapeIndex).setId("shape2") + indexRequestBuilder = prepareIndex(indexedShapeIndex).setId("shape2") .setSource(jsonBuilder().startObject().field(indexedShapePath, "BBOX(-60, -50, -50, -60)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNoFailuresAndResponse( client().prepareSearch(defaultIndexName) @@ -403,10 +420,11 @@ public void testQueryPoint() throws Exception { createMapping(defaultIndexName, defaultFieldName); ensureGreen(); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-35 -25)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Point point = new Point(-35, -25); @@ -438,10 +456,11 @@ public void testQueryMultiPoint() throws Exception { createMapping(defaultIndexName, defaultFieldName); ensureGreen(); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-35 -25)").endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); MultiPoint multiPoint = new MultiPoint(List.of(new Point(-35, -25), new Point(-15, -5))); @@ -480,7 +499,9 @@ public void testQueryPointFromGeoJSON() throws Exception { "type": "Point" } }"""; - client().index(new IndexRequest(defaultIndexName).id("1").source(doc1, XContentType.JSON).setRefreshPolicy(IMMEDIATE)).actionGet(); + IndexRequest indexRequest = new IndexRequest(defaultIndexName).id("1").source(doc1, XContentType.JSON).setRefreshPolicy(IMMEDIATE); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); Point point = new Point(-35, -25); @@ -532,10 +553,11 @@ public void testQueryPointFromMultiPoint() throws Exception { Point pointC = new Point(35, 25); Point pointD = new Point(45, 35); Object[] points = samplePointDataMultiFormat(pointA, pointB, pointC, pointD); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(jsonBuilder().startObject().field(defaultFieldName, points).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Point pointInvalid = new Point(-35, -35); for (Point point : new Point[] { pointA, pointB, pointC, pointD, pointInvalid }) { @@ -582,9 +604,11 @@ public void testIndexPointsFromLine() throws Exception { ); for (int i = 0; i < line.length(); i++) { Point point = new Point(line.getLon(i), line.getLat(i)); - prepareIndex(defaultIndexName).setSource( + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setSource( jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(point)).endObject() - ).get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } client().admin().indices().prepareRefresh(defaultIndexName).get(); // all points from a line intersect with the line @@ -607,9 +631,11 @@ public void testIndexPointsFromPolygon() throws Exception { LinearRing linearRing = polygon.getPolygon(); for (int i = 0; i < linearRing.length(); i++) { Point point = new Point(linearRing.getLon(i), linearRing.getLat(i)); - prepareIndex(defaultIndexName).setSource( + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setSource( jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(point)).endObject() - ).get(); + ); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } client().admin().indices().prepareRefresh(defaultIndexName).get(); // all points from a polygon intersect with the polygon diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeIntegTestCase.java index cd1914a579ecd..390e33dd741f7 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeIntegTestCase.java @@ -8,10 +8,12 @@ package org.elasticsearch.search.geo; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -185,7 +187,9 @@ public void testIgnoreMalformed() throws Exception { .endObject() ); - indexRandom(true, client().prepareIndex("test").setId("0").setSource("shape", polygonGeoJson)); + IndexRequestBuilder indexRequestBuilder = client().prepareIndex("test").setId("0").setSource("shape", polygonGeoJson); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertHitCount(client().prepareSearch("test").setQuery(matchAllQuery()), 1L); } @@ -216,7 +220,12 @@ public void testIndexShapeRouting() throws Exception { } }"""; - indexRandom(true, client().prepareIndex("test").setId("0").setSource(source, XContentType.JSON).setRouting("ABC")); + IndexRequestBuilder indexRequestBuilder = client().prepareIndex("test") + .setId("0") + .setSource(source, XContentType.JSON) + .setRouting("ABC"); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); assertHitCount( client().prepareSearch("test") .setQuery(queryBuilder().shapeQuery("shape", "0").indexedShapeIndex("test").indexedShapeRouting("ABC")), @@ -241,7 +250,9 @@ public void testDisallowExpensiveQueries() throws InterruptedException, IOExcept } }"""; - indexRandom(true, prepareIndex("test").setId("0").setSource(source, XContentType.JSON)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("0").setSource(source, XContentType.JSON); + indexRandom(true, indexRequestBuilder); + indexRequestBuilder.request().decRef(); refresh(); try { @@ -300,7 +311,7 @@ public void testShapeRelations() throws Exception { jsonBuilder().startObject().field("area", WellKnownText.toWKT(new MultiPolygon(polygons))).endObject() ); - prepareIndex("shapes").setId("1").setSource(data, XContentType.JSON).get(); + index("shapes", "1", data, XContentType.JSON); client().admin().indices().prepareRefresh().get(); // Point in polygon @@ -364,7 +375,7 @@ public void testShapeRelations() throws Exception { ); data = BytesReference.bytes(jsonBuilder().startObject().field("area", WellKnownText.toWKT(inverse)).endObject()); - prepareIndex("shapes").setId("2").setSource(data, XContentType.JSON).get(); + index("shapes", "2", data, XContentType.JSON); client().admin().indices().prepareRefresh().get(); // re-check point on polygon hole @@ -385,7 +396,7 @@ public void testShapeRelations() throws Exception { Polygon crossing = new Polygon(new LinearRing(new double[] { 170, 190, 190, 170, 170 }, new double[] { -10, -10, 10, 10, -10 })); data = BytesReference.bytes(jsonBuilder().startObject().field("area", WellKnownText.toWKT(crossing)).endObject()); - prepareIndex("shapes").setId("1").setSource(data, XContentType.JSON).get(); + index("shapes", "1", data, XContentType.JSON); client().admin().indices().prepareRefresh().get(); // Create a polygon crossing longitude 180 with hole. @@ -395,7 +406,7 @@ public void testShapeRelations() throws Exception { ); data = BytesReference.bytes(jsonBuilder().startObject().field("area", WellKnownText.toWKT(crossing)).endObject()); - prepareIndex("shapes").setId("1").setSource(data, XContentType.JSON).get(); + index("shapes", "1", data, XContentType.JSON); client().admin().indices().prepareRefresh().get(); assertHitCount( @@ -487,4 +498,13 @@ private byte[] unZipData(String path) throws IOException { protected byte[] convertTestData(ByteArrayOutputStream out) { return out.toByteArray(); } + + protected final DocWriteResponse index(String index, String id, BytesReference data, XContentType contentType) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(data, contentType).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java index 406625b33813f..09909feb095d3 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.geo; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoJson; import org.elasticsearch.common.geo.ShapeRelation; @@ -87,10 +88,11 @@ public void testFieldAlias() throws IOException { ensureGreen(); MultiPoint multiPoint = GeometryTestUtils.randomMultiPoint(false); - prepareIndex(defaultIndexName).setId("1") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource(GeoJson.toXContent(multiPoint, jsonBuilder().startObject().field(defaultFieldName), null).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCount(client().prepareSearch(defaultIndexName).setQuery(queryBuilder().shapeQuery("alias", multiPoint)), 1L); } @@ -102,10 +104,12 @@ public void testShapeFetchingPath() throws Exception { String geo = """ "geo" : {"type":"polygon", "coordinates":[[[-10,-10],[10,-10],[10,10],[-10,10],[-10,-10]]]}"""; - prepareIndex("shapes").setId("1").setSource(Strings.format(""" + IndexRequestBuilder indexRequestBuilder = prepareIndex("shapes").setId("1").setSource(Strings.format(""" { %s, "1" : { %s, "2" : { %s, "3" : { %s } }} } - """, geo, geo, geo, geo), XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(defaultIndexName).setId("1") + """, geo, geo, geo, geo), XContentType.JSON).setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource( jsonBuilder().startObject() .startObject(defaultFieldName) @@ -137,8 +141,9 @@ public void testShapeFetchingPath() throws Exception { .endObject() .endObject() ) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); QueryBuilder filter = queryBuilder().shapeQuery(defaultFieldName, "1") .relation(ShapeRelation.INTERSECTS) @@ -191,7 +196,11 @@ public void testRandomGeoCollectionQuery() throws Exception { ensureGreen(); XContentBuilder docSource = GeoJson.toXContent(gcb, jsonBuilder().startObject().field(defaultFieldName), null).endObject(); - prepareIndex(defaultIndexName).setId("1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") + .setSource(docSource) + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); // Create a random geometry collection to query GeometryCollection randomQueryCollection = makeRandomGeometryCollection(); @@ -215,12 +224,11 @@ public void testGeometryCollectionRelations() throws Exception { ensureGreen(); Rectangle envelope = new Rectangle(-10, 10, 10, -10); - - client().index( - new IndexRequest(defaultIndexName).source( - jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(envelope)).endObject() - ).setRefreshPolicy(IMMEDIATE) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest(defaultIndexName).source( + jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(envelope)).endObject() + ).setRefreshPolicy(IMMEDIATE); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); { // A geometry collection that is fully within the indexed shape @@ -312,7 +320,7 @@ public void testEdgeCases() throws Exception { client().admin().indices().prepareCreate(defaultIndexName).setMapping(mapping).get(); ensureGreen(); - prepareIndex(defaultIndexName).setId("blakely") + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("blakely") .setSource( jsonBuilder().startObject() .field("name", "Blakely Island") @@ -341,8 +349,9 @@ public void testEdgeCases() throws Exception { .endObject() .endObject() ) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Rectangle query = new Rectangle(-122.88, -122.82, 48.62, 48.54); @@ -366,10 +375,11 @@ public void testIndexedShapeReferenceSourceDisabled() throws Exception { Rectangle shape = new Rectangle(-45, 45, 45, -45); - prepareIndex("shapes").setId("Big_Rectangle") + IndexRequestBuilder indexRequestBuilder = prepareIndex("shapes").setId("Big_Rectangle") .setSource(jsonBuilder().startObject().field("shape", WellKnownText.toWKT(shape)).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, @@ -392,7 +402,11 @@ public void testPointQuery() throws Exception { XContentBuilder docSource = GeoJson.toXContent(gcb, jsonBuilder().startObject().field(defaultFieldName), ToXContent.EMPTY_PARAMS) .endObject(); - prepareIndex(defaultIndexName).setId("1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") + .setSource(docSource) + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCountAndNoFailures( client().prepareSearch(defaultIndexName).setQuery(queryBuilder().intersectionQuery(defaultFieldName, point)), @@ -406,7 +420,11 @@ public void testContainsShapeQuery() throws Exception { createMapping(defaultIndexName, defaultFieldName); XContentBuilder docSource = GeoJson.toXContent(polygon, jsonBuilder().startObject().field(defaultFieldName), null).endObject(); - prepareIndex(defaultIndexName).setId("1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") + .setSource(docSource) + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); QueryBuilder filter = queryBuilder().shapeQuery(defaultFieldName, innerPolygon).relation(ShapeRelation.CONTAINS); assertHitCountAndNoFailures(client().prepareSearch(defaultIndexName).setQuery(filter), 1L); @@ -420,7 +438,11 @@ public void testExistsQuery() throws Exception { createMapping(defaultIndexName, defaultFieldName); XContentBuilder docSource = GeoJson.toXContent(gcb, jsonBuilder().startObject().field(defaultFieldName), null).endObject(); - prepareIndex(defaultIndexName).setId("1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") + .setSource(docSource) + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); ExistsQueryBuilder eqb = existsQuery(defaultFieldName); assertHitCountAndNoFailures(client().prepareSearch(defaultIndexName).setQuery(eqb), 1L); @@ -433,11 +455,12 @@ public void testIndexedShapeReference() throws Exception { Rectangle shape = new Rectangle(-45, 45, 45, -45); - prepareIndex("shapes").setId("Big_Rectangle") + IndexRequestBuilder indexRequestBuilder = prepareIndex("shapes").setId("Big_Rectangle") .setSource(GeoJson.toXContent(shape, jsonBuilder().startObject().field("shape"), null).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); - prepareIndex(defaultIndexName).setId("1") + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") .setSource( jsonBuilder().startObject() .field("name", "Document 1") @@ -450,8 +473,9 @@ public void testIndexedShapeReference() throws Exception { .endObject() .endObject() ) - .setRefreshPolicy(IMMEDIATE) - .get(); + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertNoFailuresAndResponse( client().prepareSearch(defaultIndexName).setQuery(queryBuilder().intersectionQuery(defaultFieldName, "Big_Rectangle")), @@ -483,7 +507,11 @@ public void testQueryRandomGeoCollection() throws Exception { ensureGreen(); XContentBuilder docSource = GeoJson.toXContent(gcb, jsonBuilder().startObject().field(defaultFieldName), null).endObject(); - prepareIndex(defaultIndexName).setId("1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") + .setSource(docSource) + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCountAndNoFailures( client().prepareSearch(defaultIndexName).setQuery(queryBuilder().intersectionQuery(defaultFieldName, polygon)), @@ -522,7 +550,11 @@ public void testShapeFilterWithDefinedGeoCollection() throws Exception { .endArray() .endObject() .endObject(); - prepareIndex(defaultIndexName).setId("1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") + .setSource(docSource) + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Polygon polygon1 = new Polygon( new LinearRing(new double[] { 99.0, 99.0, 103.0, 103.0, 99.0 }, new double[] { -1.0, 3.0, 3.0, -1.0, -1.0 }) @@ -558,26 +590,26 @@ public void testDistanceQuery() throws Exception { double radius = fieldTypeName().contains("geo") ? 350000 : 35; Circle circle = new Circle(1, 0, radius); - client().index( - new IndexRequest(defaultIndexName).source( - jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(new Point(2, 2))).endObject() - ).setRefreshPolicy(IMMEDIATE) - ).actionGet(); - client().index( - new IndexRequest(defaultIndexName).source( - jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(new Point(3, 1))).endObject() - ).setRefreshPolicy(IMMEDIATE) - ).actionGet(); - client().index( - new IndexRequest(defaultIndexName).source( - jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(new Point(-20, -30))).endObject() - ).setRefreshPolicy(IMMEDIATE) - ).actionGet(); - client().index( - new IndexRequest(defaultIndexName).source( - jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(new Point(20, 30))).endObject() - ).setRefreshPolicy(IMMEDIATE) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest(defaultIndexName).source( + jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(new Point(2, 2))).endObject() + ).setRefreshPolicy(IMMEDIATE); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest(defaultIndexName).source( + jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(new Point(3, 1))).endObject() + ).setRefreshPolicy(IMMEDIATE); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest(defaultIndexName).source( + jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(new Point(-20, -30))).endObject() + ).setRefreshPolicy(IMMEDIATE); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + indexRequest = new IndexRequest(defaultIndexName).source( + jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(new Point(20, 30))).endObject() + ).setRefreshPolicy(IMMEDIATE); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); assertHitCount( client().prepareSearch(defaultIndexName) @@ -610,9 +642,11 @@ public void testIndexLineQueryPoints() throws Exception { Line line = makeRandomLine(); - prepareIndex(defaultIndexName).setSource(jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(line)).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setSource( + jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(line)).endObject() + ).setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); // all points from a line intersect with the line for (int i = 0; i < line.length(); i++) { Point point = new Point(line.getLon(i), line.getLat(i)); @@ -631,9 +665,11 @@ public void testIndexPolygonQueryPoints() throws Exception { Polygon polygon = makeRandomPolygon(); - prepareIndex(defaultIndexName).setSource( + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setSource( jsonBuilder().startObject().field(defaultFieldName, WellKnownText.toWKT(polygon)).endObject() - ).setRefreshPolicy(IMMEDIATE).get(); + ).setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); // all points from a polygon intersect with the polygon LinearRing linearRing = polygon.getPolygon(); @@ -665,9 +701,11 @@ public void testNeighbours() throws Exception { }; for (String polygon : polygons) { - prepareIndex(defaultIndexName).setSource(jsonBuilder().startObject().field(defaultFieldName, polygon).endObject()) - .setRefreshPolicy(IMMEDIATE) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setSource( + jsonBuilder().startObject().field(defaultFieldName, polygon).endObject() + ).setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } Geometry center = WellKnownText.fromWKT(StandardValidator.instance(false), false, polygons[0]); assertHitCountAndNoFailures( diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/DatelinePointShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/DatelinePointShapeQueryTestCase.java index 751a4d835610d..2fc206f6c362f 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/geo/DatelinePointShapeQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/DatelinePointShapeQueryTestCase.java @@ -8,6 +8,7 @@ package org.elasticsearch.search.geo; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.MultiPolygon; @@ -41,26 +42,29 @@ public void testRectangleSpanningDateline(BasePointShapeQueryTestCase querySupplier = randomFrom( @@ -155,7 +162,11 @@ public void testIndexRectangleSpanningDateLine() throws Exception { Rectangle envelope = new Rectangle(178, -178, 10, -10); XContentBuilder docSource = GeoJson.toXContent(envelope, jsonBuilder().startObject().field(defaultFieldName), null).endObject(); - prepareIndex(defaultIndexName).setId("1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(defaultIndexName).setId("1") + .setSource(docSource) + .setRefreshPolicy(IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Point filterShape = new Point(179, 0); diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 3744011b5b9f6..37cd6624c7ecb 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -496,6 +496,9 @@ protected void indexRandomDocs(String index, int numdocs) throws InterruptedExce builders[i] = prepareIndex(index).setId(Integer.toString(i)).setSource("field1", "bar " + i); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } flushAndRefresh(index); assertDocCount(index, numdocs); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractSearchCancellationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractSearchCancellationTestCase.java index c9523b9f73c38..f5bd5e0bc4397 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractSearchCancellationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractSearchCancellationTestCase.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; @@ -74,7 +75,13 @@ protected void indexTestData() { // Make sure we have a few segments try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int j = 0; j < 20; j++) { - bulkRequestBuilder.add(prepareIndex("test").setId(Integer.toString(i * 5 + j)).setSource("field", "value")); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i * 5 + j)) + .setSource("field", "value"); + try { + bulkRequestBuilder.add(indexRequestBuilder); + } finally { + indexRequestBuilder.request().decRef(); + } } assertNoFailures(bulkRequestBuilder.get()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java index 4df1e745f3bf4..21f36cb74f0c0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java @@ -134,6 +134,7 @@ private XContentTester( CheckedBiFunction toXContent, CheckedFunction fromXContent ) { + this.createParser = createParser; this.instanceSupplier = instanceSupplier; this.toXContent = toXContent; diff --git a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java index ddbd592fdf7ae..4715b7cc8fc91 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java +++ b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java @@ -19,6 +19,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.BulkShardRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.TimeValue; @@ -142,13 +143,16 @@ public void run() { for (int i = 0; i < batchSize; i++) { id = idGenerator.incrementAndGet(); if (useAutoGeneratedIDs) { - bulkRequest.add(client.prepareIndex(index).setSource(generateSource(id, threadRandom))); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index) + .setSource(generateSource(id, threadRandom)); + bulkRequest.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } else { - bulkRequest.add( - client.prepareIndex(index) - .setId(Long.toString(id)) - .setSource(generateSource(id, threadRandom)) - ); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index) + .setId(Long.toString(id)) + .setSource(generateSource(id, threadRandom)); + bulkRequest.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } } try { @@ -176,30 +180,36 @@ public void run() { id = idGenerator.incrementAndGet(); if (useAutoGeneratedIDs) { try { - DocWriteResponse indexResponse = client.prepareIndex(index) + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index) .setTimeout(timeout) - .setSource(generateSource(id, threadRandom)) - .get(); - boolean add = ids.add(indexResponse.getId()); - assert add : "ID: " + indexResponse.getId() + " already used"; + .setSource(generateSource(id, threadRandom)); + try { + DocWriteResponse indexResponse = indexRequestBuilder.get(); + boolean add = ids.add(indexResponse.getId()); + assert add : "ID: " + indexResponse.getId() + " already used"; + } finally { + indexRequestBuilder.request().decRef(); + } } catch (Exception e) { if (ignoreIndexingFailures == false) { throw e; } } } else { + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); try { - DocWriteResponse indexResponse = client.prepareIndex(index) - .setId(Long.toString(id)) + indexRequestBuilder.setId(Long.toString(id)) .setTimeout(timeout) - .setSource(generateSource(id, threadRandom)) - .get(); + .setSource(generateSource(id, threadRandom)); + DocWriteResponse indexResponse = indexRequestBuilder.get(); boolean add = ids.add(indexResponse.getId()); assert add : "ID: " + indexResponse.getId() + " already used"; } catch (Exception e) { if (ignoreIndexingFailures == false) { throw e; } + } finally { + indexRequestBuilder.request().decRef(); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 174e2b81bb3c4..5454b067ca014 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1426,7 +1426,12 @@ protected static IndexRequestBuilder prepareIndex(String index) { * */ protected final DocWriteResponse index(String index, XContentBuilder source) { - return prepareIndex(index).setSource(source).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } /** @@ -1436,7 +1441,12 @@ protected final DocWriteResponse index(String index, XContentBuilder source) { * */ protected final DocWriteResponse index(String index, String id, Map source) { - return prepareIndex(index).setId(id).setSource(source).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } /** @@ -1446,7 +1456,12 @@ protected final DocWriteResponse index(String index, String id, Map */ protected final DocWriteResponse index(String index, String id, XContentBuilder source) { - return prepareIndex(index).setId(id).setSource(source).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } /** @@ -1456,7 +1471,12 @@ protected final DocWriteResponse index(String index, String id, XContentBuilder * */ protected final DocWriteResponse indexDoc(String index, String id, Object... source) { - return prepareIndex(index).setId(id).setSource(source).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } /** @@ -1468,7 +1488,12 @@ protected final DocWriteResponse indexDoc(String index, String id, Object... sou * where source is a JSON String. */ protected final DocWriteResponse index(String index, String id, String source) { - return prepareIndex(index).setId(id).setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source, XContentType.JSON).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } /** @@ -1581,7 +1606,13 @@ public void indexRandom(boolean forceRefresh, String index, int numDocs) throws for (int i = 0; i < builders.length; i++) { builders[i] = prepareIndex(index).setSource("field", "value"); } - indexRandom(forceRefresh, Arrays.asList(builders)); + try { + indexRandom(forceRefresh, Arrays.asList(builders)); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } } /** @@ -1651,6 +1682,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma for (IndexRequestBuilder builder : builders) { indices.add(builder.request().index()); } + List newIndexRequestBuilders = new ArrayList<>(); Set> bogusIds = new HashSet<>(); // (index, type, id) if (random.nextBoolean() && builders.isEmpty() == false && dummyDocuments) { builders = new ArrayList<>(builders); @@ -1662,7 +1694,9 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma String index = RandomPicks.randomFrom(random, indices); bogusIds.add(Arrays.asList(index, id)); // We configure a routing key in case the mapping requires it - builders.add(prepareIndex(index).setId(id).setSource("{}", XContentType.JSON).setRouting(id)); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id).setSource("{}", XContentType.JSON).setRouting(id); + builders.add(indexRequestBuilder); + newIndexRequestBuilders.add(indexRequestBuilder); } } Collections.shuffle(builders, random()); @@ -1705,6 +1739,10 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma for (CountDownLatch operation : inFlightAsyncOperations) { operation.await(); } + for (IndexRequestBuilder indexRequestBuilder : newIndexRequestBuilders) { + // We created these, so we're responsible for their lifecycle + indexRequestBuilder.request().decRef(); + } final List actualErrors = new ArrayList<>(); for (Tuple tuple : errors) { Throwable t = ExceptionsHelper.unwrapCause(tuple.v2()); diff --git a/x-pack/plugin/analytics/src/internalClusterTest/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsWithRequestBreakerIT.java b/x-pack/plugin/analytics/src/internalClusterTest/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsWithRequestBreakerIT.java index 4c8666365f603..e3b9c5db75af6 100644 --- a/x-pack/plugin/analytics/src/internalClusterTest/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsWithRequestBreakerIT.java +++ b/x-pack/plugin/analytics/src/internalClusterTest/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsWithRequestBreakerIT.java @@ -39,15 +39,16 @@ public void testRequestBreaker() throws Exception { final String requestBreaker = randomIntBetween(1, 10000) + "kb"; logger.info("--> Using request breaker setting: {}", requestBreaker); - indexRandom( - true, - IntStream.range(0, randomIntBetween(10, 1000)) - .mapToObj( - i -> prepareIndex("test").setId("id_" + i) - .setSource(Map.of("field0", randomAlphaOfLength(5), "field1", randomAlphaOfLength(5))) - ) - .toArray(IndexRequestBuilder[]::new) - ); + IndexRequestBuilder[] indexRequestBuilders = IntStream.range(0, randomIntBetween(10, 1000)) + .mapToObj( + i -> prepareIndex("test").setId("id_" + i) + .setSource(Map.of("field0", randomAlphaOfLength(5), "field1", randomAlphaOfLength(5))) + ) + .toArray(IndexRequestBuilder[]::new); + indexRandom(true, indexRequestBuilders); + for (IndexRequestBuilder builder : indexRequestBuilders) { + builder.request().decRef(); + } updateClusterSettings( Settings.builder().put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), requestBreaker) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistogramPercentileAggregationTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistogramPercentileAggregationTests.java index af19448026070..1d1ab83d230eb 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistogramPercentileAggregationTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistogramPercentileAggregationTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentiles; @@ -84,7 +85,9 @@ public void testHDRHistogram() throws Exception { for (int i = 0; i < numDocs; i++) { double value = random().nextDouble(); XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("data", value).endObject(); - bulkRequest.add(new IndexRequest("raw").source(doc)); + IndexRequest indexRequest = new IndexRequest("raw").source(doc); + bulkRequest.add(indexRequest); + indexRequest.decRef(); histogram.recordValue(value); if ((i + 1) % frq == 0) { client().bulk(bulkRequest); @@ -105,7 +108,9 @@ public void testHDRHistogram() throws Exception { .field("counts", counts.toArray(new Integer[counts.size()])) .endObject() .endObject(); - prepareIndex("pre_agg").setSource(preAggDoc).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("pre_agg").setSource(preAggDoc); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); histogram.reset(); } } @@ -197,7 +202,9 @@ private void setupTDigestHistogram(int compression) throws Exception { .field("data", value) .endObject() .endObject(); - bulkRequest.add(new IndexRequest("raw").source(doc)); + IndexRequest indexRequest = new IndexRequest("raw").source(doc); + bulkRequest.add(indexRequest); + indexRequest.decRef(); histogram.add(value); if ((i + 1) % frq == 0) { client().bulk(bulkRequest); @@ -219,7 +226,9 @@ private void setupTDigestHistogram(int compression) throws Exception { .endObject() .endObject() .endObject(); - prepareIndex("pre_agg").setSource(preAggDoc).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("pre_agg").setSource(preAggDoc); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); histogram = TDigestState.create(compression); } } diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java index 58644208a1da0..62826e9d61a9d 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java @@ -92,6 +92,9 @@ public void setupSuiteScopeCluster() throws InterruptedException { reqs.add(prepareIndex(indexName).setSource("terms", keyword, "metric", metric)); } indexRandom(true, true, reqs); + for (IndexRequestBuilder indexRequestBuilder : reqs) { + indexRequestBuilder.request().decRef(); + } } @Override diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java index 3605d6365f867..ceb51181eea02 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.search.ShardSearchFailure; @@ -1745,7 +1746,12 @@ private int indexDocs(Client client, String index) { if (i == numDocs - 1) { ts = LATEST_TIMESTAMP; } - client.prepareIndex(index).setSource("f", "v", "@timestamp", ts).get(); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); + try { + indexRequestBuilder.setSource("f", "v", "@timestamp", ts).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } client.admin().indices().prepareRefresh(index).get(); return numDocs; diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java index 302bb68af6c61..202d8b806f6d2 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java @@ -45,7 +45,9 @@ protected Collection> getPlugins() { public void testFetchFailuresAllShards() throws Exception { for (int i = 0; i < 10; i++) { - DocWriteResponse indexResponse = client().index(new IndexRequest("boom" + i).id("boom" + i).source("text", "value")).get(); + IndexRequest indexRequest = new IndexRequest("boom" + i).id("boom" + i).source("text", "value"); + DocWriteResponse indexResponse = client().index(indexRequest).get(); + indexRequest.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); } client().admin().indices().refresh(new RefreshRequest()).get(); @@ -85,11 +87,15 @@ public void testFetchFailuresAllShards() throws Exception { public void testFetchFailuresOnlySomeShards() throws Exception { for (int i = 0; i < 5; i++) { - DocWriteResponse indexResponse = client().index(new IndexRequest("boom" + i).id("boom" + i).source("text", "value")).get(); + IndexRequest indexRequest = new IndexRequest("boom" + i).id("boom" + i).source("text", "value"); + DocWriteResponse indexResponse = client().index(indexRequest).get(); + indexRequest.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); } for (int i = 0; i < 5; i++) { - DocWriteResponse indexResponse = client().index(new IndexRequest("index" + i).id("index" + i).source("text", "value")).get(); + IndexRequest indexRequest = new IndexRequest("index" + i).id("index" + i).source("text", "value"); + DocWriteResponse indexResponse = client().index(indexRequest).get(); + indexRequest.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); } client().admin().indices().refresh(new RefreshRequest()).get(); diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java index 656b1ddd4d952..bdea7088c054a 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.autoscaling; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.blobcache.BlobCachePlugin; import org.elasticsearch.blobcache.shared.SharedBlobCacheService; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -85,10 +86,13 @@ protected void setupRepoAndPolicy() { protected void createAndMountIndex() throws InterruptedException, java.util.concurrent.ExecutionException { assertAcked(prepareCreate(indexName, Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true))); - indexRandom( - randomBoolean(), - IntStream.range(0, 10).mapToObj(i -> prepareIndex(indexName).setSource()).collect(Collectors.toList()) - ); + List builders = IntStream.range(0, 10) + .mapToObj(i -> prepareIndex(indexName).setSource()) + .collect(Collectors.toList()); + indexRandom(randomBoolean(), builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } final SnapshotInfo snapshotInfo = createFullSnapshot(fsRepoName, snapshotName); diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageIT.java index 0f4983c1b6994..55156cf2c37c3 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageIT.java @@ -48,16 +48,16 @@ public void testScaleUp() throws IOException, InterruptedException { createDataStreamAndTemplate(dsName); final int rolloverCount = between(1, 5); for (int i = 0; i < rolloverCount; ++i) { - indexRandom( - true, - false, - IntStream.range(1, 100) - .mapToObj( - unused -> prepareIndex(dsName).setCreate(true) - .setSource("@timestamp", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(randomMillisUpToYear9999())) - ) - .toArray(IndexRequestBuilder[]::new) - ); + IndexRequestBuilder[] builders = IntStream.range(1, 100) + .mapToObj( + unused -> prepareIndex(dsName).setCreate(true) + .setSource("@timestamp", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(randomMillisUpToYear9999())) + ) + .toArray(IndexRequestBuilder[]::new); + indexRandom(true, false, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } assertAcked(indicesAdmin().rolloverIndex(new RolloverRequest(dsName, null)).actionGet()); } forceMerge(); diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java index 2406fc6b4e92a..6e4814269fc73 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java @@ -66,12 +66,13 @@ public void testScaleUp() throws InterruptedException { .put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms") .build() ); - indexRandom( - true, - IntStream.range(1, 100) - .mapToObj(i -> prepareIndex(indexName).setSource("field", randomAlphaOfLength(50))) - .toArray(IndexRequestBuilder[]::new) - ); + IndexRequestBuilder[] builders = IntStream.range(1, 100) + .mapToObj(i -> prepareIndex(indexName).setSource("field", randomAlphaOfLength(50))) + .toArray(IndexRequestBuilder[]::new); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } forceMerge(); refresh(); @@ -265,12 +266,13 @@ public void testScaleWhileShrinking() throws Exception { .put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms") .build() ); - indexRandom( - true, - IntStream.range(1, 100) - .mapToObj(i -> prepareIndex(indexName).setSource("field", randomAlphaOfLength(50))) - .toArray(IndexRequestBuilder[]::new) - ); + IndexRequestBuilder[] builders = IntStream.range(1, 100) + .mapToObj(i -> prepareIndex(indexName).setSource("field", randomAlphaOfLength(50))) + .toArray(IndexRequestBuilder[]::new); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } forceMerge(); refresh(); @@ -419,12 +421,13 @@ public void testScaleDuringSplitOrClone() throws Exception { .put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms") .build() ); - indexRandom( - true, - IntStream.range(1, 100) - .mapToObj(i -> prepareIndex(indexName).setSource("field", randomAlphaOfLength(50))) - .toArray(IndexRequestBuilder[]::new) - ); + IndexRequestBuilder[] builders = IntStream.range(1, 100) + .mapToObj(i -> prepareIndex(indexName).setSource("field", randomAlphaOfLength(50))) + .toArray(IndexRequestBuilder[]::new); + indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } forceMerge(); refresh(); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index df8f19ab3888c..bf79e224be9d7 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; @@ -137,7 +138,10 @@ public void testAutoFollowDoNotFollowSystemIndices() throws Exception { putAutoFollowPatterns("my-pattern", new String[] { ".*", "logs-*" }); // Trigger system index creation - leaderClient().prepareIndex(FakeSystemIndex.SYSTEM_INDEX_NAME).setSource(Map.of("a", "b")).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(FakeSystemIndex.SYSTEM_INDEX_NAME) + .setSource(Map.of("a", "b")); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); Settings leaderIndexSettings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) @@ -657,10 +661,13 @@ public void testAutoFollowDatastreamWithClosingFollowerIndex() throws Exception CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(datastream); assertAcked(leaderClient().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get()); - leaderClient().prepareIndex(datastream) - .setCreate(true) - .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD_NAME, randomNonNegativeLong()) - .get(); + { + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(datastream) + .setCreate(true) + .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD_NAME, randomNonNegativeLong()); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } PutAutoFollowPatternAction.Request followRequest = new PutAutoFollowPatternAction.Request(); followRequest.setName("pattern-1"); @@ -692,10 +699,13 @@ public void testAutoFollowDatastreamWithClosingFollowerIndex() throws Exception .prepareCreate(indexInDatastream) .setMapping(MetadataIndexTemplateService.DEFAULT_TIMESTAMP_MAPPING_WITHOUT_ROUTING.toString()) ); - leaderClient().prepareIndex(indexInDatastream) - .setCreate(true) - .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD_NAME, randomNonNegativeLong()) - .get(); + { + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(indexInDatastream) + .setCreate(true) + .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD_NAME, randomNonNegativeLong()); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } leaderClient().execute( ModifyDataStreamsAction.INSTANCE, new ModifyDataStreamsAction.Request(List.of(DataStreamAction.addBackingIndex(datastream, indexInDatastream))) diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index de6992a67813d..8803e5863c8de 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; @@ -217,7 +218,11 @@ public void testDocsAreRecovered() throws Exception { logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); for (int i = 0; i < firstBatchNumDocs; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex("index1") + .setId(Integer.toString(i)) + .setSource(source, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } leaderClient().admin().indices().prepareFlush(leaderIndex).setForce(true).setWaitIfOngoing(true).get(); @@ -284,7 +289,11 @@ public void testRateLimitingIsEmployed() throws Exception { logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex("index1") + .setId(Integer.toString(i)) + .setSource(source, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } leaderClient().admin().indices().prepareFlush(leaderIndex).setForce(true).setWaitIfOngoing(true).get(); @@ -349,7 +358,11 @@ public void testIndividualActionsTimeout() throws Exception { logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex("index1") + .setId(Integer.toString(i)) + .setSource(source, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } leaderClient().admin().indices().prepareFlush(leaderIndex).setForce(true).setWaitIfOngoing(true).get(); @@ -501,7 +514,9 @@ public void testCcrRepositoryFetchesSnapshotShardSizeFromIndexShardStoreStats() if (numDocs > 0) { try (BulkRequestBuilder bulkRequest = leaderClient().prepareBulk(leaderIndex)) { for (int i = 0; i < numDocs; i++) { - bulkRequest.add(new IndexRequest(leaderIndex).id(Integer.toString(i)).source("field", i)); + IndexRequest indexRequest = new IndexRequest(leaderIndex).id(Integer.toString(i)).source("field", i); + bulkRequest.add(indexRequest); + indexRequest.decRef(); } assertThat(bulkRequest.get().hasFailures(), is(false)); } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java index 339662c996492..4ce7005fc3e45 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; @@ -140,7 +141,11 @@ private RestoreSnapshotRequest setUpRestoreSnapshotRequest( logger.info("indexing [{}] docs", numberOfDocuments); for (int i = 0; i < numberOfDocuments; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex(leaderIndex).setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(leaderIndex) + .setId(Integer.toString(i)) + .setSource(source, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); if (rarely()) { leaderClient().admin().indices().prepareFlush(leaderIndex).setForce(true).setWaitIfOngoing(true).get(); } @@ -626,7 +631,11 @@ public void testRetentionLeaseAdvancesWhileFollowing() throws Exception { logger.debug("indexing [{}] docs", numberOfDocuments); for (int i = 0; i < numberOfDocuments; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex(leaderIndex).setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(leaderIndex) + .setId(Integer.toString(i)) + .setSource(source, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); if (rarely()) { leaderClient().admin().indices().prepareFlush(leaderIndex).setForce(true).setWaitIfOngoing(true).get(); } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java index ca1f1b6c4f12c..ce67382cfbd01 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterState; @@ -94,7 +95,9 @@ public void testCloseAndReopenFollowerIndex() throws Exception { for (int i = 0; i < numThreads; i++) { threads[i] = new Thread(() -> { while (isRunning.get()) { - leaderClient().prepareIndex("index1").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex("index1").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } }); threads[i].start(); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java index 501a664d64698..b88c60d5d43ec 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -74,10 +75,7 @@ public void testFailOverOnFollower() throws Exception { } if (frequently()) { String id = Integer.toString(frequently() ? docID.incrementAndGet() : between(0, 10)); // sometimes update - DocWriteResponse indexResponse = leaderClient().prepareIndex(leaderIndex) - .setId(id) - .setSource("{\"f\":" + id + "}", XContentType.JSON) - .get(); + DocWriteResponse indexResponse = index(leaderIndex, id, "{\"f\":" + id + "}", XContentType.JSON, null); logger.info("--> index {} id={} seq_no={}", leaderIndex, indexResponse.getId(), indexResponse.getSeqNo()); } else { String id = Integer.toString(between(0, docID.get())); @@ -141,10 +139,7 @@ public void testFollowIndexAndCloseNode() throws Exception { } Object[] args = new Object[] { counter++ }; final String source = Strings.format("{\"f\":%d}", args); - DocWriteResponse indexResp = leaderClient().prepareIndex("index1") - .setSource(source, XContentType.JSON) - .setTimeout(TimeValue.timeValueSeconds(1)) - .get(); + DocWriteResponse indexResp = index("index1", null, source, XContentType.JSON, TimeValue.timeValueSeconds(1)); logger.info("--> index id={} seq_no={}", indexResp.getId(), indexResp.getSeqNo()); } }); @@ -192,10 +187,10 @@ public void testAddNewReplicasOnFollower() throws Exception { try { if (appendOnly) { String id = Integer.toString(docID.incrementAndGet()); - leaderClient().prepareIndex("leader-index").setId(id).setSource("{\"f\":" + id + "}", XContentType.JSON).get(); + index("leader-index", id, "{\"f\":" + id + "}", XContentType.JSON, null); } else if (frequently()) { String id = Integer.toString(frequently() ? docID.incrementAndGet() : between(0, 100)); - leaderClient().prepareIndex("leader-index").setId(id).setSource("{\"f\":" + id + "}", XContentType.JSON).get(); + index("leader-index", id, "{\"f\":" + id + "}", XContentType.JSON, null); } else { String id = Integer.toString(between(0, docID.get())); leaderClient().prepareDelete("leader-index", id).get(); @@ -289,12 +284,7 @@ public void testReadRequestsReturnLatestMappingVersion() throws Exception { assertNotNull(mapper); assertNotNull(mapper.mappers().getMapper("balance")); }); - DocWriteResponse indexResp = leaderCluster.client() - .prepareIndex("leader-index") - .setId("1") - .setSource("{\"balance\": 100}", XContentType.JSON) - .setTimeout(TimeValue.ZERO) - .get(); + DocWriteResponse indexResp = index("leader-index", "1", "{\"balance\": 100}", XContentType.JSON, TimeValue.ZERO); assertThat(indexResp.getResult(), equalTo(DocWriteResponse.Result.CREATED)); assertThat(indexShard.getLastKnownGlobalCheckpoint(), equalTo(0L)); // Make sure at least one read-request which requires mapping sync is completed. @@ -313,4 +303,17 @@ public void testReadRequestsReturnLatestMappingVersion() throws Exception { pauseFollow("follower-index"); } } + + private DocWriteResponse index(String index, String id, String source, XContentType contentType, TimeValue timeout) { + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source, contentType); + if (timeout != null) { + indexRequestBuilder.setTimeout(timeout); + } + return indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 097592a03d5d0..558c8ac5e3197 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; @@ -40,6 +41,7 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.ClusterState; @@ -266,7 +268,7 @@ public void testFollowIndexWithConcurrentMappingChanges() throws Exception { final int firstBatchNumDocs = randomIntBetween(2, 64); logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); for (int i = 0; i < firstBatchNumDocs; i++) { - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource("f", i).get(); + indexDoc("index1", Integer.toString(i), "f", i); } AtomicBoolean isRunning = new AtomicBoolean(true); @@ -280,7 +282,7 @@ public void testFollowIndexWithConcurrentMappingChanges() throws Exception { break; } final String field = "f-" + between(1, numFields); - leaderClient().prepareIndex("index1").setSource(field, between(0, 1000)).get(); + indexDoc("index1", null, field, between(0, 1000)); if (rarely()) { leaderClient().admin().indices().prepareFlush("index1").setWaitIfOngoing(false).setForce(false).get(); } @@ -300,7 +302,7 @@ public void testFollowIndexWithConcurrentMappingChanges() throws Exception { final int secondBatchNumDocs = randomIntBetween(2, 64); logger.info("Indexing [{}] docs as second batch", secondBatchNumDocs); for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource("f", i).get(); + indexDoc("index1", Integer.toString(i), "f", i); } for (int i = 0; i < firstBatchNumDocs + secondBatchNumDocs; i++) { assertBusy(assertExpectedDocumentRunnable(i), 1, TimeUnit.MINUTES); @@ -320,7 +322,7 @@ public void testFollowIndexWithoutWaitForComplete() throws Exception { logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); for (int i = 0; i < firstBatchNumDocs; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + index("index1", Integer.toString(i), source, XContentType.JSON); } final PutFollowAction.Request followRequest = putFollow("index1", "index2", ActiveShardCount.NONE); @@ -367,7 +369,7 @@ public void testSyncMappings() throws Exception { final long firstBatchNumDocs = randomIntBetween(2, 64); for (long i = 0; i < firstBatchNumDocs; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Long.toString(i)).setSource(source, XContentType.JSON).get(); + index("index1", Long.toString(i), source, XContentType.JSON); } assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs)); @@ -378,7 +380,7 @@ public void testSyncMappings() throws Exception { final int secondBatchNumDocs = randomIntBetween(2, 64); for (long i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { final String source = Strings.format("{\"k\":%d}", i); - leaderClient().prepareIndex("index1").setId(Long.toString(i)).setSource(source, XContentType.JSON).get(); + index("index1", Long.toString(i), source, XContentType.JSON); } assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs + secondBatchNumDocs)); @@ -406,7 +408,7 @@ public void testNoMappingDefined() throws Exception { final PutFollowAction.Request followRequest = putFollow("index1", "index2"); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); - leaderClient().prepareIndex("index1").setId("1").setSource("{\"f\":1}", XContentType.JSON).get(); + index("index1", "1", "{\"f\":1}", XContentType.JSON); assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); pauseFollow("index2"); @@ -519,6 +521,7 @@ public void afterBulk(long executionId, BulkRequest request, Exception failure) IndexRequest indexRequest = new IndexRequest("index1").source(source, XContentType.JSON) .timeout(TimeValue.timeValueSeconds(1)); bulkProcessor.add(indexRequest); + indexRequest.decRef(); } }); thread.start(); @@ -568,7 +571,11 @@ public void testFollowIndexWithNestedField() throws Exception { } builder.endArray(); builder.endObject(); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(builder).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex("index1") + .setId(Integer.toString(i)) + .setSource(builder); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } } @@ -638,7 +645,7 @@ public void testFollowIndexMaxOperationSizeInBytes() throws Exception { logger.info("Indexing [{}] docs", numDocs); for (int i = 0; i < numDocs; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + index("index1", Integer.toString(i), source, XContentType.JSON); } PutFollowAction.Request followRequest = putFollow("index1", "index2"); @@ -704,7 +711,7 @@ public void testCloseLeaderIndex() throws Exception { final PutFollowAction.Request followRequest = putFollow("index1", "index2"); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); - leaderClient().prepareIndex("index1").setId("1").setSource("{}", XContentType.JSON).get(); + index("index1", "1", "{}", XContentType.JSON); assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); leaderClient().admin().indices().close(new CloseIndexRequest("index1")).actionGet(); @@ -728,7 +735,7 @@ public void testCloseLeaderIndex() throws Exception { }); leaderClient().admin().indices().open(new OpenIndexRequest("index1")).actionGet(); - leaderClient().prepareIndex("index1").setId("2").setSource("{}", XContentType.JSON).get(); + index("index1", "2", "{}", XContentType.JSON); assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 2)); pauseFollow("index2"); @@ -750,11 +757,11 @@ public void testCloseFollowIndex() throws Exception { final PutFollowAction.Request followRequest = putFollow("index1", "index2"); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); - leaderClient().prepareIndex("index1").setId("1").setSource("{}", XContentType.JSON).get(); + index("index1", "1", "{}", XContentType.JSON); assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); followerClient().admin().indices().close(new CloseIndexRequest("index2").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet(); - leaderClient().prepareIndex("index1").setId("2").setSource("{}", XContentType.JSON).get(); + index("index1", "2", "{}", XContentType.JSON); assertBusy(() -> { StatsResponses response = followerClient().execute(FollowStatsAction.INSTANCE, new StatsRequest()).actionGet(); assertThat(response.getNodeFailures(), empty()); @@ -784,7 +791,7 @@ public void testDeleteLeaderIndex() throws Exception { final PutFollowAction.Request followRequest = putFollow("index1", "index2"); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); - leaderClient().prepareIndex("index1").setId("1").setSource("{}", XContentType.JSON).get(); + index("index1", "1", "{}", XContentType.JSON); assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); leaderClient().admin().indices().delete(new DeleteIndexRequest("index1")).actionGet(); @@ -831,7 +838,7 @@ public void testResumeFollowOnClosedIndex() throws Exception { ensureLeaderGreen(leaderIndex); final int nbDocs = randomIntBetween(10, 100); - IntStream.of(nbDocs).forEach(i -> leaderClient().prepareIndex().setIndex(leaderIndex).setSource("field", i).get()); + IntStream.of(nbDocs).forEach(i -> indexDoc(leaderIndex, null, "field", i)); final String followerIndex = "follow-test-index"; PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, putFollow(leaderIndex, followerIndex)) @@ -865,11 +872,11 @@ public void testDeleteFollowerIndex() throws Exception { final PutFollowAction.Request followRequest = putFollow("index1", "index2"); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); - leaderClient().prepareIndex("index1").setId("1").setSource("{}", XContentType.JSON).get(); + index("index1", "1", "{}", XContentType.JSON); assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); followerClient().admin().indices().delete(new DeleteIndexRequest("index2").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet(); - leaderClient().prepareIndex("index1").setId("2").setSource("{}", XContentType.JSON).get(); + index("index1", "2", "{}", XContentType.JSON); assertBusy(() -> { StatsResponses response = followerClient().execute(FollowStatsAction.INSTANCE, new StatsRequest()).actionGet(); assertThat(response.getNodeFailures(), empty()); @@ -928,7 +935,7 @@ public void testUnfollowIndex() throws Exception { assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON).get()); PutFollowAction.Request followRequest = putFollow("index1", "index2"); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); - leaderClient().prepareIndex("index1").setSource("{}", XContentType.JSON).get(); + index("index1", null, "{}", XContentType.JSON); assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); // Indexing directly into index2 would fail now, because index2 is a follow index. @@ -942,10 +949,11 @@ public void testUnfollowIndex() throws Exception { ensureFollowerGreen("index2"); // Indexing succeeds now, because index2 is no longer a follow index: - followerClient().prepareIndex("index2") + IndexRequestBuilder indexRequestBuilder = followerClient().prepareIndex("index2") .setSource("{}", XContentType.JSON) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertHitCount(followerClient().prepareSearch("index2"), 2); } @@ -1016,7 +1024,7 @@ public void testUpdateDynamicLeaderIndexSettings() throws Exception { final long firstBatchNumDocs = randomIntBetween(2, 64); for (long i = 0; i < firstBatchNumDocs; i++) { - leaderClient().prepareIndex("leader").setSource("{}", XContentType.JSON).get(); + index("leader", null, "{}", XContentType.JSON); } assertBusy(() -> assertHitCount(followerClient().prepareSearch("follower"), firstBatchNumDocs)); @@ -1034,7 +1042,7 @@ public void testUpdateDynamicLeaderIndexSettings() throws Exception { final int secondBatchNumDocs = randomIntBetween(2, 64); for (long i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { - leaderClient().prepareIndex("leader").setSource("{}", XContentType.JSON).get(); + index("leader", null, "{}", XContentType.JSON); } assertBusy(() -> { // Check that the setting has been set in follower index: @@ -1067,7 +1075,7 @@ public void testLeaderIndexSettingNotPercolatedToFollower() throws Exception { final long firstBatchNumDocs = randomIntBetween(2, 64); for (long i = 0; i < firstBatchNumDocs; i++) { - leaderClient().prepareIndex("leader").setSource("{}", XContentType.JSON).get(); + index("leader", null, "{}", XContentType.JSON); } assertBusy(() -> assertHitCount(followerClient().prepareSearch("follower"), firstBatchNumDocs)); @@ -1085,7 +1093,7 @@ public void testLeaderIndexSettingNotPercolatedToFollower() throws Exception { final int secondBatchNumDocs = randomIntBetween(2, 64); for (long i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { - leaderClient().prepareIndex("leader").setSource("{}", XContentType.JSON).get(); + index("leader", null, "{}", XContentType.JSON); } assertBusy(() -> { GetSettingsRequest getSettingsRequest = new GetSettingsRequest(); @@ -1114,7 +1122,7 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception { final long firstBatchNumDocs = randomIntBetween(2, 64); for (long i = 0; i < firstBatchNumDocs; i++) { - leaderClient().prepareIndex("leader").setSource("{}", XContentType.JSON).get(); + index("leader", null, "{}", XContentType.JSON); } assertBusy(() -> assertHitCount(followerClient().prepareSearch("follower"), firstBatchNumDocs)); @@ -1143,7 +1151,7 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception { final int secondBatchNumDocs = randomIntBetween(2, 64); for (long i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { final String source = Strings.format("{\"new_field\":\"value %d\"}", i); - leaderClient().prepareIndex("leader").setSource(source, XContentType.JSON).get(); + index("leader", null, source, XContentType.JSON); } assertBusy(() -> { @@ -1386,7 +1394,7 @@ private void runFallBehindTest( logger.info("Indexing [{}] docs as first batch", numDocs); for (int i = 0; i < numDocs; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + index("index1", Integer.toString(i), source, XContentType.JSON); } final PutFollowAction.Request followRequest = putFollow("index1", "index2"); @@ -1406,7 +1414,7 @@ private void runFallBehindTest( for (int i = 0; i < numDocs; i++) { final String source = Strings.format("{\"f\":%d}", i * 2); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + index("index1", Integer.toString(i), source, XContentType.JSON); } leaderClient().prepareDelete("index1", "1").get(); leaderClient().admin().indices().refresh(new RefreshRequest("index1")).actionGet(); @@ -1550,7 +1558,7 @@ public void testCleanUpShardFollowTasksForDeletedIndices() throws Exception { final PutFollowAction.Request followRequest = putFollow("index1", "index2"); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); - leaderClient().prepareIndex("index1").setId("1").setSource("{}", XContentType.JSON).get(); + index("index1", "1", "{}", XContentType.JSON); assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); assertBusy(() -> { @@ -1740,6 +1748,26 @@ private void putFollowerTemplate(String setting, String settingValue) { ); } + private DocWriteResponse indexDoc(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source); + return indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private DocWriteResponse index(String index, String id, String source, XContentType contentType) { + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source, contentType); + return indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + public static class PrivateSettingPlugin extends Plugin { static final Setting INDEX_INTERNAL_SETTING = Setting.simpleString( "index.internal", diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java index 05fc3b037c795..acad17ed009d3 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ccr; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -48,7 +49,7 @@ public void testFollowIndex() throws Exception { final long firstBatchNumDocs = randomIntBetween(2, 64); for (int i = 0; i < firstBatchNumDocs; i++) { - prepareIndex("leader").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("leader"); } final PutFollowAction.Request followRequest = getPutFollowRequest("leader", "follower"); @@ -58,7 +59,7 @@ public void testFollowIndex() throws Exception { final long secondBatchNumDocs = randomIntBetween(2, 64); for (int i = 0; i < secondBatchNumDocs; i++) { - prepareIndex("leader").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("leader"); } assertBusy(() -> assertHitCount(client().prepareSearch("follower"), firstBatchNumDocs + secondBatchNumDocs)); @@ -68,7 +69,7 @@ public void testFollowIndex() throws Exception { final long thirdBatchNumDocs = randomIntBetween(2, 64); for (int i = 0; i < thirdBatchNumDocs; i++) { - prepareIndex("leader").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("leader"); } client().execute(ResumeFollowAction.INSTANCE, getResumeFollowRequest("follower")).get(); @@ -88,7 +89,9 @@ public void testIndexingMetricsIncremented() throws Exception { for (int i = 0; i < firstBatchNumDocs; i++) { BytesArray source = new BytesArray("{}"); sourceSize += source.length(); - prepareIndex("leader").setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("leader").setSource(source, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } ThreadPool nodeThreadPool = getInstanceFromNode(ThreadPool.class); @@ -150,7 +153,7 @@ public void testRemoveRemoteConnection() throws Exception { .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) .build(); createIndex("logs-20200101", leaderIndexSettings); - prepareIndex("logs-20200101").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("logs-20200101"); assertBusy(() -> { CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()).actionGet(); assertThat( @@ -169,7 +172,7 @@ public void testRemoveRemoteConnection() throws Exception { // This new index should be picked up by auto follow coordinator createIndex("logs-20200102", leaderIndexSettings); // This new document should be replicated to follower index: - prepareIndex("logs-20200101").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("logs-20200101"); assertBusy(() -> { CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()).actionGet(); assertThat( @@ -194,7 +197,7 @@ public void testChangeLeaderIndex() throws Exception { ensureGreen("index-1"); int numDocs = between(1, 100); for (int i = 0; i < numDocs; i++) { - prepareIndex("index-1").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("index-1"); } client().execute(PutFollowAction.INSTANCE, getPutFollowRequest("index-1", "index-2")).get(); assertBusy(() -> assertHitCount(client().prepareSearch("index-2"), numDocs)); @@ -208,7 +211,7 @@ public void testChangeLeaderIndex() throws Exception { newDocs = numDocs + randomIntBetween(1, 100); } for (int i = 0; i < newDocs; i++) { - prepareIndex("index-0").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("index-0"); } if (randomBoolean()) { client().admin().indices().prepareFlush("index-0").get(); @@ -255,4 +258,14 @@ public static String getIndexSettings( return settings; } + private void indexEmptyDoc(String index) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + indexRequestBuilder.setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java index bab0bbdff02e2..11371454bab17 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ccr; import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -124,7 +125,9 @@ public void testAllocateFollowerPrimaryToNodesWithRemoteClusterClientRole() thro ensureFollowerGreen(followerIndex); int numDocs = between(0, 20); for (int i = 0; i < numDocs; i++) { - leaderClient().prepareIndex(leaderIndex).setSource("f", i).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(leaderIndex).setSource("f", i); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } // Empty follower primaries must be assigned to nodes with the remote cluster client role assertBusy(() -> { @@ -181,7 +184,9 @@ public void testAllocateFollowerPrimaryToNodesWithRemoteClusterClientRole() thro }, 30, TimeUnit.SECONDS); int moreDocs = between(0, 20); for (int i = 0; i < moreDocs; i++) { - leaderClient().prepareIndex(leaderIndex).setSource("f", i).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex(leaderIndex).setSource("f", i); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } assertIndexFullyReplicatedToFollower(leaderIndex, followerIndex); } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java index 5c152be35b509..58881b5a3fe1e 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest; import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.settings.Settings; @@ -76,7 +77,11 @@ public void testFollowIndex() throws Exception { logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); for (int i = 0; i < firstBatchNumDocs; i++) { final String source = Strings.format("{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex("index1") + .setId(Integer.toString(i)) + .setSource(source, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs)); @@ -87,7 +92,9 @@ public void testFollowIndex() throws Exception { final long secondBatchNumDocs = randomIntBetween(10, 200); logger.info("Indexing [{}] docs as second batch", secondBatchNumDocs); for (int i = 0; i < secondBatchNumDocs; i++) { - leaderClient().prepareIndex("index1").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex("index1").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } getLeaderCluster().fullRestart(); @@ -96,7 +103,9 @@ public void testFollowIndex() throws Exception { final long thirdBatchNumDocs = randomIntBetween(10, 200); logger.info("Indexing [{}] docs as third batch", thirdBatchNumDocs); for (int i = 0; i < thirdBatchNumDocs; i++) { - leaderClient().prepareIndex("index1").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = leaderClient().prepareIndex("index1").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } var totalDocs = firstBatchNumDocs + secondBatchNumDocs + thirdBatchNumDocs; diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java index 6f6131c8ea4e9..d7d38b9892fd7 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ccr.action; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -51,7 +52,9 @@ public void testGetOperations() throws Exception { final int numWrites = randomIntBetween(10, 4096); for (int i = 0; i < numWrites; i++) { - prepareIndex("index").setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId(Integer.toString(i)).setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } // A number of times, get operations within a range that exists: @@ -169,7 +172,9 @@ public void testGetOperationsExceedByteLimit() throws Exception { final long numWrites = 32; for (int i = 0; i < numWrites; i++) { - prepareIndex("index").setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId(Integer.toString(i)).setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } final IndexShard indexShard = indexService.getShard(0); @@ -195,7 +200,9 @@ public void testGetOperationsExceedByteLimit() throws Exception { public void testGetOperationsAlwaysReturnAtLeastOneOp() throws Exception { final IndexService indexService = createIndex("index", indexSettings(1, 0).build()); - prepareIndex("index").setId("0").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("index").setId("0").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); final IndexShard indexShard = indexService.getShard(0); final Translog.Operation[] operations = ShardChangesAction.getOperations( diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesTests.java index 78d997ef9d777..f463c0f330884 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; @@ -49,9 +50,9 @@ protected Collection> getPlugins() { public void testGetOperationsBasedOnGlobalSequenceId() throws Exception { client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_shards", 1)).get(); - prepareIndex("index").setId("1").setSource("{}", XContentType.JSON).get(); - prepareIndex("index").setId("2").setSource("{}", XContentType.JSON).get(); - prepareIndex("index").setId("3").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("index", "1"); + indexEmptyDoc("index", "2"); + indexEmptyDoc("index", "3"); ShardStats shardStats = client().admin().indices().prepareStats("index").get().getIndex("index").getShards()[0]; long globalCheckPoint = shardStats.getSeqNoStats().getGlobalCheckpoint(); @@ -75,9 +76,9 @@ public void testGetOperationsBasedOnGlobalSequenceId() throws Exception { assertThat(operation.seqNo(), equalTo(2L)); assertThat(operation.id(), equalTo("3")); - prepareIndex("index").setId("3").setSource("{}", XContentType.JSON).get(); - prepareIndex("index").setId("4").setSource("{}", XContentType.JSON).get(); - prepareIndex("index").setId("5").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("index", "3"); + indexEmptyDoc("index", "4"); + indexEmptyDoc("index", "5"); shardStats = client().admin().indices().prepareStats("index").get().getIndex("index").getShards()[0]; globalCheckPoint = shardStats.getSeqNoStats().getGlobalCheckpoint(); @@ -110,7 +111,7 @@ public void testMissingOperations() throws Exception { .get(); for (int i = 0; i < 32; i++) { - prepareIndex("index").setId("1").setSource("{}", XContentType.JSON).get(); + indexEmptyDoc("index", "1"); client().prepareDelete("index", "1").get(); client().admin().indices().flush(new FlushRequest("index").force(true)).actionGet(); } @@ -196,4 +197,13 @@ public void testMissingOperations() throws Exception { } } + private void indexEmptyDoc(String index, String id) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setId(id).setSource("{}", XContentType.JSON); + try { + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + } diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java index 9d3821d64626f..57126e0b3d811 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java @@ -172,10 +172,10 @@ public void testSnapshotWithDanglingLocalSegment() throws Exception { final String indexName = "test-idx"; createIndex(indexName); - prepareIndex(indexName).setSource("foo", "bar").get(); + indexDoc(indexName, null, "foo", "bar"); assertSuccessful(startFullSnapshot(repo, "snapshot-1")); - prepareIndex(indexName).setSource("foo", "baz").get(); + indexDoc(indexName, null, "foo", "baz"); assertSuccessful(startFullSnapshot(repo, "snapshot-2")); logger.info("--> randomly deleting files from the local _snapshot path to simulate corruption"); @@ -341,6 +341,9 @@ private IndexRequestBuilder[] snapshotAndRestore(final String sourceIdx, final b builders[i] = prepareIndex(sourceIdx).setId(Integer.toString(i)).setSource(source).setRouting("r" + i); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } flushAndRefresh(); assertHitCount(prepareSearch(sourceIdx).setQuery(QueryBuilders.idsQuery().addIds("0")), 1); diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java index 3e5be8b4ae2ff..350a1c188e0a3 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java @@ -369,9 +369,9 @@ public void testDataTierTelemetry() { indicesAdmin().prepareCreate(index + "2").setSettings(indexSettings(1, 1)).setWaitForActiveShards(0).get(); ensureGreen(); - prepareIndex(index).setSource("foo", "bar").get(); - prepareIndex(index + "2").setSource("foo", "bar").get(); - prepareIndex(index + "2").setSource("foo", "bar").get(); + indexDoc(index, null, "foo", "bar"); + indexDoc(index + "2", null, "foo", "bar"); + indexDoc(index + "2", null, "foo", "bar"); refresh(index, index + "2"); DataTiersFeatureSetUsage usage = getUsage(); diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java index 628e2c18de2f9..3671972b3e2c7 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java @@ -177,6 +177,9 @@ private void indexRandomData(String indexName) throws Exception { // a case where they are the same (using sync flush), index Random does all this goodness // already indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } } private String findNodeWithPrimaryShard(String indexName, int shard) { diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/termsenum/CCSTermsEnumIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/termsenum/CCSTermsEnumIT.java index 157628be9fbc9..f88e59f0ec7d6 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/termsenum/CCSTermsEnumIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/termsenum/CCSTermsEnumIT.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.core.termsenum; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; @@ -50,15 +51,15 @@ public void testBasic() { final Client remoteClient = client("remote_cluster"); String localIndex = "local_test"; assertAcked(localClient.admin().indices().prepareCreate(localIndex).setSettings(indexSettings)); - localClient.prepareIndex(localIndex).setSource("foo", "foo").get(); - localClient.prepareIndex(localIndex).setSource("foo", "foobar").get(); + indexDoc(localClient, localIndex, "foo", "foo"); + indexDoc(localClient, localIndex, "foo", "foobar"); localClient.admin().indices().prepareRefresh(localIndex).get(); String remoteIndex = "remote_test"; assertAcked(remoteClient.admin().indices().prepareCreate(remoteIndex).setSettings(indexSettings)); - remoteClient.prepareIndex(remoteIndex).setSource("foo", "bar").get(); - remoteClient.prepareIndex(remoteIndex).setSource("foo", "foobar").get(); - remoteClient.prepareIndex(remoteIndex).setSource("foo", "zar").get(); + indexDoc(remoteClient, remoteIndex, "foo", "bar"); + indexDoc(remoteClient, remoteIndex, "foo", "foobar"); + indexDoc(remoteClient, remoteIndex, "foo", "zar"); remoteClient.admin().indices().prepareRefresh(remoteIndex).get(); // _terms_enum on a remote cluster @@ -106,4 +107,13 @@ public void testBasic() { assertThat(response.getTerms().get(1), equalTo("foobar")); assertThat(response.getTerms().get(2), equalTo("zar")); } + + private void indexDoc(Client client, String index, Object... source) { + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); + try { + indexRequestBuilder.setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java index ce915b9a76613..46a0abe8c2060 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java @@ -266,7 +266,11 @@ private void indexResponse( } addResultFieldAndFinish(response, source); - clientWithOrigin.index(new IndexRequest(index).create(true).id(docId).source(buffer.bytes(), source.contentType()), listener); + IndexRequest indexRequest = new IndexRequest(index); + clientWithOrigin.index( + indexRequest.create(true).id(docId).source(buffer.bytes(), source.contentType()), + ActionListener.runAfter(listener, indexRequest::decRef) + ); } catch (Exception e) { listener.onFailure(e); } @@ -287,9 +291,16 @@ private void updateResponse( buffer = allocateBuffer(isFailure == false); final XContentBuilder source = XContentFactory.jsonBuilder(buffer).startObject().field(RESPONSE_HEADERS_FIELD, responseHeaders); addResultFieldAndFinish(response, source); + UpdateRequest updateRequest = new UpdateRequest().index(index); + try { + updateRequest.id(docId).doc(buffer.bytes(), source.contentType()).retryOnConflict(5); + } catch (Exception e) { + updateRequest.decRef(); + throw e; + } clientWithOrigin.update( - new UpdateRequest().index(index).id(docId).doc(buffer.bytes(), source.contentType()).retryOnConflict(5), - ActionListener.runBefore(listener, buffer::close) + updateRequest, + ActionListener.runAfter(ActionListener.runBefore(listener, buffer::close), updateRequest::decRef) ); } catch (Exception e) { // release buffer right away to save memory, particularly in case the exception came from the circuit breaker @@ -348,8 +359,14 @@ private void addResultFieldAndFinish(Writeable response, XContentBuilder source) */ public void updateExpirationTime(String docId, long expirationTimeMillis, ActionListener listener) { Map source = Collections.singletonMap(EXPIRATION_TIME_FIELD, expirationTimeMillis); - UpdateRequest request = new UpdateRequest().index(index).id(docId).doc(source, XContentType.JSON).retryOnConflict(5); - clientWithOrigin.update(request, listener); + UpdateRequest request = new UpdateRequest().index(index); + try { + request.id(docId).doc(source, XContentType.JSON).retryOnConflict(5); + } catch (Exception e) { + request.decRef(); + throw e; + } + clientWithOrigin.update(request, ActionListener.runAfter(listener, request::decRef)); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditor.java index ca057188a3038..370a5753278c8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditor.java @@ -181,13 +181,25 @@ protected void indexDoc(ToXContent toXContent) { } private void writeDoc(ToXContent toXContent) { - client.index(indexRequest(toXContent), ActionListener.wrap(AbstractAuditor::onIndexResponse, AbstractAuditor::onIndexFailure)); + IndexRequest indexRequest = indexRequest(toXContent); + client.index( + indexRequest, + ActionListener.runAfter( + ActionListener.wrap(AbstractAuditor::onIndexResponse, AbstractAuditor::onIndexFailure), + indexRequest::decRef + ) + ); } private IndexRequest indexRequest(ToXContent toXContent) { IndexRequest indexRequest = new IndexRequest(auditIndex); - indexRequest.source(toXContentBuilder(toXContent)); - indexRequest.timeout(TimeValue.timeValueSeconds(5)); + try { + indexRequest.source(toXContentBuilder(toXContent)); + indexRequest.timeout(TimeValue.timeValueSeconds(5)); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } return indexRequest; } @@ -213,7 +225,12 @@ protected void writeBacklog() { BulkRequest bulkRequest = new BulkRequest(); ToXContent doc = backlog.poll(); while (doc != null) { - bulkRequest.add(indexRequest(doc)); + IndexRequest indexRequest = indexRequest(doc); + try { + bulkRequest.add(indexRequest); + } finally { + indexRequest.decRef(); + } doc = backlog.poll(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java index 1e25a6436bd04..a496efbfc2fa6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java @@ -523,7 +523,15 @@ private void onSearchResponse(SearchResponse searchResponse) { final BulkRequest bulkRequest = new BulkRequest(); try { - iterationResult.getToIndex().forEach(bulkRequest::add); + iterationResult.getToIndex().forEach(indexRequest -> { + bulkRequest.add(indexRequest); + /* + * Ordinarily the object that created the indexRequest would be responsible for this decRef call, but since no single + * objects owns the full lifecycle of this index request, and since it is only used right here, we're calling decRef + * here instead of adding more complexity to IterationResult or AsyncTwoPhaseIndexer. + */ + indexRequest.decRef(); + }); stats.markEndProcessing(); // an iteration result might return an empty set of documents to be indexed diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java index 794f5e1f66ee5..e132a0a9622f9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java @@ -104,6 +104,7 @@ public void testInfo() throws IOException { assertThat(indexRequest.indices(), arrayContaining(TEST_INDEX)); assertThat(indexRequest.timeout(), equalTo(TimeValue.timeValueSeconds(5))); AbstractAuditMessageTests.TestAuditMessage auditMessage = parseAuditMessage(indexRequest.source()); + indexRequest.decRef(); assertThat(auditMessage.getResourceId(), equalTo("foo")); assertThat(auditMessage.getMessage(), equalTo("Here is my info")); assertThat(auditMessage.getLevel(), equalTo(Level.INFO)); @@ -123,6 +124,7 @@ public void testWarning() throws IOException { assertThat(indexRequest.indices(), arrayContaining(TEST_INDEX)); assertThat(indexRequest.timeout(), equalTo(TimeValue.timeValueSeconds(5))); AbstractAuditMessageTests.TestAuditMessage auditMessage = parseAuditMessage(indexRequest.source()); + indexRequest.decRef(); assertThat(auditMessage.getResourceId(), equalTo("bar")); assertThat(auditMessage.getMessage(), equalTo("Here is my warning")); assertThat(auditMessage.getLevel(), equalTo(Level.WARNING)); @@ -142,6 +144,7 @@ public void testError() throws IOException { assertThat(indexRequest.indices(), arrayContaining(TEST_INDEX)); assertThat(indexRequest.timeout(), equalTo(TimeValue.timeValueSeconds(5))); AbstractAuditMessageTests.TestAuditMessage auditMessage = parseAuditMessage(indexRequest.source()); + indexRequest.decRef(); assertThat(auditMessage.getResourceId(), equalTo("foobar")); assertThat(auditMessage.getMessage(), equalTo("Here is my error")); assertThat(auditMessage.getLevel(), equalTo(Level.ERROR)); @@ -163,6 +166,7 @@ public void testAudit() throws IOException { assertThat(indexRequest.indices(), arrayContaining(TEST_INDEX)); assertThat(indexRequest.timeout(), equalTo(TimeValue.timeValueSeconds(5))); AbstractAuditMessageTests.TestAuditMessage auditMessage = parseAuditMessage(indexRequest.source()); + indexRequest.decRef(); assertThat(auditMessage.getResourceId(), equalTo("r_id")); assertThat(auditMessage.getMessage(), equalTo("Here is my audit")); assertThat(auditMessage.getLevel(), equalTo(level)); @@ -195,7 +199,10 @@ public void testAuditingBeforeTemplateInstalled() throws Exception { assertThat(bulkRequest.numberOfActions(), equalTo(3)); auditor.info("foobar", "Here is another message"); - verify(client, times(1)).execute(eq(TransportIndexAction.TYPE), any(), any()); + + verify(client, times(1)).execute(eq(TransportIndexAction.TYPE), indexRequestCaptor.capture(), any()); + IndexRequest indexRequest = indexRequestCaptor.getValue(); + indexRequest.decRef(); } public void testMaxBufferSize() throws Exception { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java index 76b668a87cff5..37b766142b347 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java @@ -225,7 +225,7 @@ protected IterationResult doProcess(SearchResponse searchResponse) { ++processOps; if (processOps == 5) { - return new IterationResult<>(Stream.of(new IndexRequest()), processOps, true); + return new IterationResult<>(Stream.empty(), processOps, true); } else if (processOps % 2 == 0) { return new IterationResult<>(Stream.empty(), processOps, false); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumTests.java index 632d7e9277e08..5a7e953ee7ba8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core.termsenum; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; @@ -109,10 +110,11 @@ private void checkTermsEnumKeywords(String indexed) throws Exception { } private void indexAndRefresh(String indexName, String id, String field, String value) throws IOException { - prepareIndex(indexName).setId(id) + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId(id) .setSource(jsonBuilder().startObject().field(field, value).endObject()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } public void testTermsEnumIPBasic() throws Exception { @@ -204,11 +206,11 @@ public void testTermsEnumIPRandomized() throws Exception { try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(indexName)) { for (int i = 0; i < numDocs; i++) { randomIps[i] = randomIp(randomBoolean()); - bulkRequestBuilder.add( - prepareIndex(indexName).setSource( - jsonBuilder().startObject().field("ip_addr", NetworkAddress.format(randomIps[i])).endObject() - ) + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setSource( + jsonBuilder().startObject().field("ip_addr", NetworkAddress.format(randomIps[i])).endObject() ); + bulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } assertNoFailures(bulkRequestBuilder.get()); } diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java index 57024acee809f..41b24fa22fa56 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java @@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.is; public class DataStreamLifecycleDownsampleIT extends ESIntegTestCase { - public static final int DOC_COUNT = 50_000; + public static final int DOC_COUNT = 5_000; // TODO put this back to 50k!! @Override protected Collection> nodePlugins() { diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java index c0b0778e105cf..c53f64c1f14ec 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java @@ -187,9 +187,13 @@ private static int bulkIndex(Client client, String dataStreamName, Supplier documentsJson) { try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { - documentsJson.forEach(document -> bulkRequestBuilder.add(new IndexRequest(indexName).source(document, XContentType.JSON))); + documentsJson.forEach(document -> { + IndexRequest indexRequest = new IndexRequest(indexName).source(document, XContentType.JSON); + bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); + }); assertFalse(bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get().hasFailures()); } } diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java index 499bd132f18d2..4cddb1b251460 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java @@ -259,6 +259,7 @@ private int bulkIndex(final String indexName, final SourceSupplier sourceSupplie XContentBuilder source = sourceSupplier.get(); indexRequest.source(source); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); int duplicates = 0; diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index f74bd299916c1..b97b5d38f192a 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -435,13 +435,17 @@ public void collect(int docId, long owningBucketOrd) throws IOException { private void indexBucket(XContentBuilder doc) { IndexRequestBuilder request = client.prepareIndex(downsampleIndex); - request.setSource(doc); - if (logger.isTraceEnabled()) { - logger.trace("Indexing downsample doc: [{}]", Strings.toString(doc)); + try { + request.setSource(doc); + if (logger.isTraceEnabled()) { + logger.trace("Indexing downsample doc: [{}]", Strings.toString(doc)); + } + IndexRequest indexRequest = request.request(); + task.setLastIndexingTimestamp(System.currentTimeMillis()); + bulkProcessor.addWithBackpressure(indexRequest, () -> abort); + } finally { + request.request().decRef(); } - IndexRequest indexRequest = request.request(); - task.setLastIndexingTimestamp(System.currentTimeMillis()); - bulkProcessor.addWithBackpressure(indexRequest, () -> abort); } @Override diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index ce38e14d65a4c..bdeb688f57c7c 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -1012,10 +1012,12 @@ private void bulkIndex(SourceSupplier sourceSupplier) throws IOException { } private void bulkIndex(final String indexName, final SourceSupplier sourceSupplier, int docCount) throws IOException { + List indexRequests = new ArrayList<>(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (int i = 0; i < docCount; i++) { IndexRequest indexRequest = new IndexRequest(indexName).opType(DocWriteRequest.OpType.CREATE); + indexRequests.add(indexRequest); XContentBuilder source = sourceSupplier.get(); indexRequest.source(source); bulkRequestBuilder.add(indexRequest); @@ -1038,6 +1040,9 @@ private void bulkIndex(final String indexName, final SourceSupplier sourceSuppli logger.info("Indexed [{}] documents. Dropped [{}] duplicates.", docsIndexed, duplicates); assertHitCount(client().prepareSearch(indexName).setSize(0), docsIndexed); } + for (IndexRequest indexRequest : indexRequests) { + indexRequest.decRef(); + } } private void prepareSourceIndex(final String sourceIndex, boolean blockWrite) { diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java index 616092af2d37b..8dbdbee719bf9 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -216,25 +217,26 @@ private void putComposableIndexTemplate(final String id, final List patt } private void indexDocs(final String dataStream, int numDocs, long startTime) { + List indexRequests = new ArrayList<>(); try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < numDocs; i++) { final String timestamp = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(startTime + i); - bulkRequest.add( - new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) - .source( - String.format( - Locale.ROOT, - "{\"%s\":\"%s\",\"%s\":\"%s\",\"%s\":\"%s\"}", - DEFAULT_TIMESTAMP_FIELD, - timestamp, - "routing_field", - 0, - "counter", - i + 1 - ), - XContentType.JSON - ) - ); + IndexRequest indexRequest = new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) + .source( + String.format( + Locale.ROOT, + "{\"%s\":\"%s\",\"%s\":\"%s\",\"%s\":\"%s\"}", + DEFAULT_TIMESTAMP_FIELD, + timestamp, + "routing_field", + 0, + "counter", + i + 1 + ), + XContentType.JSON + ); + indexRequests.add(indexRequest); + bulkRequest.add(indexRequest); } final BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); final BulkItemResponse[] items = bulkResponse.getItems(); @@ -243,5 +245,8 @@ private void indexDocs(final String dataStream, int numDocs, long startTime) { final BroadcastResponse refreshResponse = indicesAdmin().refresh(new RefreshRequest(dataStream)).actionGet(); assertThat(refreshResponse.getStatus().getStatus(), equalTo(RestStatus.OK.getStatus())); } + for (IndexRequest indexRequest : indexRequests) { + indexRequest.decRef(); + } } } diff --git a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java index 3729824656101..eeb1fd398098e 100644 --- a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java +++ b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java @@ -246,6 +246,7 @@ private static void enrich(Map> keys, String coordinatingNo indexRequest.setPipeline(pipeline); indexRequest.source(Map.of(MATCH_FIELD, randomFrom(keys.get(pipeline)))); bulkRequest.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = client(coordinatingNode).bulk(bulkRequest).actionGet(); assertThat("Expected no failure, but " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures(), is(false)); @@ -306,6 +307,7 @@ private static List createSourceIndex(String indexName, int numDocs) { ) ); client().index(indexRequest).actionGet(); + indexRequest.decRef(); } indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); return List.copyOf(keys); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java index 48b9e94cb71e9..3b2e6259505ff 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java @@ -478,7 +478,7 @@ private void transferDataToEnrichIndex(final String destinationIndexName) { reindexRequest.getDestination().routing("discard"); reindexRequest.getDestination().setPipeline(EnrichPolicyReindexPipeline.pipelineName()); - client.execute(EnrichReindexAction.INSTANCE, reindexRequest, new DelegatingActionListener<>(listener) { + client.execute(EnrichReindexAction.INSTANCE, reindexRequest, ActionListener.runAfter(new DelegatingActionListener<>(listener) { @Override public void onResponse(BulkByScrollResponse bulkByScrollResponse) { // Do we want to fail the request if there were failures during the reindex process? @@ -533,7 +533,7 @@ public void onResponse(BulkByScrollResponse bulkByScrollResponse) { forceMergeEnrichIndex(destinationIndexName, 1); } } - }); + }, reindexRequest::decRef)); } private void forceMergeEnrichIndex(final String destinationIndexName, final int attempt) { diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java index 702736a8ff95d..401b1e16ba7da 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java @@ -121,6 +121,7 @@ public void testIngestDataWithMatchProcessor() { indexRequest.setPipeline(pipelineName); indexRequest.source(Map.of(MATCH_FIELD, keys.get(i))); bulkRequest.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat("Expected no failure, but " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures(), is(false)); @@ -176,6 +177,7 @@ public void testIngestDataWithGeoMatchProcessor() { ) ); client().index(indexRequest).actionGet(); + indexRequest.decRef(); client().admin().indices().refresh(new RefreshRequest(SOURCE_INDEX_NAME)).actionGet(); } @@ -214,6 +216,7 @@ public void testIngestDataWithGeoMatchProcessor() { indexRequest.setPipeline(pipelineName); indexRequest.source(Map.of(matchField, "37.386444, -122.083863")); // point within match boundary bulkRequest.add(indexRequest); + indexRequest.decRef(); BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat("Expected no failure, but " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures(), is(false)); assertThat(bulkResponse.getItems().length, equalTo(1)); @@ -245,6 +248,7 @@ public void testMultiplePolicies() { IndexRequest indexRequest = new IndexRequest("source-" + i); indexRequest.source("key", "key", "value", "val" + i); client().index(indexRequest).actionGet(); + indexRequest.decRef(); client().admin().indices().refresh(new RefreshRequest("source-" + i)).actionGet(); EnrichPolicy enrichPolicy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of("source-" + i), "key", List.of("value")); @@ -268,6 +272,7 @@ public void testMultiplePolicies() { indexRequest.setPipeline("pipeline" + i); indexRequest.source(Map.of("key", "key")); bulkRequest.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat("Expected no failure, but " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures(), is(false)); @@ -289,6 +294,7 @@ public void testAsyncTaskExecute() throws Exception { IndexRequest indexRequest = new IndexRequest(sourceIndexName); indexRequest.source("key", "key", "value", "val1"); client().index(indexRequest).actionGet(); + indexRequest.decRef(); client().admin().indices().refresh(new RefreshRequest(sourceIndexName)).actionGet(); } @@ -327,6 +333,7 @@ public void testAsyncTaskExecute() throws Exception { indexRequest.setPipeline(pipelineName); indexRequest.source(Map.of("key", "key")); bulkRequest.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat("Expected no failure, but " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures(), is(false)); @@ -367,6 +374,7 @@ public void testTemplating() throws Exception { .setPipeline(pipelineName) .source(Map.of("indirection1", MATCH_FIELD, "indirection2", "users", MATCH_FIELD, keys.get(0))); client().index(indexRequest).get(); + indexRequest.decRef(); GetResponse getResponse = client().get(new GetRequest("my-index", "1")).actionGet(); Map source = getResponse.getSourceAsMap(); Map userEntry = (Map) source.get("users"); @@ -406,8 +414,12 @@ public void testFailureAfterEnrich() throws Exception { IndexRequest indexRequest = new IndexRequest("my-index").id("1") .setPipeline(pipelineName) .source(Map.of(MATCH_FIELD, "non_existing")); - Exception e = expectThrows(IllegalArgumentException.class, client().index(indexRequest)); - assertThat(e.getMessage(), equalTo("field [users] not present as part of path [users]")); + try { + Exception e = expectThrows(IllegalArgumentException.class, client().index(indexRequest)); + assertThat(e.getMessage(), equalTo("field [users] not present as part of path [users]")); + } finally { + indexRequest.decRef(); + } } } @@ -425,6 +437,7 @@ private List createSourceMatchIndex(int numKeys, int numDocsPerKey) { Map.of(MATCH_FIELD, key, DECORATE_FIELDS[0], key + "0", DECORATE_FIELDS[1], key + "1", DECORATE_FIELDS[2], key + "2") ); client().index(indexRequest).actionGet(); + indexRequest.decRef(); } } client().admin().indices().refresh(new RefreshRequest(SOURCE_INDEX_NAME)).actionGet(); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java index 13e1df133f00b..49fa472c749c4 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.WriteRequest; @@ -116,15 +117,17 @@ public static void afterClass() { public void testRunner() throws Exception { final String sourceIndex = "source-index"; - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" { "field1": "value1", "field2": 2, "field3": "ignored", "field4": "ignored", "field5": "value5" - }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -205,10 +208,11 @@ public void testRunner() throws Exception { public void testRunnerGeoMatchType() throws Exception { final String sourceIndex = "source-index"; - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" - {"location":"POINT(10.0 10.0)","zipcode":90210}""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) - .actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" + {"location":"POINT(10.0 10.0)","zipcode":90210}""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -296,10 +300,11 @@ public void testRunnerDoubleRangeMatchType() throws Exception { private void testNumberRangeMatchType(String rangeType) throws Exception { final String sourceIndex = "source-index"; createIndex(sourceIndex, Settings.EMPTY, "_doc", "range", "type=" + rangeType + "_range"); - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" - {"range":{"gt":1,"lt":10},"zipcode":90210}""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) - .actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" + {"range":{"gt":1,"lt":10},"zipcode":90210}""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -383,10 +388,11 @@ private GetIndexResponse getGetIndexResponseAndCheck(String createdEnrichIndex) public void testRunnerRangeTypeWithIpRange() throws Exception { final String sourceIndexName = "source-index"; createIndex(sourceIndexName, Settings.EMPTY, "_doc", "subnet", "type=ip_range"); - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndexName).id("id").source(""" - {"subnet":"10.0.0.0/8","department":"research"}""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) - .actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + IndexRequest indexRequest = new IndexRequest().index(sourceIndexName).id("id").source(""" + {"subnet":"10.0.0.0/8","department":"research"}""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); GetIndexResponse sourceIndex = indicesAdmin().getIndex(new GetIndexRequest().indices(sourceIndexName)).actionGet(); // Validate Mapping @@ -473,19 +479,19 @@ public void testRunnerMultiSource() throws Exception { int numberOfSourceIndices = 3; for (int idx = 0; idx < numberOfSourceIndices; idx++) { final String sourceIndex = baseSourceName + idx; - DocWriteResponse indexRequest = client().index( - new IndexRequest().index(sourceIndex).id(randomAlphaOfLength(10)).source(Strings.format(""" - { - "idx": %s, - "key": "key%s", - "field1": "value1", - "field2": 2, - "field3": "ignored", - "field4": "ignored", - "field5": "value5" - }""", idx, idx), XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - ).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id(randomAlphaOfLength(10)).source(Strings.format(""" + { + "idx": %s, + "key": "key%s", + "field1": "value1", + "field2": 2, + "field3": "ignored", + "field4": "ignored", + "field5": "value5" + }""", idx, idx), XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); final int targetIdx = idx; assertResponse( client().search( @@ -584,8 +590,10 @@ public void testRunnerMultiSourceDocIdCollisions() throws Exception { String collidingDocId = randomAlphaOfLength(10); for (int idx = 0; idx < numberOfSourceIndices; idx++) { final String sourceIndex = baseSourceName + idx; - DocWriteResponse indexRequest = client().index( - new IndexRequest().index(sourceIndex).id(collidingDocId).routing(collidingDocId + idx).source(Strings.format(""" + IndexRequest indexRequest = new IndexRequest().index(sourceIndex) + .id(collidingDocId) + .routing(collidingDocId + idx) + .source(Strings.format(""" { "idx": %s, "key": "key%s", @@ -594,9 +602,11 @@ public void testRunnerMultiSourceDocIdCollisions() throws Exception { "field3": "ignored", "field4": "ignored", "field5": "value5" - }""", idx, idx), XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - ).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + }""", idx, idx), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); final int targetIdx = idx; assertResponse( client().search( @@ -714,19 +724,19 @@ public void testRunnerMultiSourceEnrichKeyCollisions() throws Exception { int numberOfSourceIndices = 3; for (int idx = 0; idx < numberOfSourceIndices; idx++) { final String sourceIndex = baseSourceName + idx; - DocWriteResponse indexRequest = client().index( - new IndexRequest().index(sourceIndex).id(randomAlphaOfLength(10)).source(Strings.format(""" - { - "idx": %s, - "key": "key", - "field1": "value1", - "field2": 2, - "field3": "ignored", - "field4": "ignored", - "field5": "value5" - }""", idx), XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - ).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id(randomAlphaOfLength(10)).source(Strings.format(""" + { + "idx": %s, + "key": "key", + "field1": "value1", + "field2": 2, + "field3": "ignored", + "field4": "ignored", + "field5": "value5" + }""", idx), XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); final int targetIdx = idx; assertResponse( @@ -1032,14 +1042,14 @@ public void testRunnerObjectSourceMapping() throws Exception { CreateIndexResponse createResponse = indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mappingBuilder)).actionGet(); assertTrue(createResponse.isAcknowledged()); - DocWriteResponse indexRequest = client().index( - new IndexRequest().index(sourceIndex) - .id("id") - .source(""" - {"data":{"field1":"value1","field2":2,"field3":"ignored"}}""", XContentType.JSON) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - ).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + IndexRequest indexRequest = new IndexRequest().index(sourceIndex) + .id("id") + .source(""" + {"data":{"field1":"value1","field2":2,"field3":"ignored"}}""", XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -1147,14 +1157,14 @@ public void testRunnerExplicitObjectSourceMapping() throws Exception { CreateIndexResponse createResponse = indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mappingBuilder)).actionGet(); assertTrue(createResponse.isAcknowledged()); - DocWriteResponse indexRequest = client().index( - new IndexRequest().index(sourceIndex) - .id("id") - .source(""" - {"data":{"field1":"value1","field2":2,"field3":"ignored"}}""", XContentType.JSON) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - ).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + IndexRequest indexRequest = new IndexRequest().index(sourceIndex) + .id("id") + .source(""" + {"data":{"field1":"value1","field2":2,"field3":"ignored"}}""", XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -1260,15 +1270,17 @@ public void testRunnerExplicitObjectSourceMappingRangePolicy() throws Exception CreateIndexResponse createResponse = indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mappingBuilder)).actionGet(); assertTrue(createResponse.isAcknowledged()); - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" { "data": { "subnet": "10.0.0.0/8", "department": "research", "field3": "ignored" } - }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -1380,7 +1392,7 @@ public void testRunnerTwoObjectLevelsSourceMapping() throws Exception { CreateIndexResponse createResponse = indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mappingBuilder)).actionGet(); assertTrue(createResponse.isAcknowledged()); - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" { "data": { "fields": { @@ -1389,8 +1401,10 @@ public void testRunnerTwoObjectLevelsSourceMapping() throws Exception { "field3": "ignored" } } - }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -1509,7 +1523,7 @@ public void testRunnerTwoObjectLevelsSourceMappingRangePolicy() throws Exception CreateIndexResponse createResponse = indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mappingBuilder)).actionGet(); assertTrue(createResponse.isAcknowledged()); - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" { "data": { "fields": { @@ -1518,8 +1532,10 @@ public void testRunnerTwoObjectLevelsSourceMappingRangePolicy() throws Exception "field3": "ignored" } } - }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -1637,7 +1653,7 @@ public void testRunnerTwoObjectLevelsSourceMappingDateRangeWithFormat() throws E CreateIndexResponse createResponse = indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mappingBuilder)).actionGet(); assertTrue(createResponse.isAcknowledged()); - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" { "data": { "fields": { @@ -1649,8 +1665,10 @@ public void testRunnerTwoObjectLevelsSourceMappingDateRangeWithFormat() throws E "field3": "ignored" } } - }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -1788,14 +1806,14 @@ public void testRunnerDottedKeyNameSourceMapping() throws Exception { CreateIndexResponse createResponse = indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mappingBuilder)).actionGet(); assertTrue(createResponse.isAcknowledged()); - DocWriteResponse indexRequest = client().index( - new IndexRequest().index(sourceIndex) - .id("id") - .source(""" - {"data.field1":"value1","data.field2":2,"data.field3":"ignored"}""", XContentType.JSON) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - ).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + IndexRequest indexRequest = new IndexRequest().index(sourceIndex) + .id("id") + .source(""" + {"data.field1":"value1","data.field2":2,"data.field3":"ignored"}""", XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -1873,15 +1891,17 @@ public void testRunnerDottedKeyNameSourceMapping() throws Exception { public void testRunnerWithForceMergeRetry() throws Exception { final String sourceIndex = "source-index"; - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" { "field1": "value1", "field2": 2, "field3": "ignored", "field4": "ignored", "field5": "value5" - }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -1972,12 +1992,12 @@ protected void ensureSingleSegment(String destinationIndexName, int attempt) { if (attempt == 1) { // Put and flush a document to increase the number of segments, simulating not // all segments were merged on the first try. - DocWriteResponse indexRequest = client().index( - new IndexRequest().index(createdEnrichIndex) - .source(unmergedDocument) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - ).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + IndexRequest indexRequest1 = new IndexRequest().index(createdEnrichIndex) + .source(unmergedDocument) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest1).actionGet(); + indexRequest1.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); } super.ensureSingleSegment(destinationIndexName, attempt); } @@ -2050,15 +2070,17 @@ protected void ensureSingleSegment(String destinationIndexName, int attempt) { public void testRunnerCancel() throws Exception { final String sourceIndex = "source-index"; - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" { "field1": "value1", "field2": 2, "field3": "ignored", "field4": "ignored", "field5": "value5" - }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); List enrichFields = List.of("field2", "field5"); EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of(sourceIndex), "field1", enrichFields); @@ -2151,23 +2173,28 @@ public void testRunRangePolicyWithObjectFieldAsMatchField() throws Exception { } public void testEnrichFieldsConflictMappingTypes() throws Exception { - createIndex("source-1", Settings.EMPTY, "_doc", "user", "type=keyword", "name", "type=text", "zipcode", "type=long"); - prepareIndex("source-1").setSource("user", "u1", "name", "n", "zipcode", 90000) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); - createIndex("source-2", Settings.EMPTY, "_doc", "user", "type=keyword", "zipcode", "type=long"); - - prepareIndex("source-2").setSource(""" - { - "user": "u2", - "name": { - "first": "f", - "last": "l" - }, - "zipcode": 90001 - } - """, XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); - + { + createIndex("source-1", Settings.EMPTY, "_doc", "user", "type=keyword", "name", "type=text", "zipcode", "type=long"); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source-1").setSource("user", "u1", "name", "n", "zipcode", 90000) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } + { + createIndex("source-2", Settings.EMPTY, "_doc", "user", "type=keyword", "zipcode", "type=long"); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source-2").setSource(""" + { + "user": "u2", + "name": { + "first": "f", + "last": "l" + }, + "zipcode": 90001 + } + """, XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } EnrichPolicy policy = new EnrichPolicy( EnrichPolicy.MATCH_TYPE, null, @@ -2209,16 +2236,24 @@ public void testEnrichFieldsConflictMappingTypes() throws Exception { } public void testEnrichMappingConflictFormats() throws ExecutionException, InterruptedException { - createIndex("source-1", Settings.EMPTY, "_doc", "user", "type=keyword", "date", "type=date,format=yyyy"); - prepareIndex("source-1").setSource("user", "u1", "date", "2023").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); - createIndex("source-2", Settings.EMPTY, "_doc", "user", "type=keyword", "date", "type=date,format=yyyy-MM"); - - prepareIndex("source-2").setSource(""" - { - "user": "u2", - "date": "2023-05" - } - """, XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + { + createIndex("source-1", Settings.EMPTY, "_doc", "user", "type=keyword", "date", "type=date,format=yyyy"); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source-1").setSource("user", "u1", "date", "2023") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } + { + createIndex("source-2", Settings.EMPTY, "_doc", "user", "type=keyword", "date", "type=date,format=yyyy-MM"); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source-2").setSource(""" + { + "user": "u2", + "date": "2023-05" + } + """, XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); + } EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of("source-1", "source-2"), "user", List.of("date")); String policyName = "test1"; @@ -2252,9 +2287,10 @@ public void testEnrichMappingConflictFormats() throws ExecutionException, Interr public void testEnrichObjectField() throws ExecutionException, InterruptedException { createIndex("source-1", Settings.EMPTY, "_doc", "id", "type=keyword", "name.first", "type=keyword", "name.last", "type=keyword"); - prepareIndex("source-1").setSource("user", "u1", "name.first", "F1", "name.last", "L1") - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("source-1").setSource("user", "u1", "name.first", "F1", "name.last", "L1") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of("source-1"), "user", List.of("name")); String policyName = "test1"; final long createTime = randomNonNegativeLong(); @@ -2346,15 +2382,17 @@ public void testEnrichNestedField() throws Exception { public void testRunnerValidatesIndexIntegrity() throws Exception { final String sourceIndex = "source-index"; - DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source(""" + IndexRequest indexRequest = new IndexRequest().index(sourceIndex).id("id").source(""" { "field1": "value1", "field2": 2, "field3": "ignored", "field4": "ignored", "field5": "value5" - }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); - assertEquals(RestStatus.CREATED, indexRequest.status()); + }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + DocWriteResponse indexResponse = client().index(indexRequest).actionGet(); + indexRequest.decRef(); + assertEquals(RestStatus.CREATED, indexResponse.status()); assertResponse( client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))), @@ -2523,14 +2561,11 @@ private void validateSegments(String createdEnrichIndex, int expectedDocs) { } private void ensureEnrichIndexIsReadOnly(String createdEnrichIndex) { - ElasticsearchException expected = expectThrows( - ElasticsearchException.class, - () -> client().index( - new IndexRequest().index(createdEnrichIndex) - .id(randomAlphaOfLength(10)) - .source(Map.of(randomAlphaOfLength(6), randomAlphaOfLength(10))) - ).actionGet() - ); + IndexRequest indexRequest = new IndexRequest().index(createdEnrichIndex) + .id(randomAlphaOfLength(10)) + .source(Map.of(randomAlphaOfLength(6), randomAlphaOfLength(10))); + ElasticsearchException expected = expectThrows(ElasticsearchException.class, () -> client().index(indexRequest).actionGet()); + indexRequest.decRef(); assertThat(expected.getMessage(), containsString("index [" + createdEnrichIndex + "] blocked by: [FORBIDDEN/8/index write (api)]")); } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java index 8c9d5fae5f269..443aef20328b9 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java @@ -68,12 +68,13 @@ public void testWriteThreadLivenessBackToBack() throws Exception { String enrichPolicyName = "enrich_policy_" + testSuffix; String enrichPipelineName = "enrich_pipeline_" + testSuffix; String enrichedIndexName = "enrich_results_" + testSuffix; - - client().index( - new IndexRequest(enrichIndexName).source( + { + IndexRequest indexRequest = new IndexRequest(enrichIndexName).source( JsonXContent.contentBuilder().startObject().field("my_key", "key").field("my_value", "data").endObject() - ) - ).actionGet(); + ); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + } client().admin().indices().refresh(new RefreshRequest(enrichIndexName)).actionGet(); @@ -134,7 +135,9 @@ public void testWriteThreadLivenessBackToBack() throws Exception { try (BulkRequest bulk = new BulkRequest(enrichedIndexName)) { bulk.timeout(new TimeValue(10, TimeUnit.SECONDS)); for (int idx = 0; idx < 50; idx++) { - bulk.add(new IndexRequest().source(doc).setPipeline(enrichPipelineName)); + IndexRequest indexRequest = new IndexRequest().source(doc).setPipeline(enrichPipelineName); + bulk.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkItemResponses = client().bulk(bulk).actionGet(new TimeValue(30, TimeUnit.SECONDS)); @@ -169,11 +172,13 @@ public void testWriteThreadLivenessWithPipeline() throws Exception { String enrichPipelineName1 = enrichPipelineName + "_1"; String enrichPipelineName2 = enrichPipelineName + "_2"; - client().index( - new IndexRequest(enrichIndexName).source( + { + IndexRequest indexRequest = new IndexRequest(enrichIndexName).source( JsonXContent.contentBuilder().startObject().field("my_key", "key").field("my_value", "data").endObject() - ) - ).actionGet(); + ); + client().index(indexRequest).actionGet(); + indexRequest.decRef(); + } client().admin().indices().refresh(new RefreshRequest(enrichIndexName)).actionGet(); @@ -258,7 +263,9 @@ public void testWriteThreadLivenessWithPipeline() throws Exception { try (BulkRequest bulk = new BulkRequest(enrichedIndexName)) { bulk.timeout(new TimeValue(10, TimeUnit.SECONDS)); for (int idx = 0; idx < 50; idx++) { - bulk.add(new IndexRequest().source(doc).setPipeline(enrichPipelineName1)); + IndexRequest indexRequest = new IndexRequest().source(doc).setPipeline(enrichPipelineName1); + bulk.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkItemResponses = client().bulk(bulk).actionGet(new TimeValue(30, TimeUnit.SECONDS)); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchActionTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchActionTests.java index 00f22aca2cb92..7a2eb3e3b4c0a 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchActionTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchActionTests.java @@ -45,6 +45,7 @@ public void testExecute() throws Exception { IndexRequest indexRequest = new IndexRequest(indexName); indexRequest.source(source); client().index(indexRequest).actionGet(); + indexRequest.decRef(); client().admin().indices().refresh(new RefreshRequest(indexName)).actionGet(); int numSearches = randomIntBetween(2, 32); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java index aa22dfbe2e3cb..7460c8cc1477a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java @@ -78,8 +78,11 @@ public void emitEvent( try { AnalyticsEvent event = eventFactory.fromRequest(request); IndexRequest eventIndexRequest = createIndexRequest(event); - - bulkProcessor.add(eventIndexRequest); + try { + bulkProcessor.add(eventIndexRequest); + } finally { + eventIndexRequest.decRef(); + } if (dropEvent.compareAndSet(true, false)) { logger.warn("Bulk processor has been flushed. Accepting new events again."); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 41451c76b90f8..fbb0f3fc6e146 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.application.connector.action.UpdateConnectorPipelineAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorSchedulingAction; +import java.io.IOException; import java.time.Instant; import java.util.Arrays; import java.util.Collections; @@ -48,6 +49,7 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiConsumer; +import java.util.function.Function; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; @@ -84,14 +86,15 @@ public void createConnectorWithDocId(PutConnectorAction.Request request, ActionL request.getName(), request.getServiceType() ); - + final IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME); try { - final IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + indexRequest.opType(DocWriteRequest.OpType.INDEX) .id(request.getConnectorId()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - clientWithOrigin.index(indexRequest, listener); + clientWithOrigin.index(indexRequest, ActionListener.runAfter(listener, indexRequest::decRef)); } catch (Exception e) { + indexRequest.decRef(); listener.onFailure(e); } } @@ -116,16 +119,23 @@ public void createConnectorWithAutoGeneratedId( request.getServiceType() ); + final IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME); try { - final IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + indexRequest.opType(DocWriteRequest.OpType.INDEX) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); clientWithOrigin.index( indexRequest, - listener.delegateFailureAndWrap((l, indexResponse) -> l.onResponse(new PostConnectorAction.Response(indexResponse.getId()))) + ActionListener.runAfter( + listener.delegateFailureAndWrap( + (l, indexResponse) -> l.onResponse(new PostConnectorAction.Response(indexResponse.getId())) + ), + indexRequest::decRef + ) ); } catch (Exception e) { + indexRequest.decRef(); listener.onFailure(e); } } @@ -274,29 +284,40 @@ public void onFailure(Exception e) { public void updateConnectorConfiguration(UpdateConnectorConfigurationAction.Request request, ActionListener listener) { try { String connectorId = request.getConnectorId(); - final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( - new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source( - Map.of( - Connector.CONFIGURATION_FIELD.getPreferredName(), - request.getConfiguration(), - Connector.STATUS_FIELD.getPreferredName(), - ConnectorStatus.CONFIGURED.toString() - ) - ) - ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorId)); - return; - } - l.onResponse(updateResponse); - }) - ); + IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME); + try { + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId); + try { + updateRequest.doc( + indexRequest.opType(DocWriteRequest.OpType.INDEX) + .id(connectorId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source( + Map.of( + Connector.CONFIGURATION_FIELD.getPreferredName(), + request.getConfiguration(), + Connector.STATUS_FIELD.getPreferredName(), + ConnectorStatus.CONFIGURED.toString() + ) + ) + ); + clientWithOrigin.update( + updateRequest, + ActionListener.runAfter(new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorId)); + return; + } + l.onResponse(updateResponse); + }), updateRequest::decRef) + ); + } catch (Exception e) { + updateRequest.decRef(); + throw e; + } + } finally { + indexRequest.decRef(); + } } catch (Exception e) { listener.onFailure(e); } @@ -309,27 +330,11 @@ public void updateConnectorConfiguration(UpdateConnectorConfigurationAction.Requ * @param listener The listener for handling responses, including successful updates or errors. */ public void updateConnectorError(UpdateConnectorErrorAction.Request request, ActionListener listener) { - try { - String connectorId = request.getConnectorId(); - final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( - new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(Map.of(Connector.ERROR_FIELD.getPreferredName(), request.getError())) - ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorId)); - return; - } - l.onResponse(updateResponse); - }) - ); - } catch (Exception e) { - listener.onFailure(e); - } + updateConfiguration( + request.getConnectorId(), + listener, + indexRequest -> indexRequest.source(Map.of(Connector.ERROR_FIELD.getPreferredName(), request.getError())) + ); } /** @@ -339,28 +344,13 @@ public void updateConnectorError(UpdateConnectorErrorAction.Request request, Act * @param listener The listener for handling responses, including successful updates or errors. */ public void updateConnectorNameOrDescription(UpdateConnectorNameAction.Request request, ActionListener listener) { - try { - String connectorId = request.getConnectorId(); - - final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( - new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) - ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorId)); - return; - } - l.onResponse(updateResponse); - }) - ); - } catch (Exception e) { - listener.onFailure(e); - } + updateConfiguration(request.getConnectorId(), listener, indexRequest -> { + try { + return indexRequest.source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } /** @@ -370,27 +360,11 @@ public void updateConnectorNameOrDescription(UpdateConnectorNameAction.Request r * @param listener Listener to respond to a successful response or an error. */ public void updateConnectorFiltering(UpdateConnectorFilteringAction.Request request, ActionListener listener) { - try { - String connectorId = request.getConnectorId(); - final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( - new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), request.getFiltering())) - ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorId)); - return; - } - l.onResponse(updateResponse); - }) - ); - } catch (Exception e) { - listener.onFailure(e); - } + updateConfiguration( + request.getConnectorId(), + listener, + indexRequest -> indexRequest.source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), request.getFiltering())) + ); } /** @@ -400,26 +374,11 @@ public void updateConnectorFiltering(UpdateConnectorFilteringAction.Request requ * @param listener The listener for handling responses, including successful updates or errors. */ public void checkInConnector(String connectorId, ActionListener listener) { - try { - final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( - new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(Map.of(Connector.LAST_SEEN_FIELD.getPreferredName(), Instant.now())) - ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorId)); - return; - } - l.onResponse(updateResponse); - }) - ); - } catch (Exception e) { - listener.onFailure(e); - } + updateConfiguration( + connectorId, + listener, + indexRequest -> indexRequest.source(Map.of(Connector.LAST_SEEN_FIELD.getPreferredName(), Instant.now())) + ); } /** @@ -429,27 +388,13 @@ public void checkInConnector(String connectorId, ActionListener * @param listener Listener to respond to a successful response or an error. */ public void updateConnectorLastSyncStats(UpdateConnectorLastSyncStatsAction.Request request, ActionListener listener) { - try { - String connectorId = request.getConnectorId(); - final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( - new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) - ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorId)); - return; - } - l.onResponse(updateResponse); - }) - ); - } catch (Exception e) { - listener.onFailure(e); - } + updateConfiguration(request.getConnectorId(), listener, indexRequest -> { + try { + return indexRequest.source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } /** @@ -459,28 +404,11 @@ public void updateConnectorLastSyncStats(UpdateConnectorLastSyncStatsAction.Requ * @param listener Listener to respond to a successful response or an error. */ public void updateConnectorPipeline(UpdateConnectorPipelineAction.Request request, ActionListener listener) { - try { - String connectorId = request.getConnectorId(); - final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( - new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(Map.of(Connector.PIPELINE_FIELD.getPreferredName(), request.getPipeline())) - .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) - ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorId)); - return; - } - l.onResponse(updateResponse); - }) - ); - } catch (Exception e) { - listener.onFailure(e); - } + updateConfiguration( + request.getConnectorId(), + listener, + indexRequest -> indexRequest.source(Map.of(Connector.PIPELINE_FIELD.getPreferredName(), request.getPipeline())) + ); } /** @@ -490,24 +418,50 @@ public void updateConnectorPipeline(UpdateConnectorPipelineAction.Request reques * @param listener The listener for handling responses, including successful updates or errors. */ public void updateConnectorScheduling(UpdateConnectorSchedulingAction.Request request, ActionListener listener) { + updateConfiguration( + request.getConnectorId(), + listener, + indexRequest -> indexRequest.source(Map.of(Connector.SCHEDULING_FIELD.getPreferredName(), request.getScheduling())) + ); + } + + /** + * Updates the {@link ConnectorIngestPipeline} property of a {@link Connector}. + * + * @param connectorId Used as the document id + * @param listener Listener to respond to a successful response or an error. + * @param sourceSetter A function that sets the source on the IndexRequest. + */ + private void updateConfiguration( + String connectorId, + ActionListener listener, + Function sourceSetter + ) { try { - String connectorId = request.getConnectorId(); - final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( - new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(Map.of(Connector.SCHEDULING_FIELD.getPreferredName(), request.getScheduling())) - ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorId)); - return; - } - l.onResponse(updateResponse); - }) - ); + IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME); + try { + indexRequest.opType(DocWriteRequest.OpType.INDEX).id(connectorId).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + sourceSetter.apply(indexRequest); + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId); + try { + updateRequest.doc(indexRequest); + clientWithOrigin.update( + updateRequest, + ActionListener.runAfter(new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorId)); + return; + } + l.onResponse(updateResponse); + }), updateRequest::decRef) + ); + } catch (Exception e) { + updateRequest.decRef(); + throw e; + } + } finally { + indexRequest.decRef(); + } } catch (Exception e) { listener.onFailure(e); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index ee35d8fb6372c..34567788e9331 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -96,10 +96,6 @@ public void createConnectorSyncJob( try { String syncJobId = generateId(); - final IndexRequest indexRequest = new IndexRequest(CONNECTOR_SYNC_JOB_INDEX_NAME).id(syncJobId) - .opType(DocWriteRequest.OpType.INDEX) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - ConnectorSyncJob syncJob = new ConnectorSyncJob.Builder().setId(syncJobId) .setJobType(jobType) .setTriggerMethod(triggerMethod) @@ -113,14 +109,27 @@ public void createConnectorSyncJob( .setDeletedDocumentCount(ZERO) .build(); - indexRequest.source(syncJob.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - - clientWithOrigin.index( - indexRequest, - l.delegateFailureAndWrap( - (ll, indexResponse) -> ll.onResponse(new PostConnectorSyncJobAction.Response(indexResponse.getId())) - ) - ); + final IndexRequest indexRequest = new IndexRequest(CONNECTOR_SYNC_JOB_INDEX_NAME); + try { + indexRequest.id(syncJobId) + .opType(DocWriteRequest.OpType.INDEX) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + indexRequest.source(syncJob.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); + + clientWithOrigin.index( + indexRequest, + ActionListener.runAfter( + l.delegateFailureAndWrap( + (ll, indexResponse) -> ll.onResponse(new PostConnectorSyncJobAction.Response(indexResponse.getId())) + ), + indexRequest::decRef + ) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } catch (IOException e) { l.onFailure(e); } @@ -173,15 +182,19 @@ public void checkInConnectorSyncJob(String connectorSyncJobId, ActionListener(connectorSyncJobId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorSyncJobId)); - return; - } - l.onResponse(updateResponse); - }) + ActionListener.runAfter( + new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorSyncJobId)); + return; + } + l.onResponse(updateResponse); + }), + updateRequest::decRef + ) ); } catch (Exception e) { + updateRequest.decRef(); listener.onFailure(e); } } @@ -246,15 +259,19 @@ public void cancelConnectorSyncJob(String connectorSyncJobId, ActionListener(connectorSyncJobId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorSyncJobId)); - return; - } - l.onResponse(updateResponse); - }) + ActionListener.runAfter( + new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorSyncJobId)); + return; + } + l.onResponse(updateResponse); + }), + updateRequest::decRef + ) ); } catch (Exception e) { + updateRequest.decRef(); listener.onFailure(e); } } @@ -390,15 +407,19 @@ public void updateConnectorSyncJobIngestionStats( try { clientWithOrigin.update( updateRequest, - new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(syncJobId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(syncJobId)); - return; - } - l.onResponse(updateResponse); - }) + ActionListener.runAfter( + new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(syncJobId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(syncJobId)); + return; + } + l.onResponse(updateResponse); + }), + updateRequest::decRef + ) ); } catch (Exception e) { + updateRequest.decRef(); listener.onFailure(e); } @@ -472,15 +493,19 @@ public void updateConnectorSyncJobError(String connectorSyncJobId, String error, try { clientWithOrigin.update( updateRequest, - new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorSyncJobId)); - return; - } - l.onResponse(updateResponse); - }) + ActionListener.runAfter( + new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorSyncJobId)); + return; + } + l.onResponse(updateResponse); + }), + updateRequest::decRef + ) ); } catch (Exception e) { + updateRequest.decRef(); listener.onFailure(e); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java index 4e001d38bf279..62000df551a45 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java @@ -232,15 +232,17 @@ private static List parseCriteria(List> r * @param listener The action listener to invoke on response/failure. */ public void putQueryRuleset(QueryRuleset queryRuleset, ActionListener listener) { + final IndexRequest indexRequest = new IndexRequest(QUERY_RULES_ALIAS_NAME); try { validateQueryRuleset(queryRuleset); - final IndexRequest indexRequest = new IndexRequest(QUERY_RULES_ALIAS_NAME).opType(DocWriteRequest.OpType.INDEX) + indexRequest.opType(DocWriteRequest.OpType.INDEX) .id(queryRuleset.id()) .opType(DocWriteRequest.OpType.INDEX) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(queryRuleset.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - clientWithOrigin.index(indexRequest, listener); + clientWithOrigin.index(indexRequest, ActionListener.runAfter(listener, indexRequest::decRef)); } catch (Exception e) { + indexRequest.decRef(); listener.onFailure(e); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java index 46e4d45f2c146..b31474dad6d95 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java @@ -291,12 +291,18 @@ private void updateSearchApplication(SearchApplication app, boolean create, Acti .endObject(); } DocWriteRequest.OpType opType = (create ? DocWriteRequest.OpType.CREATE : DocWriteRequest.OpType.INDEX); - final IndexRequest indexRequest = new IndexRequest(SEARCH_APPLICATION_ALIAS_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(app.name()) - .opType(opType) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(buffer.bytes(), XContentType.JSON); - clientWithOrigin.index(indexRequest, listener); + final IndexRequest indexRequest = new IndexRequest(SEARCH_APPLICATION_ALIAS_NAME); + try { + indexRequest.opType(DocWriteRequest.OpType.INDEX) + .id(app.name()) + .opType(opType) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(buffer.bytes(), XContentType.JSON); + clientWithOrigin.index(indexRequest, ActionListener.runAfter(listener, indexRequest::decRef)); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } catch (Exception e) { listener.onFailure(e); } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java index 85d8826b98683..a8960fc591ce1 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java @@ -77,14 +77,19 @@ public void setup() throws Exception { } private void createConnector(Connector connector) throws IOException, InterruptedException, ExecutionException, TimeoutException { - final IndexRequest indexRequest = new IndexRequest(ConnectorIndexService.CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connector.getConnectorId()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - ActionFuture index = client().index(indexRequest); - - // wait 10 seconds for connector creation - index.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + IndexRequest indexRequest = new IndexRequest(ConnectorIndexService.CONNECTOR_INDEX_NAME); + try { + indexRequest.opType(DocWriteRequest.OpType.INDEX) + .id(connector.getConnectorId()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); + ActionFuture index = client().index(indexRequest); + + // wait 10 seconds for connector creation + index.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } finally { + indexRequest.decRef(); + } } public void testCreateConnectorSyncJob() throws Exception { diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java index 5387e5b08c2fd..7784c56d89ae9 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java @@ -104,6 +104,9 @@ private void prepareIndex() throws Exception { ); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } } public void testBasicAsyncExecution() throws Exception { diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/EqlCancellationIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/EqlCancellationIT.java index f2d552fa6bbdb..40425761d08a4 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/EqlCancellationIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/EqlCancellationIT.java @@ -62,6 +62,9 @@ public void testCancellation() throws Exception { } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } boolean cancelDuringSearch = randomBoolean(); List plugins = initBlockFactory(cancelDuringSearch, cancelDuringSearch == false); EqlSearchRequest request = new EqlSearchRequest().indices("test").query("my_event where val==1").eventCategoryField("event_type"); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java index 57cbee320ece0..f6cb5668afbdb 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -92,7 +93,9 @@ public void setupIndex() throws IOException { BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); try { for (int i = 0; i < numberOfDocs(); i++) { - bulk.add(prepareIndex("test").setId(Integer.toString(i)).setSource("foo", i)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)).setSource("foo", i); + bulk.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } bulk.get(); } finally { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java index 5573755d9850a..6fc368763aece 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java @@ -153,7 +153,12 @@ private void createRemoteIndex(int numDocs) throws Exception { BulkRequestBuilder bulk = client(REMOTE_CLUSTER).prepareBulk("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); try { for (int i = 0; i < numDocs; i++) { - bulk.add(new IndexRequest().source("foo", i)); + IndexRequest indexRequest = new IndexRequest().source("foo", i); + try { + bulk.add(indexRequest); + } finally { + indexRequest.decRef(); + } } bulk.get(); } finally { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java index e3a01bd6f4dd9..85b6c3900e713 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.action; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.internal.Client; @@ -93,7 +94,9 @@ record Host(String ip, String os) { } var hosts = List.of(new Host("192.168.1.3", "Windows")); for (var h : hosts) { - localClient.prepareIndex("hosts").setSource("ip", h.ip, "os", h.os).get(); + IndexRequestBuilder indexRequestBuilder = localClient.prepareIndex("hosts").setSource("ip", h.ip, "os", h.os); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } localClient.admin().indices().prepareRefresh("hosts").get(); EnrichPolicy policy = new EnrichPolicy("match", null, List.of("hosts"), "ip", List.of("ip", "os")); @@ -108,7 +111,10 @@ record Event(String ip, String message) { var events = List.of(new Event("192.168.1.4", "access denied"), new Event("192.168.1.3", "restart")); assertAcked(client(cluster).admin().indices().prepareCreate("events").setMapping("ip", "type=ip", "message", "type=text")); for (Event e : events) { - client(cluster).prepareIndex("events").setSource("ip", e.ip, "message", e.message).get(); + IndexRequestBuilder indexRequestBuilder = client(cluster).prepareIndex("events") + .setSource("ip", e.ip, "message", e.message); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } client(cluster).admin().indices().prepareRefresh("events").get(); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java index 35e019e3a140b..111646528cdd5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.Build; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Setting; @@ -78,7 +79,10 @@ public void populateLocalIndices() { .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long") ); for (int i = 0; i < 10; i++) { - localClient.prepareIndex("logs-1").setSource("id", "local-" + i, "tag", "local", "v", i).get(); + IndexRequestBuilder indexRequestBuilder = localClient.prepareIndex("logs-1") + .setSource("id", "local-" + i, "tag", "local", "v", i); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } localClient.admin().indices().prepareRefresh("logs-1").get(); } @@ -94,7 +98,10 @@ public void populateRemoteIndices() { .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long") ); for (int i = 0; i < 10; i++) { - remoteClient.prepareIndex("logs-2").setSource("id", "remote-" + i, "tag", "remote", "v", i * i).get(); + IndexRequestBuilder indexRequestBuilder = remoteClient.prepareIndex("logs-2") + .setSource("id", "remote-" + i, "tag", "remote", "v", i * i); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } remoteClient.admin().indices().prepareRefresh("logs-2").get(); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index daefa8899b443..c0858997abbf6 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -147,7 +147,7 @@ record Song(String id, String title, String artist, double length) { new Song("s4", "The Sound Of Silence", "Disturbed", 4.08) ); for (var s : songs) { - client().prepareIndex("songs").setSource("song_id", s.id, "title", s.title, "artist", s.artist, "length", s.length).get(); + indexDoc("songs", null, "song_id", s.id, "title", s.title, "artist", s.artist, "length", s.length); } client().admin().indices().prepareRefresh("songs").get(); EnrichPolicy policy = new EnrichPolicy("match", null, List.of("songs"), "song_id", List.of("title", "artist", "length")); @@ -180,9 +180,7 @@ public void setupMainIndex() { .setMapping("timestamp", "type=long", "song_id", "type=keyword", "duration", "type=double") .get(); for (Listen listen : localListens) { - client().prepareIndex("listens") - .setSource("timestamp", listen.timestamp, "song_id", listen.songId, "duration", listen.duration) - .get(); + indexDoc("listens", null, "timestamp", listen.timestamp, "song_id", listen.songId, "duration", listen.duration); } client().admin().indices().prepareRefresh("listens").get(); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java index 5e1c3128d4076..8ad081a8e963c 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java @@ -103,9 +103,7 @@ public void testBreaker() { .get(); int numDocs = between(1000, 5000); for (int i = 0; i < numDocs; i++) { - DocWriteResponse response = prepareIndex("test_breaker").setId(Integer.toString(i)) - .setSource("foo", "foo-" + i, "bar", "bar-" + (i * 2)) - .get(); + DocWriteResponse response = indexDoc("test_breaker", Integer.toString(i), "foo", "foo-" + i, "bar", "bar-" + (i * 2)); assertThat(Strings.toString(response), response.getResult(), equalTo(DocWriteResponse.Result.CREATED)); } client().admin().indices().prepareRefresh("test_breaker").get(); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index fd14f6d412c42..d68994bc0d43f 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.Build; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -19,6 +20,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; @@ -231,6 +233,7 @@ public void testFromGroupingByNumericFieldWithNulls() { .add(new IndexRequest("test").id("no_count_new_" + i).source("data", 99, "data_d", 1d)) .add(new IndexRequest("test").id("no_data_" + i).source("count", 12, "count_d", 12d)) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } if (randomBoolean()) { client().admin().indices().prepareRefresh("test").get(); @@ -285,6 +288,7 @@ public void testFromStatsGroupingByKeywordWithNulls() { .add(new IndexRequest("test").id("no_count_red_" + i).source("data", 2, "data_d", 2d, "color", "red")) .add(new IndexRequest("test").id("no_count_yellow_" + i).source("data", 2, "data_d", 2d, "color", "yellow")) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } if (randomBoolean()) { client().admin().indices().prepareRefresh("test").get(); @@ -601,6 +605,7 @@ public void testFilterWithNullAndEvalFromIndex() { // append entry, with an absent count, to the index try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { bulkRequestBuilder.add(new IndexRequest("test").id("no_count").source("data", 12, "data_d", 2d, "color", "red")).get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } client().admin().indices().prepareRefresh("test").get(); @@ -750,6 +755,7 @@ public void testRefreshSearchIdleShards() throws Exception { long value = randomLongBetween(1, 1000); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { bulkRequestBuilder.add(new IndexRequest(indexName).id("doc-" + i).source("data", 1, "value", value)).get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } sum += value; } @@ -804,6 +810,9 @@ public void testESFilter() throws Exception { indexRequests.add(prepareIndex(indexName).setId(id).setSource(Map.of("val", value))); } indexRandom(true, randomBoolean(), indexRequests); + for (IndexRequestBuilder indexRequestBuilder : indexRequests) { + indexRequestBuilder.request().decRef(); + } String command = "from test_filter | stats avg = avg(val)"; long from = randomBoolean() ? Long.MIN_VALUE : randomLongBetween(-1000, 1000); long to = randomBoolean() ? Long.MAX_VALUE : randomLongBetween(from, from + 1000); @@ -846,6 +855,9 @@ record Doc(long val, String tag) { indexRequests.add(prepareIndex(indexName).setId(Integer.toString(i)).setSource(Map.of("val", d.val, "tag", d.tag))); } indexRandom(true, randomBoolean(), indexRequests); + for (IndexRequestBuilder indexRequestBuilder : indexRequests) { + indexRequestBuilder.request().decRef(); + } int limit = randomIntBetween(1, 10); String command = "from test_extract_fields | sort val | limit " + limit; try (EsqlQueryResponse results = run(command)) { @@ -929,6 +941,7 @@ public void testIndexPatterns() throws Exception { .add(new IndexRequest(indexName).id("4").source("data", ++i, "count", i * 1000)) .add(new IndexRequest(indexName).id("5").source("data", ++i, "count", i * 1000)) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } } @@ -986,6 +999,7 @@ public void testOverlappingIndexPatterns() throws Exception { bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .add(new IndexRequest("test_overlapping_index_patterns_1").id("1").source("field", 10)) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); assertAcked( client().admin() @@ -1000,6 +1014,7 @@ public void testOverlappingIndexPatterns() throws Exception { bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .add(new IndexRequest("test_overlapping_index_patterns_2").id("1").source("field", "foo")) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } assertVerificationException("from test_overlapping_index_patterns_* | sort field"); @@ -1087,6 +1102,7 @@ public void testTopNPushedToLucene() { .add(new IndexRequest("test").id(yellowNullCountDocId).source("data", i, "color", "yellow")) .add(new IndexRequest("test").id(yellowNullDataDocId).source("count", i * 10, "color", "yellow")) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } if (randomBoolean()) { client().admin().indices().prepareRefresh("test").get(); @@ -1218,7 +1234,7 @@ public void testGroupingMultiValueByOrdinals() { if (values.isEmpty() == false) { source.put("v", values); } - prepareIndex(indexName).setSource(source).get(); + index(indexName, null, source); if (randomInt(100) < 20) { client().admin().indices().prepareRefresh(indexName).get(); } @@ -1256,8 +1272,8 @@ public void testUnsupportedTypesOrdinalGrouping() { long v = randomIntBetween(1, 10); groups.merge(k, v, Long::sum); groups.merge(null, v, Long::sum); // null group - prepareIndex("index-1").setSource("f1", k, "v", v).get(); - prepareIndex("index-2").setSource("f2", k, "v", v).get(); + indexDoc("index-1", null, "f1", k, "v", v); + indexDoc("index-2", null, "f2", k, "v", v); } client().admin().indices().prepareRefresh("index-1", "index-2").get(); for (String field : List.of("f1", "f2")) { @@ -1377,6 +1393,7 @@ public void testCountTextField() { indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); } indexRequest.get(); + indexRequest.request().decRef(); } client().admin().indices().prepareRefresh("test_count").get(); try (EsqlQueryResponse resp = run("FROM test_count | stats COUNT_DISTINCT(name)")) { @@ -1448,6 +1465,7 @@ private int indexDocsIntoNestedMappingIndex(String indexName, int docsCount) thr builder.endObject(); bulkBuilder.add(new IndexRequest(indexName).id(Integer.toString(j)).source(builder)); } + decRefAllRequestsInBulkRequestBuilder(bulkBuilder); bulkBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); } ensureYellow(indexName); @@ -1522,6 +1540,7 @@ private void createAndPopulateIndex(String indexName, Settings additionalSetting ) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } } ensureYellow(indexName); @@ -1573,4 +1592,13 @@ private void clearPersistentSettings(Setting... settings) { private DiscoveryNode randomDataNode() { return randomFrom(clusterService().state().nodes().getDataNodes().values()); } + + private void decRefAllRequestsInBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) { + List> requests = bulkRequestBuilder.request().requests(); + for (DocWriteRequest request : requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionRuntimeFieldIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionRuntimeFieldIT.java index a12835b6582e2..ea10b074d68d9 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionRuntimeFieldIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionRuntimeFieldIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; @@ -113,7 +114,9 @@ private void createIndexWithConstRuntimeField(String type) throws InterruptedExc try (BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int i = 0; i < SIZE; i++) { - bulk.add(prepareIndex("test").setId(Integer.toString(i)).setSource("foo", i)); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId(Integer.toString(i)).setSource("foo", i); + bulk.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } bulk.get(); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java index 3410c89e92511..1b4db35785377 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java @@ -45,7 +45,9 @@ public void testConcurrentQueries() throws Exception { for (int d = 0; d < numDocs; d++) { String user = randomFrom("u1", "u2", "u3"); String tag = randomFrom("java", "elasticsearch", "lucene"); - bulk.add(new IndexRequest().source(Map.of("user", user, "tags", tag))); + IndexRequest indexRequest = new IndexRequest().source(Map.of("user", user, "tags", tag)); + bulk.add(indexRequest); + indexRequest.decRef(); } bulk.get(); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SyntheticSourceIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SyntheticSourceIT.java index b924ad492c0c6..b2ed0490b96a5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SyntheticSourceIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SyntheticSourceIT.java @@ -41,6 +41,7 @@ public void testMatchOnlyText() throws Exception { indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); } indexRequest.get(); + indexRequest.request().decRef(); } client().admin().indices().prepareRefresh("test").get(); @@ -62,6 +63,7 @@ public void testText() throws Exception { indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); } indexRequest.get(); + indexRequest.request().decRef(); } client().admin().indices().prepareRefresh("test").get(); try (EsqlQueryResponse resp = run("from test | keep field, id | sort id asc | limit 1")) { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java index c953a9cbdc2f4..2be259296ee79 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java @@ -28,7 +28,9 @@ public void testFilter() { int oldDocs = between(10, 100); for (int i = 0; i < oldDocs; i++) { long timestamp = epoch - TimeValue.timeValueHours(between(1, 2)).millis(); - bulk.add(new IndexRequest().source("@timestamp", timestamp, "value", -i)); + IndexRequest indexRequest = new IndexRequest().source("@timestamp", timestamp, "value", -i); + bulk.add(indexRequest); + indexRequest.decRef(); } int newDocs = between(10, 100); for (int i = 0; i < newDocs; i++) { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/WarningsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/WarningsIT.java index fb6d23695f837..b1004eb38ef2f 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/WarningsIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/WarningsIT.java @@ -42,7 +42,7 @@ public void testCollectWarnings() throws Exception { .setMapping("host", "type=keyword") ); for (int i = 0; i < numDocs1; i++) { - client().prepareIndex("index-1").setSource("host", "192." + i).get(); + indexDoc("index-1", null, "host", "192." + i); } int numDocs2 = randomIntBetween(1, 15); assertAcked( @@ -53,7 +53,7 @@ public void testCollectWarnings() throws Exception { .setMapping("host", "type=keyword") ); for (int i = 0; i < numDocs2; i++) { - client().prepareIndex("index-2").setSource("host", "10." + i).get(); + indexDoc("index-2", null, "host", "10." + i); } DiscoveryNode coordinator = randomFrom(clusterService().state().nodes().stream().toList()); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/CanMatchIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/CanMatchIT.java index 8fc61872c4acd..de195ba96ce0f 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/CanMatchIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/CanMatchIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.plugin; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; @@ -14,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.shard.ShardId; @@ -57,6 +59,7 @@ public void testCanMatch() { .add(new IndexRequest().source("@timestamp", "2022-05-02", "uid", "u1")) .add(new IndexRequest().source("@timestamp", "2022-12-15", "uid", "u1")) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } ElasticsearchAssertions.assertAcked( client().admin().indices().prepareCreate("events_2023").setMapping("@timestamp", "type=date", "uid", "type=keyword") @@ -68,6 +71,7 @@ public void testCanMatch() { .add(new IndexRequest().source("@timestamp", "2023-02-11", "uid", "u1")) .add(new IndexRequest().source("@timestamp", "2023-03-25", "uid", "u1")) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } try { Set queriedIndices = ConcurrentCollections.newConcurrentSet(); @@ -139,6 +143,7 @@ public void testAliasFilters() { .add(new IndexRequest().source("emp_no", 105, "dept", "engineering", "hired", "2012-06-30", "salary", 25)) .add(new IndexRequest().source("emp_no", 106, "dept", "sales", "hired", "2012-08-09", "salary", 30.1)) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } ElasticsearchAssertions.assertAcked( @@ -230,6 +235,7 @@ public void testFailOnUnavailableShards() throws Exception { .add(new IndexRequest().source("timestamp", 2, "message", "b")) .add(new IndexRequest().source("timestamp", 3, "message", "c")) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } ElasticsearchAssertions.assertAcked( client().admin() @@ -247,6 +253,7 @@ public void testFailOnUnavailableShards() throws Exception { .add(new IndexRequest().source("timestamp", 10, "message", "aa")) .add(new IndexRequest().source("timestamp", 11, "message", "bb")) .get(); + decRefAllRequestsInBulkRequestBuilder(bulkRequestBuilder); } try (EsqlQueryResponse resp = run("from events,logs | KEEP timestamp,message")) { assertThat(getValuesList(resp), hasSize(5)); @@ -256,4 +263,13 @@ public void testFailOnUnavailableShards() throws Exception { assertThat(error.getMessage(), containsString("no shard copies found")); } } + + private void decRefAllRequestsInBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) { + List> requests = bulkRequestBuilder.request().requests(); + for (DocWriteRequest request : requests) { + if (request instanceof RefCounted refCounted) { + refCounted.decRef(); + } + } + } } diff --git a/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsActionIT.java b/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsActionIT.java index 88f3b126b228c..59eebadc8f25b 100644 --- a/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsActionIT.java +++ b/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsActionIT.java @@ -9,8 +9,10 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -73,7 +75,7 @@ public void testGetGlobalCheckpoints() throws Exception { final int totalDocuments = shards * 3; for (int i = 0; i < totalDocuments; ++i) { - prepareIndex(indexName).setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get(); + indexEmptyDoc(indexName, Integer.toString(i)); } final GetGlobalCheckpointsAction.Request request2 = new GetGlobalCheckpointsAction.Request( @@ -117,7 +119,10 @@ public void testPollGlobalCheckpointAdvancement() throws Exception { final int totalDocuments = between(25, 50); new Thread(() -> { for (int i = 0; i < totalDocuments; ++i) { - prepareIndex(indexName).setId(Integer.toString(i)).setSource("{}", XContentType.JSON).execute(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName); + indexRequestBuilder.setId(Integer.toString(i)) + .setSource("{}", XContentType.JSON) + .execute(ActionListener.running(() -> indexRequestBuilder.request().decRef())); } }).start(); @@ -146,7 +151,7 @@ public void testPollGlobalCheckpointAdvancementTimeout() { final int totalDocuments = 30; for (int i = 0; i < totalDocuments; ++i) { - prepareIndex(indexName).setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get(); + indexEmptyDoc(indexName, Integer.toString(i)); } final GetGlobalCheckpointsAction.Request request = new GetGlobalCheckpointsAction.Request( @@ -251,7 +256,7 @@ public void testWaitOnIndexCreated() throws Exception { indicesAdmin().prepareCreate(indexName) .setSettings(indexSettings(1, 0).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST)) .get(); - prepareIndex(indexName).setId(Integer.toString(0)).setSource("{}", XContentType.JSON).get(); + indexEmptyDoc(indexName, Integer.toString(0)); GetGlobalCheckpointsAction.Response response = future.actionGet(); long elapsed = TimeValue.timeValueNanos(System.nanoTime() - start).seconds(); @@ -328,7 +333,7 @@ public void testWaitOnPrimaryShardsReady() throws Exception { ActionFuture future = client().execute(GetGlobalCheckpointsAction.INSTANCE, request); Thread.sleep(randomIntBetween(10, 100)); updateIndexSettings(Settings.builder().put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "node", ""), indexName); - prepareIndex(indexName).setId(Integer.toString(0)).setSource("{}", XContentType.JSON).get(); + indexEmptyDoc(indexName, Integer.toString(0)); GetGlobalCheckpointsAction.Response response = future.actionGet(); long elapsed = TimeValue.timeValueNanos(System.nanoTime() - start).seconds(); @@ -354,7 +359,7 @@ public void testWaitOnPrimaryShardThrottled() throws Exception { Thread.sleep(randomIntBetween(10, 100)); updateClusterSettings(Settings.builder().putNull(CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.getKey())); - prepareIndex(indexName).setId(Integer.toString(0)).setSource("{}", XContentType.JSON).get(); + indexEmptyDoc(indexName, Integer.toString(0)); var response = future.actionGet(); long elapsed = TimeValue.timeValueNanos(System.nanoTime() - start).seconds(); @@ -362,4 +367,14 @@ public void testWaitOnPrimaryShardThrottled() throws Exception { assertThat(response.globalCheckpoints()[0], equalTo(0L)); assertFalse(response.timedOut()); } + + private void indexEmptyDoc(String index, String id) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java index ba4e1d98f63a6..ad7423efe0de3 100644 --- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java +++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java @@ -74,8 +74,7 @@ public void testTimestampRangeRecalculatedOnStalePrimaryAllocation() throws IOEx createIndex("index", 1, 1); - final DocWriteResponse indexResponse = prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567Z") - .get(); + DocWriteResponse indexResponse = indexDoc("index", null, DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567Z"); ensureGreen("index"); @@ -171,7 +170,7 @@ public void testTimestampFieldTypeExposedByAllIndicesServices() throws Exception ensureGreen("index"); if (randomBoolean()) { - prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, date).get(); + indexDoc("index", null, DataStream.TIMESTAMP_FIELD_NAME, date); } for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) { @@ -224,7 +223,7 @@ public void testRetryPointInTime() throws Exception { ); int numDocs = randomIntBetween(1, 100); for (int i = 0; i < numDocs; i++) { - prepareIndex(indexName).setSource("created_date", "2011-02-02").get(); + indexDoc(indexName, null, "created_date", "2011-02-02"); } assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet()); final OpenPointInTimeRequest openPointInTimeRequest = new OpenPointInTimeRequest(indexName).indicesOptions( @@ -263,13 +262,13 @@ public void testPointInTimeWithDeletedIndices() { int index1 = randomIntBetween(10, 50); for (int i = 0; i < index1; i++) { String id = Integer.toString(i); - prepareIndex("index-1").setId(id).setSource("value", i).get(); + indexDoc("index-1", id, "value", i); } int index2 = randomIntBetween(10, 50); for (int i = 0; i < index2; i++) { String id = Integer.toString(i); - prepareIndex("index-2").setId(id).setSource("value", i).get(); + indexDoc("index-2", id, "value", i); } assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index-1", "index-2")).actionGet()); @@ -304,7 +303,7 @@ public void testOpenPointInTimeWithNoIndexMatched() { int numDocs = randomIntBetween(10, 50); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); - prepareIndex("test-index").setId(id).setSource("value", i).get(); + indexDoc("test-index", id, "value", i); } assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("test-index")).actionGet()); // include the frozen indices diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexRecoveryTests.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexRecoveryTests.java index 8bc32f0171f29..48d908007af2f 100644 --- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexRecoveryTests.java +++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexRecoveryTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.index.engine.frozen; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -57,12 +58,13 @@ public void testRecoverExistingReplica() throws Exception { .put("index.routing.allocation.include._name", String.join(",", dataNodes)) .build() ); - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, randomIntBetween(0, 50)).mapToObj(n -> prepareIndex(indexName).setSource("num", n)).collect(toList()) - ); + List indexRequestBuilders = IntStream.range(0, randomIntBetween(0, 50)) + .mapToObj(n -> prepareIndex(indexName).setSource("num", n)) + .collect(toList()); + indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), indexRequestBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } ensureGreen(indexName); indicesAdmin().prepareFlush(indexName).get(); // index more documents while one shard copy is offline @@ -72,7 +74,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { Client client = client(dataNodes.get(0)); int moreDocs = randomIntBetween(1, 50); for (int i = 0; i < moreDocs; i++) { - client.prepareIndex(indexName).setSource("num", i).get(); + indexDoc(indexName, null, "num", i); } assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet()); return super.onNodeStopped(nodeName); diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java index f1d23739b0938..bbb94afd693bb 100644 --- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java +++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.ClosePointInTimeRequest; import org.elasticsearch.action.search.OpenPointInTimeRequest; import org.elasticsearch.action.search.OpenPointInTimeResponse; @@ -94,14 +95,11 @@ String openReaders(TimeValue keepAlive, String... indices) { public void testCloseFreezeAndOpen() throws Exception { String indexName = "index"; createIndex(indexName, Settings.builder().put("index.number_of_shards", 2).build()); - prepareIndex(indexName).setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(indexName).setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(indexName).setId("3").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate(indexName, "1", "field", "value"); + indexDocImmediate(indexName, "2", "field", "value"); + indexDocImmediate(indexName, "3", "field", "value"); assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet()); - expectThrows( - ClusterBlockException.class, - prepareIndex(indexName).setId("4").setSource("field", "value").setRefreshPolicy(IMMEDIATE) - ); + expectThrows(ClusterBlockException.class, () -> indexDocImmediate(indexName, "4", "field", "value")); IndicesService indexServices = getInstanceFromNode(IndicesService.class); Index index = resolveIndex(indexName); IndexService indexService = indexServices.indexServiceSafe(index); @@ -185,7 +183,7 @@ public void testSearchAndGetAPIsAreThrottled() throws IOException { String indexName = "index"; createIndex(indexName, Settings.builder().put("index.number_of_shards", 2).build(), mapping); for (int i = 0; i < 10; i++) { - prepareIndex(indexName).setId("" + i).setSource("field", "foo bar baz").get(); + indexDoc(indexName, "" + i, "field", "foo bar baz"); } assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet()); int numRequests = randomIntBetween(20, 50); @@ -223,9 +221,9 @@ public void testFreezeAndUnfreeze() { final IndexService originalIndexService = createIndex("index", Settings.builder().put("index.number_of_shards", 2).build()); assertThat(originalIndexService.getMetadata().getTimestampRange(), sameInstance(IndexLongFieldRange.UNKNOWN)); - prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("index").setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("index").setId("3").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("index", "1", "field", "value"); + indexDocImmediate("index", "2", "field", "value"); + indexDocImmediate("index", "3", "field", "value"); if (randomBoolean()) { // sometimes close it @@ -256,7 +254,7 @@ public void testFreezeAndUnfreeze() { assertThat(engine, Matchers.instanceOf(InternalEngine.class)); assertThat(indexService.getMetadata().getTimestampRange(), sameInstance(IndexLongFieldRange.UNKNOWN)); } - prepareIndex("index").setId("4").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("index", "4", "field", "value"); } private void assertIndexFrozen(String idx) { @@ -284,9 +282,9 @@ public void testDoubleFreeze() { public void testUnfreezeClosedIndices() { createIndex("idx", Settings.builder().put("index.number_of_shards", 1).build()); - prepareIndex("idx").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("idx", "1", "field", "value"); createIndex("idx-closed", Settings.builder().put("index.number_of_shards", 1).build()); - prepareIndex("idx-closed").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("idx-closed", "1", "field", "value"); assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("idx")).actionGet()); assertAcked(indicesAdmin().prepareClose("idx-closed").get()); assertAcked( @@ -304,9 +302,9 @@ public void testUnfreezeClosedIndices() { public void testFreezePattern() { String indexName = "test-idx"; createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build()); - prepareIndex(indexName).setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate(indexName, "1", "field", "value"); createIndex("test-idx-1", Settings.builder().put("index.number_of_shards", 1).build()); - prepareIndex("test-idx-1").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test-idx-1", "1", "field", "value"); assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet()); assertIndexFrozen(indexName); @@ -328,8 +326,8 @@ public void testFreezePattern() { public void testCanMatch() throws IOException { createIndex("index"); - prepareIndex("index").setId("1").setSource("field", "2010-01-05T02:00").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("index").setId("2").setSource("field", "2010-01-06T02:00").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("index", "1", "field", "2010-01-05T02:00"); + indexDocImmediate("index", "2", "field", "2010-01-06T02:00"); { IndicesService indexServices = getInstanceFromNode(IndicesService.class); Index index = resolveIndex("index"); @@ -466,10 +464,10 @@ public void testCanMatch() throws IOException { public void testWriteToFrozenIndex() { createIndex("idx", Settings.builder().put("index.number_of_shards", 1).build()); - prepareIndex("idx").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("idx", "1", "field", "value"); assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("idx")).actionGet()); assertIndexFrozen("idx"); - expectThrows(ClusterBlockException.class, prepareIndex("idx").setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE)); + expectThrows(ClusterBlockException.class, () -> indexDocImmediate("idx", "2", "field", "value")); } public void testIgnoreUnavailable() { @@ -511,7 +509,7 @@ public void testUnfreezeClosedIndex() { public void testFreezeIndexIncreasesIndexSettingsVersion() { final String index = "test"; createIndex(index, indexSettings(1, 0).build()); - prepareIndex(index).setSource("field", "value").get(); + indexDoc(index, null, "field", "value"); final long settingsVersion = clusterAdmin().prepareState().get().getState().metadata().index(index).getSettingsVersion(); @@ -551,7 +549,7 @@ public void testRecoveryState() { final long nbDocs = randomIntBetween(0, 50); for (long i = 0; i < nbDocs; i++) { - final DocWriteResponse indexResponse = prepareIndex(indexName).setId(Long.toString(i)).setSource("field", i).get(); + final DocWriteResponse indexResponse = indexDoc(indexName, Long.toString(i), "field", i); assertThat(indexResponse.status(), is(RestStatus.CREATED)); } @@ -582,7 +580,7 @@ public void testTranslogStats() { final int nbDocs = randomIntBetween(0, 50); int uncommittedOps = 0; for (long i = 0; i < nbDocs; i++) { - final DocWriteResponse indexResponse = prepareIndex(indexName).setId(Long.toString(i)).setSource("field", i).get(); + final DocWriteResponse indexResponse = indexDoc(indexName, Long.toString(i), "field", i); assertThat(indexResponse.status(), is(RestStatus.CREATED)); if (rarely()) { indicesAdmin().prepareFlush(indexName).get(); @@ -616,8 +614,8 @@ public void testTranslogStats() { public void testComputesTimestampRangeFromMilliseconds() { final int shardCount = between(1, 3); createIndex("index", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shardCount).build()); - prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-05T01:02:03.456Z").get(); - prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567Z").get(); + indexDoc("index", null, DataStream.TIMESTAMP_FIELD_NAME, "2010-01-05T01:02:03.456Z"); + indexDoc("index", null, DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567Z"); assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet()); @@ -648,8 +646,8 @@ public void testComputesTimestampRangeFromNanoseconds() throws IOException { final int shardCount = between(1, 3); createIndex("index", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shardCount).build(), mapping); - prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-05T01:02:03.456789012Z").get(); - prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567890123Z").get(); + indexDoc("index", null, DataStream.TIMESTAMP_FIELD_NAME, "2010-01-05T01:02:03.456789012Z"); + indexDoc("index", null, DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567890123Z"); assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet()); @@ -667,4 +665,21 @@ public void testComputesTimestampRangeFromNanoseconds() throws IOException { assertThat(timestampFieldRange.getMax(), equalTo(resolution.convert(Instant.parse("2010-01-06T02:03:04.567890123Z")))); } + private DocWriteResponse indexDocImmediate(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).setRefreshPolicy(IMMEDIATE).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + private DocWriteResponse indexDoc(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/x-pack/plugin/graph/src/internalClusterTest/java/org/elasticsearch/xpack/graph/test/GraphTests.java b/x-pack/plugin/graph/src/internalClusterTest/java/org/elasticsearch/xpack/graph/test/GraphTests.java index 3623d3671e83f..9341542b65d73 100644 --- a/x-pack/plugin/graph/src/internalClusterTest/java/org/elasticsearch/xpack/graph/test/GraphTests.java +++ b/x-pack/plugin/graph/src/internalClusterTest/java/org/elasticsearch/xpack/graph/test/GraphTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; import org.elasticsearch.action.admin.indices.segments.ShardSegments; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -88,9 +89,10 @@ public void setUp() throws Exception { for (DocTemplate dt : socialNetTemplate) { for (int i = 0; i < dt.numDocs; i++) { // Supply a doc ID for deterministic routing of docs to shards - prepareIndex("test").setId("doc#" + numDocs) - .setSource("decade", dt.decade, "people", dt.people, "description", dt.description) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("test").setId("doc#" + numDocs) + .setSource("decade", dt.decade, "people", dt.people, "description", dt.description); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); numDocs++; } } diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java index bd425487b9ad0..edbd28d219811 100644 --- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java +++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java @@ -261,20 +261,23 @@ private void _writeDocument( // Due to the lack of "alias templates" (at the current time), we cannot write to the alias if it doesn't exist yet // - that would cause the alias to be created as a concrete index, which is not what we want. // So, until we know that the alias exists we have to write to the expected index name instead. - final IndexRequest request = new IndexRequest(aliasExists ? ALIAS_NAME : INDEX_NAME).opType(opType) - .source(xContentBuilder) - .id(document.docId) - .setRefreshPolicy(refreshPolicy); - client.index(request, listener.delegateFailureAndWrap((l, response) -> { - logger.debug( - "Wrote service provider [{}][{}] as document [{}] ({})", - document.name, - document.entityId, - response.getId(), - response.getResult() - ); - l.onResponse(response); - })); + final IndexRequest request = new IndexRequest(aliasExists ? ALIAS_NAME : INDEX_NAME).opType(opType); + try { + request.source(xContentBuilder).id(document.docId).setRefreshPolicy(refreshPolicy); + client.index(request, ActionListener.runAfter(listener.delegateFailureAndWrap((l, response) -> { + logger.debug( + "Wrote service provider [{}][{}] as document [{}] ({})", + document.name, + document.entityId, + response.getId(), + response.getResult() + ); + l.onResponse(response); + }), request::decRef)); + } catch (Exception e) { + request.decRef(); + throw e; + } } catch (IOException e) { listener.onFailure(e); } diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java index 164287755895e..eda86cbcacfcb 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java @@ -935,10 +935,15 @@ static void indexDocs(String dataStream, int numDocs) { try (BulkRequest bulkRequest = new BulkRequest()) { for (int i = 0; i < numDocs; i++) { String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); - bulkRequest.add( - new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) - .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) - ); + IndexRequest indexRequest = new IndexRequest(dataStream); + try { + bulkRequest.add( + indexRequest.opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems().length, equalTo(numDocs)); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java index 4164911dcad79..e871e7c306468 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ilm; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -97,7 +98,9 @@ public void testShrinkOnTiers() throws Exception { TransportPutComposableIndexTemplateAction.TYPE, new TransportPutComposableIndexTemplateAction.Request("template").indexTemplate(template) ).actionGet(); - prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertBusy(() -> { ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*")) diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java index 7314587573f96..be1b76383c5bf 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ilm; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -97,7 +98,9 @@ public void testShrinkOnTiers() throws Exception { TransportPutComposableIndexTemplateAction.TYPE, new TransportPutComposableIndexTemplateAction.Request("template").indexTemplate(template) ).actionGet(); - prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); assertBusy(() -> { ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*")) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java index b8af3e8e0daa2..c88f51057ee70 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java @@ -182,7 +182,11 @@ public void putAsync(ILMHistoryItem item) { try (XContentBuilder builder = XContentFactory.jsonBuilder()) { item.toXContent(builder, ToXContent.EMPTY_PARAMS); IndexRequest request = new IndexRequest(ILM_HISTORY_DATA_STREAM).source(builder).opType(DocWriteRequest.OpType.CREATE); - processor.add(request); + try { + processor.add(request); + } finally { + request.decRef(); + } } catch (Exception e) { logger.error(() -> format("failed to send ILM history item to index [%s]: [%s]", ILM_HISTORY_DATA_STREAM, item), e); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java index 96b9f219d353d..d37e62d7dadee 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java @@ -256,25 +256,36 @@ private ModelConfigMap createModelConfigMap(SearchHits hits, String modelId) { public void storeModel(Model model, ActionListener listener) { ActionListener bulkResponseActionListener = getStoreModelListener(model, listener); - IndexRequest configRequest = createIndexRequest( - Model.documentId(model.getConfigurations().getModelId()), - InferenceIndex.INDEX_NAME, - model.getConfigurations(), - false - ); - - IndexRequest secretsRequest = createIndexRequest( - Model.documentId(model.getConfigurations().getModelId()), - InferenceSecretsIndex.INDEX_NAME, - model.getSecrets(), - false - ); - BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); - bulkRequestBuilder.add(configRequest) - .add(secretsRequest) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .execute(ActionListener.releaseAfter(bulkResponseActionListener, bulkRequestBuilder)); + try { + IndexRequest configRequest = createIndexRequest( + Model.documentId(model.getConfigurations().getModelId()), + InferenceIndex.INDEX_NAME, + model.getConfigurations(), + false + ); + try { + bulkRequestBuilder.add(configRequest); + } finally { + configRequest.decRef(); + } + IndexRequest secretsRequest = createIndexRequest( + Model.documentId(model.getConfigurations().getModelId()), + InferenceSecretsIndex.INDEX_NAME, + model.getSecrets(), + false + ); + try { + bulkRequestBuilder.add(secretsRequest); + } finally { + secretsRequest.decRef(); + } + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute(ActionListener.releaseAfter(bulkResponseActionListener, bulkRequestBuilder)); + } catch (Exception e) { + bulkRequestBuilder.request().decRef(); + throw e; + } } private static ActionListener getStoreModelListener(Model model, ActionListener listener) { @@ -361,10 +372,15 @@ public void deleteModel(String modelId, ActionListener listener) { private static IndexRequest createIndexRequest(String docId, String indexName, ToXContentObject body, boolean allowOverwriting) { try (XContentBuilder builder = XContentFactory.jsonBuilder()) { var request = new IndexRequest(indexName); - XContentBuilder source = body.toXContent(builder, ToXContent.EMPTY_PARAMS); - var operation = allowOverwriting ? DocWriteRequest.OpType.INDEX : DocWriteRequest.OpType.CREATE; - - return request.opType(operation).id(docId).source(source); + try { + XContentBuilder source = body.toXContent(builder, ToXContent.EMPTY_PARAMS); + var operation = allowOverwriting ? DocWriteRequest.OpType.INDEX : DocWriteRequest.OpType.CREATE; + + return request.opType(operation).id(docId).source(source); + } catch (Exception e) { + request.decRef(); + throw e; + } } catch (IOException ex) { throw new ElasticsearchException(format("Unexpected serialization exception for index [%s] doc [%s]", indexName, docId), ex); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/SearchIdleTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/SearchIdleTests.java index 2da4e2802bdbe..9d93e7963c53a 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/SearchIdleTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/SearchIdleTests.java @@ -7,9 +7,11 @@ package org.elasticsearch.xpack.constantkeyword.mapper; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -117,8 +119,8 @@ public void testSearchIdleConstantKeywordMatchNoIndex() throws InterruptedExcept "type=keyword" ); - assertEquals(RestStatus.CREATED, prepareIndex(idleIndex).setSource("keyword", randomAlphaOfLength(10)).get().status()); - assertEquals(RestStatus.CREATED, prepareIndex(activeIndex).setSource("keyword", randomAlphaOfLength(10)).get().status()); + assertEquals(RestStatus.CREATED, indexDoc(idleIndex, null, "keyword", randomAlphaOfLength(10)).status()); + assertEquals(RestStatus.CREATED, indexDoc(activeIndex, null, "keyword", randomAlphaOfLength(10)).status()); assertEquals(RestStatus.OK, indicesAdmin().prepareRefresh(idleIndex, activeIndex).get().getStatus()); waitUntil( @@ -187,8 +189,8 @@ public void testSearchIdleConstantKeywordMatchOneIndex() throws InterruptedExcep "type=keyword" ); - assertEquals(RestStatus.CREATED, prepareIndex(idleIndex).setSource("keyword", randomAlphaOfLength(10)).get().status()); - assertEquals(RestStatus.CREATED, prepareIndex(activeIndex).setSource("keyword", randomAlphaOfLength(10)).get().status()); + assertEquals(RestStatus.CREATED, indexDoc(idleIndex, null, "keyword", randomAlphaOfLength(10)).status()); + assertEquals(RestStatus.CREATED, indexDoc(activeIndex, null, "keyword", randomAlphaOfLength(10)).status()); assertEquals(RestStatus.OK, indicesAdmin().prepareRefresh(idleIndex, activeIndex).get().getStatus()); waitUntil( @@ -254,8 +256,8 @@ public void testSearchIdleConstantKeywordMatchTwoIndices() throws InterruptedExc "type=keyword" ); - assertEquals(RestStatus.CREATED, prepareIndex(idleIndex).setSource("keyword", randomAlphaOfLength(10)).get().status()); - assertEquals(RestStatus.CREATED, prepareIndex(activeIndex).setSource("keyword", randomAlphaOfLength(10)).get().status()); + assertEquals(RestStatus.CREATED, indexDoc(idleIndex, null, "keyword", randomAlphaOfLength(10)).status()); + assertEquals(RestStatus.CREATED, indexDoc(activeIndex, null, "keyword", randomAlphaOfLength(10)).status()); assertEquals(RestStatus.OK, indicesAdmin().prepareRefresh(idleIndex, activeIndex).get().getStatus()); waitUntil( @@ -314,8 +316,8 @@ public void testSearchIdleWildcardQueryMatchOneIndex() throws InterruptedExcepti "type=constant_keyword,value=test2_value" ); - assertEquals(RestStatus.CREATED, prepareIndex(idleIndex).setSource("keyword", "value").get().status()); - assertEquals(RestStatus.CREATED, prepareIndex(activeIndex).setSource("keyword", "value").get().status()); + assertEquals(RestStatus.CREATED, indexDoc(idleIndex, null, "keyword", "value").status()); + assertEquals(RestStatus.CREATED, indexDoc(activeIndex, null, "keyword", "value").status()); assertEquals(RestStatus.OK, indicesAdmin().prepareRefresh(idleIndex, activeIndex).get().getStatus()); waitUntil( @@ -379,4 +381,13 @@ private static boolean canMatch(final SearchService service, final IndexShard in new ShardSearchRequest(OriginalIndices.NONE, request, indexShard.shardId(), 0, 1, AliasFilter.EMPTY, 1f, -1, null) ).canMatch(); } + + private DocWriteResponse indexDoc(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java index 955f03bd8b240..dc6f1d0f5a70c 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java @@ -81,16 +81,24 @@ public void setupSuiteScopeCluster() throws Exception { builders.add(prepareIndex("idx-sort").setSource(jsonBuilder().startObject().field("ul_field", values[i]).endObject())); } indexRandom(true, builders); + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } prepareCreate("idx2").setMapping("ul_field", "type=long").setSettings(settings).get(); + List indexRequests = new ArrayList<>(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (int i = 0; i < 4; i++) { IndexRequest indexRequest = new IndexRequest("idx2").source("ul_field", values[i]); + indexRequests.add(indexRequest); bulkRequestBuilder.add(indexRequest); } bulkRequestBuilder.get(); } + for (IndexRequest indexRequest : indexRequests) { + indexRequest.decRef(); + } ensureSearchable(); } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java index 94d8a144b0bd6..e3d4907e34990 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.versionfield; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; @@ -17,6 +19,7 @@ import org.elasticsearch.search.aggregations.metrics.Cardinality; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.elasticsearch.xpack.analytics.stringstats.InternalStringStats; import org.elasticsearch.xpack.analytics.stringstats.StringStatsAggregationBuilder; @@ -41,12 +44,12 @@ public String setUpIndex(String indexName) throws IOException { createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); - prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "11.1.0").endObject()).get(); - prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.0.0").endObject()).get(); - prepareIndex(indexName).setId("3").setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()).get(); - prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()).get(); - prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - prepareIndex(indexName).setId("6").setSource(jsonBuilder().startObject().field("version", "21.11.0").endObject()).get(); + index(indexName, "1", jsonBuilder().startObject().field("version", "11.1.0").endObject()); + index(indexName, "2", jsonBuilder().startObject().field("version", "1.0.0").endObject()); + index(indexName, "3", jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()); + index(indexName, "4", jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()); + index(indexName, "5", jsonBuilder().startObject().field("version", "2.1.0").endObject()); + index(indexName, "6", jsonBuilder().startObject().field("version", "21.11.0").endObject()); client().admin().indices().prepareRefresh(indexName).get(); return indexName; } @@ -126,8 +129,8 @@ public void testPrefixQuery() throws IOException { public void testSort() throws IOException { String indexName = setUpIndex("test"); // also adding some invalid versions that should be sorted after legal ones - prepareIndex(indexName).setSource(jsonBuilder().startObject().field("version", "1.2.3alpha").endObject()).get(); - prepareIndex(indexName).setSource(jsonBuilder().startObject().field("version", "1.3.567#12").endObject()).get(); + index(indexName, jsonBuilder().startObject().field("version", "1.2.3alpha").endObject()); + index(indexName, jsonBuilder().startObject().field("version", "1.3.567#12").endObject()); client().admin().indices().prepareRefresh(indexName).get(); // sort based on version field @@ -169,13 +172,11 @@ public void testRegexQuery() throws Exception { createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); - prepareIndex(indexName).setId("1") - .setSource(jsonBuilder().startObject().field("version", "1.0.0alpha2.1.0-rc.1").endObject()) - .get(); - prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()).get(); - prepareIndex(indexName).setId("3").setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()).get(); - prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); + index(indexName, "1", jsonBuilder().startObject().field("version", "1.0.0alpha2.1.0-rc.1").endObject()); + index(indexName, "2", jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()); + index(indexName, "3", jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()); + index(indexName, "4", jsonBuilder().startObject().field("version", "2.1.0").endObject()); + index(indexName, "5", jsonBuilder().startObject().field("version", "2.33.0").endObject()); client().admin().indices().prepareRefresh(indexName).get(); assertResponse(client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "2.*0")), response -> { @@ -223,14 +224,12 @@ public void testFuzzyQuery() throws Exception { createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); - prepareIndex(indexName).setId("1") - .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) - .get(); - prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()).get(); - prepareIndex(indexName).setId("3").setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()).get(); - prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); - prepareIndex(indexName).setId("6").setSource(jsonBuilder().startObject().field("version", "2.a3.0").endObject()).get(); + index(indexName, "1", jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()); + index(indexName, "2", jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()); + index(indexName, "3", jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()); + index(indexName, "4", jsonBuilder().startObject().field("version", "2.1.0").endObject()); + index(indexName, "5", jsonBuilder().startObject().field("version", "2.33.0").endObject()); + index(indexName, "6", jsonBuilder().startObject().field("version", "2.a3.0").endObject()); client().admin().indices().prepareRefresh(indexName).get(); assertResponse(client().prepareSearch(indexName).setQuery(QueryBuilders.fuzzyQuery("version", "2.3.0")), response -> { @@ -264,7 +263,7 @@ public void testWildcardQuery() throws Exception { "3.1.1+b", "3.1.123" )) { - prepareIndex(indexName).setSource(jsonBuilder().startObject().field("version", version).endObject()).get(); + index(indexName, jsonBuilder().startObject().field("version", version).endObject()); } client().admin().indices().prepareRefresh(indexName).get(); @@ -314,10 +313,10 @@ public void testStoreMalformed() throws Exception { createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); - prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "1.invalid.0").endObject()).get(); - prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "2.2.0").endObject()).get(); - prepareIndex(indexName).setId("3").setSource(jsonBuilder().startObject().field("version", "2.2.0-badchar!").endObject()).get(); - prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "").endObject()).get(); + index(indexName, "1", jsonBuilder().startObject().field("version", "1.invalid.0").endObject()); + index(indexName, "2", jsonBuilder().startObject().field("version", "2.2.0").endObject()); + index(indexName, "3", jsonBuilder().startObject().field("version", "2.2.0-badchar!").endObject()); + index(indexName, "4", jsonBuilder().startObject().field("version", "").endObject()); client().admin().indices().prepareRefresh(indexName).get(); assertResponse(client().prepareSearch(indexName).addDocValueField("version"), response -> { @@ -381,11 +380,11 @@ public void testAggs() throws Exception { createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); - prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "1.0").endObject()).get(); - prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.3.0").endObject()).get(); - prepareIndex(indexName).setId("3").setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject()).get(); - prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "3.11.5").endObject()).get(); + index(indexName, "1", jsonBuilder().startObject().field("version", "1.0").endObject()); + index(indexName, "2", jsonBuilder().startObject().field("version", "1.3.0").endObject()); + index(indexName, "3", jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject()); + index(indexName, "4", jsonBuilder().startObject().field("version", "2.1.0").endObject()); + index(indexName, "5", jsonBuilder().startObject().field("version", "3.11.5").endObject()); client().admin().indices().prepareRefresh(indexName).get(); // terms aggs @@ -429,11 +428,9 @@ public void testMultiValues() throws Exception { createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); - prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().array("version", "1.0.0", "3.0.0").endObject()).get(); - prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().array("version", "2.0.0", "4.alpha.0").endObject()).get(); - prepareIndex(indexName).setId("3") - .setSource(jsonBuilder().startObject().array("version", "2.1.0", "2.2.0", "5.99.0").endObject()) - .get(); + index(indexName, "1", jsonBuilder().startObject().array("version", "1.0.0", "3.0.0").endObject()); + index(indexName, "2", jsonBuilder().startObject().array("version", "2.0.0", "4.alpha.0").endObject()); + index(indexName, "3", jsonBuilder().startObject().array("version", "2.1.0", "2.2.0", "5.99.0").endObject()); client().admin().indices().prepareRefresh(indexName).get(); assertResponse(client().prepareSearch(indexName).addSort("version", SortOrder.ASC), response -> { @@ -469,4 +466,17 @@ public void testMultiValues() throws Exception { response -> assertEquals(1, response.getHits().getTotalHits().value) ); } + + protected final DocWriteResponse index(String index, String id, XContentBuilder source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + + protected final DocWriteResponse index(String index, XContentBuilder source) { + return index(index, null, source); + } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java index 51497c99ba064..6860980d3952e 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java @@ -161,10 +161,14 @@ private void writeData(Logger logger, String index, long numDocs, long start, lo try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { for (int i = 0; i < numDocs; i++) { IndexRequest indexRequest = new IndexRequest(index); - long timestamp = start + randomIntBetween(0, maxDelta); - assert timestamp >= start && timestamp < end; - indexRequest.source("time", timestamp, "value", i); - bulkRequestBuilder.add(indexRequest); + try { + long timestamp = start + randomIntBetween(0, maxDelta); + assert timestamp >= start && timestamp < end; + indexRequest.source("time", timestamp, "value", i); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); if (bulkResponse.hasFailures()) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java index 383edab2df133..09e3044e55cae 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java @@ -71,6 +71,7 @@ public void setUpData() { IndexRequest indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source("time", nowMillis - TimeValue.timeValueHours(2).millis(), "msg", "Node 1 started", "part", "nodes"); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source( "time", @@ -81,9 +82,11 @@ public void setUpData() { "shutdowns" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source("time", nowMillis - TimeValue.timeValueHours(1).millis(), "msg", "Node 2 started", "part", "nodes"); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source( "time", @@ -94,9 +97,11 @@ public void setUpData() { "shutdowns" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source("time", nowMillis, "msg", "Node 3 started", "part", "nodes"); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); BulkResponse bulkResponse = bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL).get(); assertThat(bulkResponse.hasFailures(), is(false)); @@ -449,6 +454,7 @@ public void testNumMatchesAndCategoryPreference() throws Exception { "2015-10-18 18:01:51,963 INFO [main] org.mortbay.log: jetty-6.1.26" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(index); indexRequest.source( "time", @@ -458,6 +464,7 @@ public void testNumMatchesAndCategoryPreference() throws Exception { + "Started HttpServer2$SelectChannelConnectorWithSafeStartup@0.0.0.0:62267" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(index); indexRequest.source( "time", @@ -466,6 +473,7 @@ public void testNumMatchesAndCategoryPreference() throws Exception { "2015-10-18 18:01:53,400 INFO [main] org.apache.hadoop.yarn.webapp.WebApps: Registered webapp guice modules" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(index); indexRequest.source( "time", @@ -475,6 +483,7 @@ public void testNumMatchesAndCategoryPreference() throws Exception { + "nodeBlacklistingEnabled:true" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(index); indexRequest.source( "time", @@ -483,6 +492,7 @@ public void testNumMatchesAndCategoryPreference() throws Exception { "2015-10-18 18:01:52,728 INFO [main] org.apache.hadoop.yarn.webapp.WebApps: Web app /mapreduce started at 62267" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(index); indexRequest.source( "time", @@ -492,6 +502,7 @@ public void testNumMatchesAndCategoryPreference() throws Exception { + "Connecting to ResourceManager at msra-sa-41/10.190.173.170:8030" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(index); indexRequest.source( @@ -502,6 +513,7 @@ public void testNumMatchesAndCategoryPreference() throws Exception { + "maxContainerCapability: " ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(index); indexRequest.source( @@ -512,6 +524,7 @@ public void testNumMatchesAndCategoryPreference() throws Exception { + "yarn.client.max-cached-nodemanagers-proxies : 0" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); BulkResponse bulkResponse = bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); assertThat(bulkResponse.hasFailures(), is(false)); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java index 1b60f238aa55e..e24b70c188f9f 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.aggregations.MultiBucketConsumerService.TooManyBucketsException; @@ -117,9 +118,10 @@ public void testEvaluate_AllMetrics_KeywordField_CaseSensitivity() { String actualField = "fieldA"; String predictedField = "fieldB"; client().admin().indices().prepareCreate(indexName).setMapping(actualField, "type=keyword", predictedField, "type=keyword").get(); - prepareIndex(indexName).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setSource(actualField, "crocodile", predictedField, "cRoCoDiLe") - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setSource(actualField, "crocodile", predictedField, "cRoCoDiLe"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); EvaluateDataFrameAction.Response evaluateDataFrameResponse = evaluateDataFrame( indexName, @@ -727,32 +729,37 @@ static void indexAnimalsData(String indexName) { put("class_probability", 0.4 - 0.1 * ix); } }).collect(toList()); - bulkRequestBuilder.add( - new IndexRequest(indexName).source( - ANIMAL_NAME_KEYWORD_FIELD, - animalNames.get(i), - ANIMAL_NAME_PREDICTION_KEYWORD_FIELD, - animalNames.get((i + j) % animalNames.size()), - ANIMAL_NAME_PREDICTION_PROB_FIELD, - animalNames.get((i + j) % animalNames.size()), - NO_LEGS_KEYWORD_FIELD, - String.valueOf(i + 1), - NO_LEGS_INTEGER_FIELD, - i + 1, - NO_LEGS_PREDICTION_INTEGER_FIELD, - j + 1, - IS_PREDATOR_KEYWORD_FIELD, - String.valueOf(i % 2 == 0), - IS_PREDATOR_BOOLEAN_FIELD, - i % 2 == 0, - IS_PREDATOR_PREDICTION_BOOLEAN_FIELD, - (i + j) % 2 == 0, - IS_PREDATOR_PREDICTION_PROBABILITY_FIELD, - i % 2 == 0 ? 1.0 - 0.1 * i : 0.1 * i, - ML_TOP_CLASSES_FIELD, - topClasses - ) - ); + IndexRequest indexRequest = new IndexRequest(indexName); + try { + bulkRequestBuilder.add( + indexRequest.source( + ANIMAL_NAME_KEYWORD_FIELD, + animalNames.get(i), + ANIMAL_NAME_PREDICTION_KEYWORD_FIELD, + animalNames.get((i + j) % animalNames.size()), + ANIMAL_NAME_PREDICTION_PROB_FIELD, + animalNames.get((i + j) % animalNames.size()), + NO_LEGS_KEYWORD_FIELD, + String.valueOf(i + 1), + NO_LEGS_INTEGER_FIELD, + i + 1, + NO_LEGS_PREDICTION_INTEGER_FIELD, + j + 1, + IS_PREDATOR_KEYWORD_FIELD, + String.valueOf(i % 2 == 0), + IS_PREDATOR_BOOLEAN_FIELD, + i % 2 == 0, + IS_PREDATOR_PREDICTION_BOOLEAN_FIELD, + (i + j) % 2 == 0, + IS_PREDATOR_PREDICTION_PROBABILITY_FIELD, + i % 2 == 0 ? 1.0 - 0.1 * i : 0.1 * i, + ML_TOP_CLASSES_FIELD, + topClasses + ) + ); + } finally { + indexRequest.decRef(); + } } } } @@ -766,14 +773,19 @@ static void indexAnimalsData(String indexName) { private static void indexDistinctAnimals(String indexName, int distinctAnimalCount) { try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int i = 0; i < distinctAnimalCount; i++) { - bulkRequestBuilder.add( - new IndexRequest(indexName).source( - ANIMAL_NAME_KEYWORD_FIELD, - "animal_" + i, - ANIMAL_NAME_PREDICTION_KEYWORD_FIELD, - randomAlphaOfLength(5) - ) - ); + IndexRequest indexRequest = new IndexRequest(indexName); + try { + bulkRequestBuilder.add( + indexRequest.source( + ANIMAL_NAME_KEYWORD_FIELD, + "animal_" + i, + ANIMAL_NAME_PREDICTION_KEYWORD_FIELD, + randomAlphaOfLength(5) + ) + ); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationHousePricingIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationHousePricingIT.java index e320129e78fdc..47a675fb8b189 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationHousePricingIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationHousePricingIT.java @@ -1625,6 +1625,7 @@ static void indexData(String sourceIndex) { ); IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java index 220ab5f3b8a97..efc7d2a4fdaa6 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java @@ -598,8 +598,13 @@ public void testDependentVariableCardinalityTooHighError() throws Exception { // Index enough documents to have more classes than the allowed limit try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int i = 0; i < Classification.MAX_DEPENDENT_VARIABLE_CARDINALITY - 1; i++) { - IndexRequest indexRequest = new IndexRequest(sourceIndex).source(KEYWORD_FIELD, "fox-" + i); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(sourceIndex); + try { + indexRequest.source(KEYWORD_FIELD, "fox-" + i); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -622,8 +627,13 @@ public void testDependentVariableCardinalityTooHighButWithQueryMakesItWithinRang // Index enough documents to have more classes than the allowed limit try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int i = 0; i < Classification.MAX_DEPENDENT_VARIABLE_CARDINALITY - 1; i++) { - IndexRequest indexRequest = new IndexRequest(sourceIndex).source(KEYWORD_FIELD, "fox-" + i); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(sourceIndex); + try { + indexRequest.source(KEYWORD_FIELD, "fox-" + i); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -1147,8 +1157,13 @@ static void indexData(String sourceIndex, int numTrainingRows, int numNonTrainin NESTED_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()) ); - IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(sourceIndex); + try { + indexRequest.source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } for (int i = numTrainingRows; i < numTrainingRows + numNonTrainingRows; i++) { List source = new ArrayList<>(); @@ -1173,8 +1188,13 @@ static void indexData(String sourceIndex, int numTrainingRows, int numNonTrainin source.addAll(List.of(NESTED_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()))); } source.addAll(List.of("@timestamp", "2020-12-12")); - IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(sourceIndex); + try { + indexRequest.source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalysisCustomFeatureIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalysisCustomFeatureIT.java index 579049788c602..790fdca5de928 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalysisCustomFeatureIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalysisCustomFeatureIT.java @@ -245,8 +245,13 @@ private static void indexData(String sourceIndex, int numTrainingRows, int numNo NESTED_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()) ); - IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(sourceIndex); + try { + indexRequest.source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } for (int i = numTrainingRows; i < numTrainingRows + numNonTrainingRows; i++) { List source = new ArrayList<>(); @@ -271,8 +276,13 @@ private static void indexData(String sourceIndex, int numTrainingRows, int numNo source.addAll(List.of(NESTED_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()))); } source.addAll(List.of("@timestamp", "2020-12-12")); - IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(sourceIndex); + try { + indexRequest.source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedWithAggsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedWithAggsIT.java index 96b4a5ba32bee..2acbb00972dd6 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedWithAggsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedWithAggsIT.java @@ -122,6 +122,7 @@ private void testDfWithAggs(AggregatorFactories.Builder aggs, Detector.Builder d IndexRequest indexRequest = new IndexRequest(dfId); indexRequest.source("time", curTime, "field", randomFrom("foo", "bar", "baz")); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); curTime += TimeValue.timeValueSeconds(1).millis(); } BulkResponse bulkResponse = bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DelayedDataDetectorIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DelayedDataDetectorIT.java index 226056b61f9af..feadbe4ba7f79 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DelayedDataDetectorIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DelayedDataDetectorIT.java @@ -223,10 +223,14 @@ private void writeData(Logger logger, String index, long numDocs, long start, lo try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { for (int i = 0; i < numDocs; i++) { IndexRequest indexRequest = new IndexRequest(index); - long timestamp = start + randomIntBetween(0, maxDelta); - assert timestamp >= start && timestamp < end; - indexRequest.source("time", timestamp, "value", i); - bulkRequestBuilder.add(indexRequest); + try { + long timestamp = start + randomIntBetween(0, maxDelta); + assert timestamp >= start && timestamp < end; + indexRequest.source("time", timestamp, "value", i); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); if (bulkResponse.hasFailures()) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index 54a5a8821a2f9..44126c7a5bfa4 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -85,8 +85,12 @@ public void setUpData() { int bucketRate = bucket == anomalousBucket ? anomalousRate : normalRate; for (int point = 0; point < bucketRate; point++) { IndexRequest indexRequest = new IndexRequest(DATA_INDEX); - indexRequest.source("time", timestamp); - bulkRequestBuilder.add(indexRequest); + try { + indexRequest.source("time", timestamp); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } } @@ -127,10 +131,10 @@ public void testDeleteExpiredDataNoThrottle() throws Exception { public void testDeleteExpiredDataActionDeletesEmptyStateIndices() throws Exception { client().admin().indices().prepareCreate(".ml-state").get(); client().admin().indices().prepareCreate(".ml-state-000001").get(); - prepareIndex(".ml-state-000001").setSource("field_1", "value_1").get(); + indexDoc(".ml-state-000001", null, "field_1", "value_1"); client().admin().indices().prepareCreate(".ml-state-000003").get(); client().admin().indices().prepareCreate(".ml-state-000005").get(); - prepareIndex(".ml-state-000005").setSource("field_5", "value_5").get(); + indexDoc(".ml-state-000005", null, "field_5", "value_5"); client().admin().indices().prepareCreate(".ml-state-000007").addAlias(new Alias(".ml-state-write").isHidden(true)).get(); refresh(); @@ -164,8 +168,13 @@ private void testExpiredDeletion(Float customThrottle, int numUnusedState) throw try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int i = 0; i < numUnusedState; i++) { String docId = "non_existing_job_" + randomFrom("model_state_1234567#" + i, "quantiles", "categorizer_state#" + i); - IndexRequest indexRequest = new IndexRequest(mlStateIndexName).id(docId).source(Collections.emptyMap()); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(mlStateIndexName); + try { + indexRequest.id(docId).source(Collections.emptyMap()); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } ActionFuture indexUnusedStateDocsResponse = bulkRequestBuilder.execute(); List jobs = new ArrayList<>(); @@ -233,8 +242,12 @@ private void testExpiredDeletion(Float customThrottle, int numUnusedState) throw // Update snapshot timestamp to force it out of snapshot retention window String snapshotUpdate = "{ \"timestamp\": " + oneDayAgo + "}"; UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + job.getId(), snapshotDocId); - updateSnapshotRequest.doc(snapshotUpdate.getBytes(StandardCharsets.UTF_8), XContentType.JSON); - client().execute(TransportUpdateAction.TYPE, updateSnapshotRequest).get(); + try { + updateSnapshotRequest.doc(snapshotUpdate.getBytes(StandardCharsets.UTF_8), XContentType.JSON); + client().execute(TransportUpdateAction.TYPE, updateSnapshotRequest).get(); + } finally { + updateSnapshotRequest.decRef(); + } // Now let's create some forecasts openJob(job.getId()); @@ -404,19 +417,15 @@ public void testDeleteExpiresDataDeletesAnnotations() throws Exception { assertThatNumberOfAnnotationsIsEqualTo(1); // The following 4 annotations are created by the system and the 2 oldest ones *will* be deleted - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(1)), InternalUsers.XPACK_USER.principal())) - .actionGet(); - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(2)), InternalUsers.XPACK_USER.principal())) - .actionGet(); - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(3)), InternalUsers.XPACK_USER.principal())) - .actionGet(); - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(4)), InternalUsers.XPACK_USER.principal())) - .actionGet(); + indexRandomAnnotation(jobId, now.minus(Duration.ofDays(1)), InternalUsers.XPACK_USER.principal()); + indexRandomAnnotation(jobId, now.minus(Duration.ofDays(2)), InternalUsers.XPACK_USER.principal()); + indexRandomAnnotation(jobId, now.minus(Duration.ofDays(3)), InternalUsers.XPACK_USER.principal()); + indexRandomAnnotation(jobId, now.minus(Duration.ofDays(4)), InternalUsers.XPACK_USER.principal()); // The following 4 annotations are created by the user and *will not* be deleted - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(1)), USER_NAME)).actionGet(); - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(2)), USER_NAME)).actionGet(); - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(3)), USER_NAME)).actionGet(); - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(4)), USER_NAME)).actionGet(); + indexRandomAnnotation(jobId, now.minus(Duration.ofDays(1)), USER_NAME); + indexRandomAnnotation(jobId, now.minus(Duration.ofDays(2)), USER_NAME); + indexRandomAnnotation(jobId, now.minus(Duration.ofDays(3)), USER_NAME); + indexRandomAnnotation(jobId, now.minus(Duration.ofDays(4)), USER_NAME); assertThatNumberOfAnnotationsIsEqualTo(9); @@ -426,14 +435,18 @@ public void testDeleteExpiresDataDeletesAnnotations() throws Exception { assertThatNumberOfAnnotationsIsEqualTo(7); } - private static IndexRequest randomAnnotationIndexRequest(String jobId, Instant timestamp, String createUsername) throws IOException { + private static void indexRandomAnnotation(String jobId, Instant timestamp, String createUsername) throws IOException { Annotation annotation = new Annotation.Builder(randomAnnotation(jobId)).setTimestamp(Date.from(timestamp)) .setCreateUsername(createUsername) .build(); try (XContentBuilder xContentBuilder = annotation.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) { - return new IndexRequest(AnnotationIndex.WRITE_ALIAS_NAME).source(xContentBuilder) - .setRequireAlias(true) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexRequest indexRequest = new IndexRequest(AnnotationIndex.WRITE_ALIAS_NAME); + try { + indexRequest.source(xContentBuilder).setRequireAlias(true).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client().index(indexRequest).actionGet(); + } finally { + indexRequest.decRef(); + } } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java index 40f0167e5121a..7fff22042b102 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java @@ -55,15 +55,15 @@ public void testDeleteJobDeletesAnnotations() throws Exception { assertThatNumberOfAnnotationsIsEqualTo(0); runJob(jobIdA, datafeedIdA); - client().index(randomAnnotationIndexRequest(jobIdA, InternalUsers.XPACK_USER.principal())).actionGet(); - client().index(randomAnnotationIndexRequest(jobIdA, InternalUsers.XPACK_USER.principal())).actionGet(); - client().index(randomAnnotationIndexRequest(jobIdA, "real_user")).actionGet(); + indexRandomAnnotation(jobIdA, InternalUsers.XPACK_USER.principal()); + indexRandomAnnotation(jobIdA, InternalUsers.XPACK_USER.principal()); + indexRandomAnnotation(jobIdA, "real_user"); // 3 jobA annotations (2 _xpack, 1 real_user) assertThatNumberOfAnnotationsIsEqualTo(3); runJob(jobIdB, datafeedIdB); - client().index(randomAnnotationIndexRequest(jobIdB, InternalUsers.XPACK_USER.principal())).actionGet(); - client().index(randomAnnotationIndexRequest(jobIdB, "other_real_user")).actionGet(); + indexRandomAnnotation(jobIdB, InternalUsers.XPACK_USER.principal()); + indexRandomAnnotation(jobIdB, "other_real_user"); // 3 jobA annotations (2 _xpack, 1 real_user) and 2 jobB annotations (1 _xpack, 1 real_user) assertThatNumberOfAnnotationsIsEqualTo(5); @@ -109,12 +109,16 @@ private void runJob(String jobId, String datafeedId) throws Exception { waitUntilJobIsClosed(jobId); } - private static IndexRequest randomAnnotationIndexRequest(String jobId, String createUsername) throws IOException { + private static void indexRandomAnnotation(String jobId, String createUsername) throws IOException { Annotation annotation = new Annotation.Builder(randomAnnotation(jobId)).setCreateUsername(createUsername).build(); try (XContentBuilder xContentBuilder = annotation.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) { - return new IndexRequest(AnnotationIndex.WRITE_ALIAS_NAME).source(xContentBuilder) - .setRequireAlias(true) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexRequest indexRequest = new IndexRequest(AnnotationIndex.WRITE_ALIAS_NAME); + try { + indexRequest.source(xContentBuilder).setRequireAlias(true).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client().index(indexRequest).actionGet(); + } finally { + indexRequest.decRef(); + } } } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsIT.java index 1a5a11fe595ff..32cb2e284cbc3 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsIT.java @@ -87,6 +87,7 @@ public void testSourceQueryIsApplied() throws IOException { i < 2 ? "bingo" : "rest" ); // We tag bingo on the first two docs to ensure we have 2 classes bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -249,6 +250,7 @@ public void testRuntimeFields() { Object[] source = new Object[] { "mapped_field", i }; IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source).opType(DocWriteRequest.OpType.CREATE); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java index db0e3de8f16c1..5a91473dbadc0 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java @@ -54,6 +54,7 @@ public void setUpData() { IndexRequest indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source("time", timestamp, "user", user); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelSnapshotRetentionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelSnapshotRetentionIT.java index 078aa27ae368d..1cd5614f22c43 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelSnapshotRetentionIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelSnapshotRetentionIT.java @@ -230,15 +230,19 @@ private void persistModelSnapshotDoc(String jobId, String snapshotId, Date times IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.resultsWriteAlias(jobId)).id( ModelSnapshot.documentId(jobId, snapshotId) ).setRequireAlias(true); - if (immediateRefresh) { - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + try { + if (immediateRefresh) { + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + } + XContentBuilder xContentBuilder = JsonXContent.contentBuilder(); + modelSnapshotBuilder.build().toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + indexRequest.source(xContentBuilder); + + DocWriteResponse indexResponse = client().execute(TransportIndexAction.TYPE, indexRequest).actionGet(); + assertThat(indexResponse.getResult(), is(DocWriteResponse.Result.CREATED)); + } finally { + indexRequest.decRef(); } - XContentBuilder xContentBuilder = JsonXContent.contentBuilder(); - modelSnapshotBuilder.build().toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); - indexRequest.source(xContentBuilder); - - DocWriteResponse indexResponse = client().execute(TransportIndexAction.TYPE, indexRequest).actionGet(); - assertThat(indexResponse.getResult(), is(DocWriteResponse.Result.CREATED)); } private void persistModelStateDocs(String jobId, String snapshotId, int numDocs) { @@ -253,6 +257,7 @@ private void persistModelStateDocs(String jobId, String snapshotId, int numDocs) .source(Collections.singletonMap("compressed", Collections.singletonList("foo"))) .setRequireAlias(true); bulkRequest.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = client().execute(BulkAction.INSTANCE, bulkRequest).actionGet(); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelSnapshotSearchIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelSnapshotSearchIT.java index 4b238393d50df..ca901bec796c8 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelSnapshotSearchIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelSnapshotSearchIT.java @@ -119,15 +119,19 @@ private void persistModelSnapshotDoc(String jobId, String snapshotId, Date times IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.resultsWriteAlias(jobId)).id( ModelSnapshot.documentId(jobId, snapshotId) ).setRequireAlias(true); - if (immediateRefresh) { - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + try { + if (immediateRefresh) { + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + } + XContentBuilder xContentBuilder = JsonXContent.contentBuilder(); + modelSnapshotBuilder.build().toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + indexRequest.source(xContentBuilder); + + DocWriteResponse indexResponse = client().execute(TransportIndexAction.TYPE, indexRequest).actionGet(); + assertThat(indexResponse.getResult(), is(DocWriteResponse.Result.CREATED)); + } finally { + indexRequest.decRef(); } - XContentBuilder xContentBuilder = JsonXContent.contentBuilder(); - modelSnapshotBuilder.build().toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); - indexRequest.source(xContentBuilder); - - DocWriteResponse indexResponse = client().execute(TransportIndexAction.TYPE, indexRequest).actionGet(); - assertThat(indexResponse.getResult(), is(DocWriteResponse.Result.CREATED)); } private void persistModelStateDocs(String jobId, String snapshotId, int numDocs) { @@ -142,6 +146,7 @@ private void persistModelStateDocs(String jobId, String snapshotId, int numDocs) .source(Collections.singletonMap("compressed", Collections.singletonList("foo"))) .setRequireAlias(true); bulkRequest.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = client().execute(BulkAction.INSTANCE, bulkRequest).actionGet(); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java index fa3e828030727..6593bac6085bc 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java @@ -46,6 +46,7 @@ public void testMissingFields() throws Exception { IndexRequest indexRequest = new IndexRequest(sourceIndex); indexRequest.source("numeric", 42.0); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } // Add a doc with missing field @@ -53,6 +54,7 @@ public void testMissingFields() throws Exception { IndexRequest missingIndexRequest = new IndexRequest(sourceIndex); missingIndexRequest.source("categorical", "foo"); bulkRequestBuilder.add(missingIndexRequest); + missingIndexRequest.decRef(); } // Add a doc with numeric being array which is also treated as missing @@ -60,6 +62,7 @@ public void testMissingFields() throws Exception { IndexRequest arrayIndexRequest = new IndexRequest(sourceIndex); arrayIndexRequest.source("numeric", new double[] { 1.0, 2.0 }, "categorical", "foo"); bulkRequestBuilder.add(arrayIndexRequest); + arrayIndexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java index 48364dd9da148..2537d1466bd9b 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java @@ -574,8 +574,13 @@ public void testAliasFields() throws Exception { try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { for (int i = 0; i < totalDocCount; i++) { List source = List.of("field_1", i, "field_2", 2 * i); - IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(sourceIndex); + try { + indexRequest.source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -927,8 +932,13 @@ static void indexData(String sourceIndex, int numTrainingRows, int numNonTrainin "@timestamp", Instant.now().toEpochMilli() ); - IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(sourceIndex); + try { + indexRequest.source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } for (int i = numTrainingRows; i < numTrainingRows + numNonTrainingRows; i++) { List source = List.of( @@ -939,8 +949,13 @@ static void indexData(String sourceIndex, int numTrainingRows, int numNonTrainin "@timestamp", Instant.now().toEpochMilli() ); - IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); - bulkRequestBuilder.add(indexRequest); + IndexRequest indexRequest = new IndexRequest(sourceIndex); + try { + indexRequest.source(source.toArray()).opType(DocWriteRequest.OpType.CREATE); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java index 260a5dea0a3c1..24ee5d54683ac 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java @@ -191,9 +191,9 @@ private void testRunJobInTwoPartsAndRevertSnapshotAndRunToCompletion(String jobI // Add 3 new annotations... Instant lastResultTimestamp = revertSnapshot.getLatestResultTimeStamp().toInstant(); - client().index(randomAnnotationIndexRequest(job.getId(), lastResultTimestamp.plusSeconds(10), Event.DELAYED_DATA)).actionGet(); - client().index(randomAnnotationIndexRequest(job.getId(), lastResultTimestamp.plusSeconds(20), Event.MODEL_CHANGE)).actionGet(); - client().index(randomAnnotationIndexRequest(job.getId(), lastResultTimestamp.minusSeconds(10), Event.MODEL_CHANGE)).actionGet(); + indexRandomAnnotation(job.getId(), lastResultTimestamp.plusSeconds(10), Event.DELAYED_DATA); + indexRandomAnnotation(job.getId(), lastResultTimestamp.plusSeconds(20), Event.MODEL_CHANGE); + indexRandomAnnotation(job.getId(), lastResultTimestamp.minusSeconds(10), Event.MODEL_CHANGE); // ... and check there are 5 annotations in total now assertThatNumberOfAnnotationsIsEqualTo(5); @@ -309,15 +309,20 @@ private Quantiles getQuantiles(String jobId) throws Exception { return quantilesSetOnce.get(); } - private static IndexRequest randomAnnotationIndexRequest(String jobId, Instant timestamp, Event event) throws IOException { + private void indexRandomAnnotation(String jobId, Instant timestamp, Event event) throws IOException { Annotation annotation = new Annotation.Builder(randomAnnotation(jobId)).setTimestamp(Date.from(timestamp)) .setCreateUsername(InternalUsers.XPACK_USER.principal()) .setEvent(event) .build(); try (XContentBuilder xContentBuilder = annotation.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) { - return new IndexRequest(AnnotationIndex.WRITE_ALIAS_NAME).source(xContentBuilder) - .setRequireAlias(true) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexRequest indexRequest = new IndexRequest(AnnotationIndex.WRITE_ALIAS_NAME); + try { + client().index( + indexRequest.source(xContentBuilder).setRequireAlias(true).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + ).actionGet(); + } finally { + indexRequest.decRef(); + } } } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java index 1d021a3b8aaef..edf2f2a96c8f0 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java @@ -77,6 +77,7 @@ public void testOutlierDetectionWithFewDocuments() throws Exception { indexRequest.id(docId); indexRequest.source("numeric_1", i == 0 ? 100.0 : 1.0, "numeric_2", 1, "categorical_1", "foo_" + i); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -174,6 +175,7 @@ public void testPreview() throws Exception { indexRequest.id(docId); indexRequest.source("numeric_1", i == 0 ? 100.0 : 1.0, "numeric_2", 1, "categorical_1", "foo_" + i); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -212,6 +214,7 @@ public void testOutlierDetectionWithEnoughDocumentsToScroll() throws Exception { IndexRequest indexRequest = new IndexRequest(sourceIndex); indexRequest.source("numeric_1", randomDouble(), "numeric_2", randomFloat(), "categorical_1", randomAlphaOfLength(10)); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -295,6 +298,7 @@ public void testOutlierDetectionWithMoreFieldsThanDocValueFieldLimit() throws Ex IndexRequest indexRequest = new IndexRequest(sourceIndex); indexRequest.source(source.toString(), XContentType.JSON); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -372,6 +376,7 @@ public void testStopOutlierDetectionWithEnoughDocumentsToScroll() throws Excepti IndexRequest indexRequest = new IndexRequest(sourceIndex); indexRequest.source("numeric_1", randomDouble(), "numeric_2", randomFloat(), "categorical_1", randomAlphaOfLength(10)); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -446,6 +451,7 @@ public void testOutlierDetectionWithMultipleSourceIndices() throws Exception { IndexRequest indexRequest = new IndexRequest(index); indexRequest.source("numeric_1", randomDouble(), "numeric_2", randomFloat(), "categorical_1", "foo_" + i); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } } BulkResponse bulkResponse = bulkRequestBuilder.get(); @@ -514,6 +520,7 @@ public void testOutlierDetectionWithPreExistingDestIndex() throws Exception { IndexRequest indexRequest = new IndexRequest(sourceIndex); indexRequest.source("numeric_1", randomDouble(), "numeric_2", randomFloat(), "categorical_1", "foo_" + i); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -566,6 +573,7 @@ public void testModelMemoryLimitLowerThanEstimatedMemoryUsage() throws Exception for (int i = 0; i < 10000; i++) { // This number of rows should make memory usage estimate greater than 1MB IndexRequest indexRequest = new IndexRequest(sourceIndex).id("doc_" + i).source("col_1", 1.0, "col_2", 1.0, "col_3", "str"); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -599,6 +607,7 @@ public void testLazyAssignmentWithModelMemoryLimitTooHighForAssignment() throws try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) { IndexRequest indexRequest = new IndexRequest(sourceIndex).id("doc_1").source("col_1", 1.0, "col_2", 1.0, "col_3", "str"); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { fail("Failed to index data: " + bulkResponse.buildFailureMessage()); @@ -658,6 +667,7 @@ public void testOutlierDetectionStopAndRestart() throws Exception { IndexRequest indexRequest = new IndexRequest(sourceIndex); indexRequest.source("numeric_1", randomDouble(), "numeric_2", randomFloat(), "categorical_1", randomAlphaOfLength(10)); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -730,6 +740,7 @@ public void testOutlierDetectionWithCustomParams() throws Exception { indexRequest.id(docId); indexRequest.source("numeric_1", i == 0 ? 100.0 : 1.0, "numeric_2", 1.0, "categorical_1", "foo_" + i); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -842,6 +853,7 @@ public void testOutlierDetection_GivenIndexWithRuntimeFields() throws Exception indexRequest.id(docId); indexRequest.source("numeric", i == 0 ? 100.0 : 1.0); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -944,6 +956,7 @@ public void testOutlierDetection_GivenSearchRuntimeMappings() throws Exception { indexRequest.id(docId); indexRequest.source("numeric", i == 0 ? 100.0 : 1.0); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { @@ -1046,6 +1059,7 @@ public void testStart_GivenTimeout_Returns408() throws Exception { indexRequest.id(String.valueOf(i)); indexRequest.source("numeric_1", randomInt(), "numeric_2", randomInt()); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } BulkResponse bulkResponse = bulkRequestBuilder.get(); if (bulkResponse.hasFailures()) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureResetIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureResetIT.java index 7158e494bee68..ab18a59e53f07 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureResetIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureResetIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateAction; import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.DeletePipelineTransportAction; import org.elasticsearch.action.ingest.PutPipelineRequest; @@ -308,7 +309,12 @@ private void putTrainedModelIngestPipeline(String pipelineId) throws Exception { } private void indexDocForInference(String pipelineId) { - prepareIndex("foo").setPipeline(pipelineId).setSource("{\"text\": \"this is some plain text.\"}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("foo"); + try { + indexRequestBuilder.setPipeline(pipelineId).setSource("{\"text\": \"this is some plain text.\"}", XContentType.JSON).get(); + } finally { + indexRequestBuilder.request().decRef(); + } } } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/license/MachineLearningLicensingIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/license/MachineLearningLicensingIT.java index 7d2de13f967ae..fd3e779bfc23b 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/license/MachineLearningLicensingIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/license/MachineLearningLicensingIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.ingest.PutPipelineTransportAction; import org.elasticsearch.action.ingest.SimulateDocumentBaseResult; @@ -541,7 +542,7 @@ public void testMachineLearningCreateInferenceProcessorRestricted() { AcknowledgedResponse putPipelineResponse = putPipelineListener.actionGet(); assertTrue(putPipelineResponse.isAcknowledged()); - prepareIndex("infer_license_test").setPipeline("test_infer_license_pipeline").setSource("{}", XContentType.JSON).get(); + index("infer_license_test", "test_infer_license_pipeline", "{}", XContentType.JSON); String simulateSource = Strings.format(""" { @@ -570,7 +571,7 @@ public void testMachineLearningCreateInferenceProcessorRestricted() { // Inference against the previous pipeline should still work try { - prepareIndex("infer_license_test").setPipeline("test_infer_license_pipeline").setSource("{}", XContentType.JSON).get(); + index("infer_license_test", "test_infer_license_pipeline", "{}", XContentType.JSON); } catch (ElasticsearchSecurityException ex) { fail(ex.getMessage()); } @@ -591,7 +592,7 @@ public void testMachineLearningCreateInferenceProcessorRestricted() { // Inference against the new pipeline should fail since it has never previously succeeded ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> { - prepareIndex("infer_license_test").setPipeline("test_infer_license_pipeline_again").setSource("{}", XContentType.JSON).get(); + index("infer_license_test", "test_infer_license_pipeline_again", "{}", XContentType.JSON); }); assertThat(e.status(), is(RestStatus.FORBIDDEN)); assertThat(e.getMessage(), containsString("non-compliant")); @@ -634,8 +635,8 @@ public void testMachineLearningCreateInferenceProcessorRestricted() { // both ingest pipelines should work - prepareIndex("infer_license_test").setPipeline("test_infer_license_pipeline").setSource("{}", XContentType.JSON).get(); - prepareIndex("infer_license_test").setPipeline("test_infer_license_pipeline_again").setSource("{}", XContentType.JSON).get(); + index("infer_license_test", "test_infer_license_pipeline", "{}", XContentType.JSON); + index("infer_license_test", "test_infer_license_pipeline_again", "{}", XContentType.JSON); } public void testMachineLearningInferModelRestricted() { @@ -727,12 +728,17 @@ public void testInferenceAggRestricted() { String index = "inference-agg-licence-test"; client().admin().indices().prepareCreate(index).setMapping("feature1", "type=double", "feature2", "type=keyword").get(); try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(index)) { - bulkRequestBuilder.add(new IndexRequest().source("feature1", "10.0", "feature2", "foo")) - .add(new IndexRequest().source("feature1", "20.0", "feature2", "foo")) - .add(new IndexRequest().source("feature1", "20.0", "feature2", "bar")) - .add(new IndexRequest().source("feature1", "20.0", "feature2", "bar")) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); + List indexRequests = List.of( + new IndexRequest().source("feature1", "10.0", "feature2", "foo"), + new IndexRequest().source("feature1", "20.0", "feature2", "foo"), + new IndexRequest().source("feature1", "20.0", "feature2", "bar"), + new IndexRequest().source("feature1", "20.0", "feature2", "bar") + ); + for (IndexRequest indexRequest : indexRequests) { + bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); + } + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); } TermsAggregationBuilder termsAgg = new TermsAggregationBuilder("foobar").field("feature2"); @@ -834,4 +840,14 @@ public static void enableLicensing(License.OperationMode operationMode) { licenseState.update(new XPackLicenseStatus(operationMode, true, null)); } } + + private void index(String index, String pipeline, String source, XContentType contentType) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + indexRequestBuilder.setPipeline(pipeline).setSource(source, contentType); + indexRequestBuilder.get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java index 9daf353b11380..882c09ada7d9d 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java @@ -304,6 +304,7 @@ public void testNotCreatedWhenAfterOtherMlIndexAndResetInProgress() throws Excep IndexRequest stateDoc = new IndexRequest(".ml-state"); stateDoc.source(Collections.singletonMap("state", "blah")); DocWriteResponse indexResponse = client().index(stateDoc).actionGet(); + stateDoc.decRef(); assertEquals(RestStatus.CREATED, indexResponse.status()); // Creating the .ml-state index would normally cause .ml-annotations diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java index 35d99b97f99c9..38ced4613fcfb 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java @@ -113,35 +113,32 @@ public void testUpdateModelMemoryLimitOnceEstablished() { public void testCreateWithExistingCategorizerDocs() { String jobId = "job-id-with-existing-docs"; - testCreateWithExistingDocs( - prepareIndex(".ml-state-000001").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setId(jobId + "_categorizer_state#1") - .setSource("{}", XContentType.JSON) - .request(), - jobId - ); + IndexRequest indexRequest = prepareIndex(".ml-state-000001").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setId(jobId + "_categorizer_state#1") + .setSource("{}", XContentType.JSON) + .request(); + testCreateWithExistingDocs(indexRequest, jobId); + indexRequest.decRef(); } public void testCreateWithExistingQuantilesDocs() { String jobId = "job-id-with-existing-docs"; - testCreateWithExistingDocs( - prepareIndex(".ml-state-000001").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setId(jobId + "_quantiles") - .setSource("{}", XContentType.JSON) - .request(), - jobId - ); + IndexRequest indexRequest = prepareIndex(".ml-state-000001").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setId(jobId + "_quantiles") + .setSource("{}", XContentType.JSON) + .request(); + testCreateWithExistingDocs(indexRequest, jobId); + indexRequest.decRef(); } public void testCreateWithExistingResultsDocs() { String jobId = "job-id-with-existing-docs"; - testCreateWithExistingDocs( - prepareIndex(".ml-anomalies-shared").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setId(jobId + "_1464739200000_1") - .setSource("{\"job_id\": \"" + jobId + "\"}", XContentType.JSON) - .request(), - jobId - ); + IndexRequest indexRequest = prepareIndex(".ml-anomalies-shared").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setId(jobId + "_1464739200000_1") + .setSource("{\"job_id\": \"" + jobId + "\"}", XContentType.JSON) + .request(); + testCreateWithExistingDocs(indexRequest, jobId); + indexRequest.decRef(); } public void testPutJobWithClosedResultsIndex() { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index c62840cb8b199..99bf2d4e8eafa 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -182,12 +182,15 @@ public void testJobAutoClose() throws Exception { IndexRequest indexRequest = new IndexRequest("data"); indexRequest.source("time", 1407081600L); client().index(indexRequest).get(); + indexRequest.decRef(); indexRequest = new IndexRequest("data"); indexRequest.source("time", 1407082600L); client().index(indexRequest).get(); + indexRequest.decRef(); indexRequest = new IndexRequest("data"); indexRequest.source("time", 1407083600L); client().index(indexRequest).get(); + indexRequest.decRef(); refresh("*"); Job.Builder job = createScheduledJob("job_id"); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BucketCorrelationAggregationIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BucketCorrelationAggregationIT.java index a6e2b8b91a58d..42d07b0e42e20 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BucketCorrelationAggregationIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BucketCorrelationAggregationIT.java @@ -51,6 +51,7 @@ public void testCountCorrelation() { isDog[i] = 0; indexRequest.source("metric", x, "term", "cat").opType(DocWriteRequest.OpType.CREATE); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } sendAndMaybeFail(bulkRequestBuilder); } @@ -64,6 +65,7 @@ public void testCountCorrelation() { isDog[i] = 1; indexRequest.source("metric", x, "term", "dog").opType(DocWriteRequest.OpType.CREATE); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); } sendAndMaybeFail(bulkRequestBuilder); } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/CategorizeTextAggregationIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/CategorizeTextAggregationIT.java index b35c858cea79f..e29477e1ae1a1 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/CategorizeTextAggregationIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/CategorizeTextAggregationIT.java @@ -120,6 +120,7 @@ private void createSourceData() { IndexRequest indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source("time", nowMillis - TimeValue.timeValueHours(2).millis(), "msg", "Node 1 started", "part", "nodes"); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source( "time", @@ -130,6 +131,7 @@ private void createSourceData() { "shutdowns" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source( "time", @@ -140,16 +142,20 @@ private void createSourceData() { "shutdowns" ); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source("time", nowMillis - TimeValue.timeValueHours(1).millis(), "msg", "Node 2 started", "part", "nodes"); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source("time", nowMillis, "msg", "Node 3 started", "part", "nodes"); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); indexRequest = new IndexRequest(DATA_INDEX); indexRequest.source("time", nowMillis, "msg", "Node 3 stopped", "part", "nodes"); bulkRequestBuilder.add(indexRequest); + indexRequest.decRef(); BulkResponse bulkResponse = bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); assertThat(bulkResponse.hasFailures(), is(false)); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/CategorizeTextDistributedIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/CategorizeTextDistributedIT.java index cd5c67c06af6f..62f95975c28d1 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/CategorizeTextDistributedIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/CategorizeTextDistributedIT.java @@ -61,18 +61,22 @@ public void testDistributedCategorizeText() { Map.of("message", "Aug 29, 2019 2:02:51 PM org.apache.coyote.http11.Http11BaseProtocol destroy") ); bulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); indexRequestBuilder = prepareIndex(indexName).setSource( Map.of("message", "Aug 29, 2019 2:02:51 PM org.apache.coyote.http11.Http11BaseProtocol init") ); bulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); indexRequestBuilder = prepareIndex(indexName).setSource( Map.of("message", "Aug 29, 2019 2:02:51 PM org.apache.coyote.http11.Http11BaseProtocol start") ); bulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); indexRequestBuilder = prepareIndex(indexName).setSource( Map.of("message", "Aug 29, 2019 2:02:51 PM org.apache.coyote.http11.Http11BaseProtocol stop") ); bulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } bulkRequestBuilder.get(); } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ChunkedTrainedModelRestorerIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ChunkedTrainedModelRestorerIT.java index 06404d14f7594..59fe097e9379c 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ChunkedTrainedModelRestorerIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ChunkedTrainedModelRestorerIT.java @@ -206,6 +206,7 @@ private void putModelDefinitions(List docs, String in .setId(TrainedModelDefinitionDoc.docId(doc.getModelId(), startingDocNum++)); bulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalyticsCRUDIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalyticsCRUDIT.java index a8e97263647ea..e6af7c5be0d39 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalyticsCRUDIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalyticsCRUDIT.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.ml.integration; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; @@ -76,22 +78,28 @@ public void testDeleteConfigWithStateAndStats() throws InterruptedException { assertThat(configHolder.get(), is(equalTo(config))); OriginSettingClient originSettingClient = new OriginSettingClient(client(), ClientHelper.ML_ORIGIN); - originSettingClient.prepareIndex(".ml-state-000001") - .setId("delete-config-with-state-and-stats_regression_state#1") - .setSource("{}", XContentType.JSON) - .get(); - originSettingClient.prepareIndex(".ml-state-000001") - .setId("data_frame_analytics-delete-config-with-state-and-stats-progress") - .setSource("{}", XContentType.JSON) - .get(); - originSettingClient.prepareIndex(".ml-stats-000001") - .setId("delete-config-with-state-and-stats_1") - .setSource("{\"job_id\": \"delete-config-with-state-and-stats\"}", XContentType.JSON) - .get(); - originSettingClient.prepareIndex(".ml-stats-000001") - .setId("delete-config-with-state-and-stats_2") - .setSource("{\"job_id\": \"delete-config-with-state-and-stats\"}", XContentType.JSON) - .get(); + index(originSettingClient, ".ml-state-000001", "delete-config-with-state-and-stats_regression_state#1", "{}", XContentType.JSON); + index( + originSettingClient, + ".ml-state-000001", + "data_frame_analytics-delete-config-with-state-and-stats-progress", + "{}", + XContentType.JSON + ); + index( + originSettingClient, + ".ml-stats-000001", + "delete-config-with-state-and-stats_1", + "{\"job_id\": \"delete-config-with-state-and-stats\"}", + XContentType.JSON + ); + index( + originSettingClient, + ".ml-stats-000001", + "delete-config-with-state-and-stats_2", + "{\"job_id\": \"delete-config-with-state-and-stats\"}", + XContentType.JSON + ); originSettingClient.admin().indices().prepareRefresh(".ml-stat*").get(); @@ -118,4 +126,13 @@ public void testDeleteConfigWithStateAndStats() throws InterruptedException { ); } + private void index(Client client, String index, String id, String source, XContentType contentType) { + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source, contentType).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index 55ea5deffe1af..6d1a48d70d6c9 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -793,10 +794,12 @@ public void testGetSnapshots() { .build() ); // Add a snapshot WITHOUT a min version. - prepareIndex(AnomalyDetectorsIndex.jobResultsAliasedName("other_job")).setId(ModelSnapshot.documentId("other_job", "11")) - .setSource(""" - {"job_id":"other_job","snapshot_id":"11", "snapshot_doc_count":1,"retain":false}""", XContentType.JSON) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(AnomalyDetectorsIndex.jobResultsAliasedName("other_job")).setId( + ModelSnapshot.documentId("other_job", "11") + ).setSource(""" + {"job_id":"other_job","snapshot_id":"11", "snapshot_doc_count":1,"retain":false}""", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); indicesAdmin().prepareRefresh(AnomalyDetectorsIndex.jobStateIndexPattern(), AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*") .get(); @@ -1048,6 +1051,7 @@ private void indexScheduledEvents(List events) throws IOExceptio ); indexRequest.source(event.toXContent(builder, params)); bulkRequest.add(indexRequest); + indexRequest.decRef(); } } BulkResponse response = bulkRequest.get(); @@ -1074,6 +1078,7 @@ private void indexFilters(List filters) throws IOException { ); indexRequest.source(filter.toXContent(builder, params)); bulkRequest.add(indexRequest); + indexRequest.decRef(); } } bulkRequest.get(); @@ -1125,6 +1130,7 @@ private void indexCalendars(List calendars) throws IOException { ); indexRequest.source(calendar.toXContent(builder, params)); bulkRequest.add(indexRequest); + indexRequest.decRef(); } } bulkRequest.get(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobsAndModelsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobsAndModelsIT.java index 7f0a4cc381ee2..089e3d7364bf0 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobsAndModelsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobsAndModelsIT.java @@ -107,11 +107,10 @@ public void testCluster_GivenAnomalyDetectionJobAndTrainedModelDeployment_Should ); try (XContentBuilder builder = JsonXContent.contentBuilder()) { modelDefinitionDoc.toXContent(builder, null); - client().execute( - TransportIndexAction.TYPE, - new IndexRequest(InferenceIndexConstants.nativeDefinitionStore()).source(builder) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - ).actionGet(); + IndexRequest indexRequest = new IndexRequest(InferenceIndexConstants.nativeDefinitionStore()); + client().execute(TransportIndexAction.TYPE, indexRequest.source(builder).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)) + .actionGet(); + indexRequest.decRef(); } client().execute(PutTrainedModelAction.INSTANCE, new PutTrainedModelAction.Request(model, true)).actionGet(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlAutoUpdateServiceIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlAutoUpdateServiceIT.java index dc0c090900393..1a320d0d86df8 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlAutoUpdateServiceIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlAutoUpdateServiceIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -74,9 +75,11 @@ public void createComponents() throws Exception { public void testAutomaticModelUpdate() throws Exception { ensureGreen("_all"); IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); - prepareIndex(MlConfigIndex.indexName()).setId(DatafeedConfig.documentId("farequote-datafeed-with-old-agg")) - .setSource(AGG_WITH_OLD_DATE_HISTOGRAM_INTERVAL, XContentType.JSON) - .get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex(MlConfigIndex.indexName()).setId( + DatafeedConfig.documentId("farequote" + "-datafeed-with-old-agg") + ).setSource(AGG_WITH_OLD_DATE_HISTOGRAM_INTERVAL, XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); AtomicReference getConfigHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index 2d52100935861..2859cd4680064 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -813,10 +813,14 @@ private void indexModelSnapshotFromCurrentJobStats(String jobId) throws IOExcept try (XContentBuilder xContentBuilder = JsonXContent.contentBuilder()) { modelSnapshot.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.jobResultsAliasedName(jobId)); - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - indexRequest.id(ModelSnapshot.documentId(modelSnapshot)); - indexRequest.source(xContentBuilder); - client().index(indexRequest).actionGet(); + try { + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequest.id(ModelSnapshot.documentId(modelSnapshot)); + indexRequest.source(xContentBuilder); + client().index(indexRequest).actionGet(); + } finally { + indexRequest.decRef(); + } } JobUpdate jobUpdate = new JobUpdate.Builder(jobId).setModelSnapshotId(modelSnapshot.getSnapshotId()).build(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelProviderIT.java index 394d6705ce0c3..659d3a013ae29 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelProviderIT.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.ml.integration; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -321,14 +322,12 @@ public void testGetTruncatedModelDeprecatedDefinition() throws Exception { ) ) { AtomicReference putDocHolder = new AtomicReference<>(); - blockingCall( - listener -> prepareIndex(InferenceIndexConstants.LATEST_INDEX_NAME).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setSource(xContentBuilder) - .setId(TrainedModelDefinitionDoc.docId(modelId, 0)) - .execute(listener), - putDocHolder, - exceptionHolder - ); + blockingCall(listener -> { + IndexRequestBuilder indexRequestBuilder = prepareIndex(InferenceIndexConstants.LATEST_INDEX_NAME).setRefreshPolicy( + WriteRequest.RefreshPolicy.IMMEDIATE + ).setSource(xContentBuilder).setId(TrainedModelDefinitionDoc.docId(modelId, 0)); + indexRequestBuilder.execute(ActionListener.runAfter(listener, () -> indexRequestBuilder.request().decRef())); + }, putDocHolder, exceptionHolder); assertThat(exceptionHolder.get(), is(nullValue())); } @@ -376,6 +375,7 @@ public void testGetTruncatedModelDefinition() throws Exception { ).setId(TrainedModelDefinitionDoc.docId(modelId, i)); bulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } } bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -423,6 +423,7 @@ public void testGetTrainedModelForInference() throws InterruptedException, IOExc ).setId(TrainedModelDefinitionDoc.docId(modelId, i)); bulkRequestBuilder.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } } bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/UnusedStatsRemoverIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/UnusedStatsRemoverIT.java index 4c8382047e796..79cb936ba3a4d 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/UnusedStatsRemoverIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/UnusedStatsRemoverIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ml.integration; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.client.internal.OriginSettingClient; @@ -65,7 +66,9 @@ public void createComponents() { public void testRemoveUnusedStats() throws Exception { - prepareIndex("foo").setId("some-empty-doc").setSource("{}", XContentType.JSON).get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("foo").setId("some-empty-doc").setSource("{}", XContentType.JSON); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); PutDataFrameAnalyticsAction.Request request = new PutDataFrameAnalyticsAction.Request( new DataFrameAnalyticsConfig.Builder().setId("analytics-with-stats") @@ -144,11 +147,15 @@ private void indexStatDocument(ToXContentObject object, String docId) throws Exc Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, Boolean.toString(true)) ); IndexRequest doc = new IndexRequest(MlStatsIndex.writeAlias()); - doc.id(docId); - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - object.toXContent(builder, params); - doc.source(builder); - client.index(doc).actionGet(); + try { + doc.id(docId); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + object.toXContent(builder, params); + doc.source(builder); + client.index(doc).actionGet(); + } + } finally { + doc.decRef(); } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java index 5aed29fd6d152..56677ac019f59 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java @@ -86,19 +86,27 @@ protected void masterOperation( for (String jobId : request.getJobIds()) { UpdateRequest updateRequest = new UpdateRequest(MlConfigIndex.indexName(), Job.documentId(jobId)); - updateRequest.retryOnConflict(3); - updateRequest.doc(update); - updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + try { + updateRequest.retryOnConflict(3); + updateRequest.doc(update); + updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - voidChainTaskExecutor.add(chainedListener -> { - executeAsyncWithOrigin( - client, - ML_ORIGIN, - TransportUpdateAction.TYPE, - updateRequest, - chainedListener.delegateFailureAndWrap((l, updateResponse) -> l.onResponse(null)) - ); - }); + voidChainTaskExecutor.add(chainedListener -> { + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportUpdateAction.TYPE, + updateRequest, + ActionListener.runAfter( + chainedListener.delegateFailureAndWrap((l, updateResponse) -> l.onResponse(null)), + updateRequest::decRef + ) + ); + }); + } catch (Exception e) { + updateRequest.decRef(); + throw e; + } } voidChainTaskExecutor.execute(listener.delegateFailureAndWrap((l, aVoids) -> { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java index 4e0806336be10..99e8d5a99d83d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java @@ -76,47 +76,55 @@ protected void doExecute( ActionListener calendarListener = ActionListener.wrap(calendar -> { BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); - - for (ScheduledEvent event : events) { - IndexRequest indexRequest = new IndexRequest(MlMetaIndex.indexName()); - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - indexRequest.source( - event.toXContent( - builder, - new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")) - ) - ); - } catch (IOException e) { - throw new IllegalStateException("Failed to serialise event", e); + try { + for (ScheduledEvent event : events) { + IndexRequest indexRequest = new IndexRequest(MlMetaIndex.indexName()); + try { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + indexRequest.source( + event.toXContent( + builder, + new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")) + ) + ); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialise event", e); + } + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } } - bulkRequestBuilder.add(indexRequest); - } - bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - executeAsyncWithOrigin( - client, - ML_ORIGIN, - BulkAction.INSTANCE, - bulkRequestBuilder.request(), - ActionListener.releaseAfter(new ActionListener<>() { - @Override - public void onResponse(BulkResponse response) { - jobManager.updateProcessOnCalendarChanged( - calendar.getJobIds(), - ActionListener.wrap( - r -> listener.onResponse(new PostCalendarEventsAction.Response(events)), - listener::onFailure - ) - ); - } + executeAsyncWithOrigin( + client, + ML_ORIGIN, + BulkAction.INSTANCE, + bulkRequestBuilder.request(), + ActionListener.releaseAfter(new ActionListener<>() { + @Override + public void onResponse(BulkResponse response) { + jobManager.updateProcessOnCalendarChanged( + calendar.getJobIds(), + ActionListener.wrap( + r -> listener.onResponse(new PostCalendarEventsAction.Response(events)), + listener::onFailure + ) + ); + } - @Override - public void onFailure(Exception e) { - listener.onFailure(ExceptionsHelper.serverError("Error indexing event", e)); - } - }, bulkRequestBuilder) - ); + @Override + public void onFailure(Exception e) { + listener.onFailure(ExceptionsHelper.serverError("Error indexing event", e)); + } + }, bulkRequestBuilder) + ); + } catch (Exception e) { + bulkRequestBuilder.request().decRef(); + throw e; + } }, listener::onFailure); jobResultsProvider.calendar(request.getCalendarId(), calendarListener); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java index 119b1d2e9ab1b..f50e51f10b78d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java @@ -49,40 +49,54 @@ public TransportPutCalendarAction(TransportService transportService, ActionFilte protected void doExecute(Task task, PutCalendarAction.Request request, ActionListener listener) { Calendar calendar = request.getCalendar(); - IndexRequest indexRequest = new IndexRequest(MlMetaIndex.indexName()).id(calendar.documentId()); - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - indexRequest.source( - calendar.toXContent( - builder, - new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")) - ) - ); - } catch (IOException e) { - throw new IllegalStateException("Failed to serialise calendar with id [" + calendar.getId() + "]", e); - } + IndexRequest indexRequest = new IndexRequest(MlMetaIndex.indexName()); + try { + indexRequest.id(calendar.documentId()); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + indexRequest.source( + calendar.toXContent( + builder, + new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")) + ) + ); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialise calendar with id [" + calendar.getId() + "]", e); + } - // Make it an error to overwrite an existing calendar - indexRequest.opType(DocWriteRequest.OpType.CREATE); - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + // Make it an error to overwrite an existing calendar + indexRequest.opType(DocWriteRequest.OpType.CREATE); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - executeAsyncWithOrigin(client, ML_ORIGIN, TransportIndexAction.TYPE, indexRequest, new ActionListener<>() { - @Override - public void onResponse(DocWriteResponse indexResponse) { - listener.onResponse(new PutCalendarAction.Response(calendar)); - } + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(new ActionListener<>() { + @Override + public void onResponse(DocWriteResponse indexResponse) { + listener.onResponse(new PutCalendarAction.Response(calendar)); + } - @Override - public void onFailure(Exception e) { - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - listener.onFailure( - ExceptionsHelper.badRequestException( - "Cannot create calendar with id [" + calendar.getId() + "] as it already exists" - ) - ); - } else { - listener.onFailure(ExceptionsHelper.serverError("Error putting calendar with id [" + calendar.getId() + "]", e)); - } - } - }); + @Override + public void onFailure(Exception e) { + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + listener.onFailure( + ExceptionsHelper.badRequestException( + "Cannot create calendar with id [" + calendar.getId() + "] as it already exists" + ) + ); + } else { + listener.onFailure( + ExceptionsHelper.serverError("Error putting calendar with id [" + calendar.getId() + "]", e) + ); + } + } + }, indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java index 9237c23e537c1..0c74d002e0692 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java @@ -50,31 +50,46 @@ public TransportPutFilterAction(TransportService transportService, ActionFilters protected void doExecute(Task task, PutFilterAction.Request request, ActionListener listener) { MlFilter filter = request.getFilter(); IndexRequest indexRequest = new IndexRequest(MlMetaIndex.indexName()).id(filter.documentId()); - indexRequest.opType(DocWriteRequest.OpType.CREATE); - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")); - indexRequest.source(filter.toXContent(builder, params)); - } catch (IOException e) { - throw new IllegalStateException("Failed to serialise filter with id [" + filter.getId() + "]", e); - } - - executeAsyncWithOrigin(client, ML_ORIGIN, TransportIndexAction.TYPE, indexRequest, new ActionListener<>() { - @Override - public void onResponse(DocWriteResponse indexResponse) { - listener.onResponse(new PutFilterAction.Response(filter)); + try { + indexRequest.opType(DocWriteRequest.OpType.CREATE); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + ToXContent.MapParams params = new ToXContent.MapParams( + Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true") + ); + indexRequest.source(filter.toXContent(builder, params)); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialise filter with id [" + filter.getId() + "]", e); } - @Override - public void onFailure(Exception e) { - Exception reportedException; - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - reportedException = new ResourceAlreadyExistsException("A filter with id [" + filter.getId() + "] already exists"); - } else { - reportedException = ExceptionsHelper.serverError("Error putting filter with id [" + filter.getId() + "]", e); - } - listener.onFailure(reportedException); - } - }); + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(new ActionListener<>() { + @Override + public void onResponse(DocWriteResponse indexResponse) { + listener.onResponse(new PutFilterAction.Response(filter)); + } + + @Override + public void onFailure(Exception e) { + Exception reportedException; + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + reportedException = new ResourceAlreadyExistsException( + "A filter with id [" + filter.getId() + "] already exists" + ); + } else { + reportedException = ExceptionsHelper.serverError("Error putting filter with id [" + filter.getId() + "]", e); + } + listener.onFailure(reportedException); + } + }, indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java index 22d22cb9d0f73..708d03e0b8bba 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java @@ -119,43 +119,59 @@ private void indexUpdatedFilter( UpdateFilterAction.Request request, ActionListener listener ) { - IndexRequest indexRequest = new IndexRequest(MlMetaIndex.indexName()).id(filter.documentId()); - indexRequest.setIfSeqNo(seqNo); - indexRequest.setIfPrimaryTerm(primaryTerm); - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")); - indexRequest.source(filter.toXContent(builder, params)); - } catch (IOException e) { - throw new IllegalStateException("Failed to serialise filter with id [" + filter.getId() + "]", e); - } - - executeAsyncWithOrigin(client, ML_ORIGIN, TransportIndexAction.TYPE, indexRequest, new ActionListener<>() { - @Override - public void onResponse(DocWriteResponse indexResponse) { - jobManager.notifyFilterChanged( - filter, - request.getAddItems(), - request.getRemoveItems(), - listener.delegateFailureAndWrap((l, response) -> l.onResponse(new PutFilterAction.Response(filter))) + IndexRequest indexRequest = new IndexRequest(MlMetaIndex.indexName()); + try { + indexRequest.id(filter.documentId()); + indexRequest.setIfSeqNo(seqNo); + indexRequest.setIfPrimaryTerm(primaryTerm); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + ToXContent.MapParams params = new ToXContent.MapParams( + Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true") ); + indexRequest.source(filter.toXContent(builder, params)); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialise filter with id [" + filter.getId() + "]", e); } - @Override - public void onFailure(Exception e) { - Exception reportedException; - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - reportedException = ExceptionsHelper.conflictStatusException( - "Error updating filter with id [" + filter.getId() + "] because it was modified while the update was in progress", - e - ); - } else { - reportedException = ExceptionsHelper.serverError("Error updating filter with id [" + filter.getId() + "]", e); - } - listener.onFailure(reportedException); - } - }); + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(new ActionListener<>() { + @Override + public void onResponse(DocWriteResponse indexResponse) { + jobManager.notifyFilterChanged( + filter, + request.getAddItems(), + request.getRemoveItems(), + listener.delegateFailureAndWrap((l, response) -> l.onResponse(new PutFilterAction.Response(filter))) + ); + } + + @Override + public void onFailure(Exception e) { + Exception reportedException; + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + reportedException = ExceptionsHelper.conflictStatusException( + "Error updating filter with id [" + + filter.getId() + + "] because it was modified while the update was in progress", + e + ); + } else { + reportedException = ExceptionsHelper.serverError("Error updating filter with id [" + filter.getId() + "]", e); + } + listener.onFailure(reportedException); + } + }, indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } private void getFilterWithVersion(String filterId, ActionListener listener) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java index 3a5a07768a094..cdebdb2fbcb87 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java @@ -103,33 +103,38 @@ private static Result applyUpdate(UpdateModelSnapshotAction.Reque } private void indexModelSnapshot(Result modelSnapshot, Consumer handler, Consumer errorHandler) { - IndexRequest indexRequest = new IndexRequest(modelSnapshot.index).id(ModelSnapshot.documentId(modelSnapshot.result)); - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - modelSnapshot.result.toXContent(builder, ToXContent.EMPTY_PARAMS); - indexRequest.source(builder); - } catch (IOException e) { - errorHandler.accept(e); - return; - } - BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); - bulkRequestBuilder.add(indexRequest); - bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - executeAsyncWithOrigin( - client, - ML_ORIGIN, - BulkAction.INSTANCE, - bulkRequestBuilder.request(), - ActionListener.releaseAfter(new ActionListener<>() { - @Override - public void onResponse(BulkResponse indexResponse) { - handler.accept(true); - } + IndexRequest indexRequest = new IndexRequest(modelSnapshot.index); + try { + indexRequest.id(ModelSnapshot.documentId(modelSnapshot.result)); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + modelSnapshot.result.toXContent(builder, ToXContent.EMPTY_PARAMS); + indexRequest.source(builder); + } catch (IOException e) { + errorHandler.accept(e); + return; + } + BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); + bulkRequestBuilder.add(indexRequest); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + executeAsyncWithOrigin( + client, + ML_ORIGIN, + BulkAction.INSTANCE, + bulkRequestBuilder.request(), + ActionListener.releaseAfter(new ActionListener<>() { + @Override + public void onResponse(BulkResponse indexResponse) { + handler.accept(true); + } - @Override - public void onFailure(Exception e) { - errorHandler.accept(e); - } - }, bulkRequestBuilder) - ); + @Override + public void onFailure(Exception e) { + errorHandler.accept(e); + } + }, bulkRequestBuilder) + ); + } finally { + indexRequest.decRef(); + } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/annotations/AnnotationPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/annotations/AnnotationPersister.java index b195f4ea9b369..95d6fa511d88b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/annotations/AnnotationPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/annotations/AnnotationPersister.java @@ -94,7 +94,12 @@ public Builder persistAnnotation(Annotation annotation) { public Builder persistAnnotation(@Nullable String annotationId, Annotation annotation) { Objects.requireNonNull(annotation); try (XContentBuilder xContentBuilder = annotation.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) { - bulkRequest.add(new IndexRequest().id(annotationId).source(xContentBuilder).setRequireAlias(true)); + IndexRequest indexRequest = new IndexRequest(); + try { + bulkRequest.add(indexRequest.id(annotationId).source(xContentBuilder).setRequireAlias(true)); + } finally { + indexRequest.decRef(); + } } catch (IOException e) { logger.error(() -> "[" + jobId + "] Error serialising annotation", e); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index e226056217351..8828becc0ad92 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -130,26 +130,31 @@ public void putDatafeedConfig( try (XContentBuilder builder = XContentFactory.jsonBuilder()) { XContentBuilder source = finalConfig.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); - IndexRequest indexRequest = new IndexRequest(MlConfigIndex.indexName()).id(DatafeedConfig.documentId(datafeedId)) - .source(source) - .opType(DocWriteRequest.OpType.CREATE) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - executeAsyncWithOrigin( - client, - ML_ORIGIN, - TransportIndexAction.TYPE, - indexRequest, - ActionListener.wrap(r -> listener.onResponse(Tuple.tuple(finalConfig, r)), e -> { - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - // the datafeed already exists - listener.onFailure(ExceptionsHelper.datafeedAlreadyExists(datafeedId)); - } else { - listener.onFailure(e); - } - }) - ); - + IndexRequest indexRequest = new IndexRequest(MlConfigIndex.indexName()); + try { + indexRequest.id(DatafeedConfig.documentId(datafeedId)) + .source(source) + .opType(DocWriteRequest.OpType.CREATE) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(ActionListener.wrap(r -> listener.onResponse(Tuple.tuple(finalConfig, r)), e -> { + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + // the datafeed already exists + listener.onFailure(ExceptionsHelper.datafeedAlreadyExists(datafeedId)); + } else { + listener.onFailure(e); + } + }), indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } catch (IOException e) { listener.onFailure(new ElasticsearchParseException("Failed to serialise datafeed config with id [" + datafeedId + "]", e)); } @@ -367,14 +372,26 @@ public void onResponse(GetResponse getResponse) { private void indexUpdatedConfig(DatafeedConfig updatedConfig, long seqNo, long primaryTerm, ActionListener listener) { try (XContentBuilder builder = XContentFactory.jsonBuilder()) { XContentBuilder updatedSource = updatedConfig.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); - IndexRequest indexRequest = new IndexRequest(MlConfigIndex.indexName()).id(DatafeedConfig.documentId(updatedConfig.getId())) - .source(updatedSource) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - indexRequest.setIfSeqNo(seqNo); - indexRequest.setIfPrimaryTerm(primaryTerm); - - executeAsyncWithOrigin(client, ML_ORIGIN, TransportIndexAction.TYPE, indexRequest, listener); + IndexRequest indexRequest = new IndexRequest(MlConfigIndex.indexName()); + try { + indexRequest.id(DatafeedConfig.documentId(updatedConfig.getId())) + .source(updatedSource) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + indexRequest.setIfSeqNo(seqNo); + indexRequest.setIfPrimaryTerm(primaryTerm); + + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(listener, indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } catch (IOException e) { listener.onFailure( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsTask.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsTask.java index 1467bd6ca0530..ec5a0e835bbda 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsTask.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsTask.java @@ -254,15 +254,26 @@ void persistProgress(Client clientToUse, String jobId, Runnable runnable) { return; } - IndexRequest indexRequest = new IndexRequest(indexOrAlias).id(progressDocId) - .setRequireAlias(AnomalyDetectorsIndex.jobStateIndexWriteAlias().equals(indexOrAlias)) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - try (XContentBuilder jsonBuilder = JsonXContent.contentBuilder()) { - LOGGER.debug(() -> format("[%s] Persisting progress is: %s", jobId, progress)); - storedProgress.get().toXContent(jsonBuilder, Payload.XContent.EMPTY_PARAMS); - indexRequest.source(jsonBuilder); + IndexRequest indexRequest = new IndexRequest(indexOrAlias).id(progressDocId); + try { + indexRequest.setRequireAlias(AnomalyDetectorsIndex.jobStateIndexWriteAlias().equals(indexOrAlias)) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + try (XContentBuilder jsonBuilder = JsonXContent.contentBuilder()) { + LOGGER.debug(() -> format("[%s] Persisting progress is: %s", jobId, progress)); + storedProgress.get().toXContent(jsonBuilder, Payload.XContent.EMPTY_PARAMS); + indexRequest.source(jsonBuilder); + } + executeAsyncWithOrigin( + clientToUse, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(indexProgressDocListener, indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; } - executeAsyncWithOrigin(clientToUse, ML_ORIGIN, TransportIndexAction.TYPE, indexRequest, indexProgressDocListener); }, e -> { LOGGER.error( () -> format("[%s] cannot persist progress as an error occurred while retrieving former progress document", jobId), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java index c9ce6e0d4e3c7..ca800682d9f8e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java @@ -195,7 +195,12 @@ void inferTestDocs(LocalModel model, TestDocsIterator testDocsIterator, long pro for (SearchHit doc : batch) { dataCountsTracker.incrementTestDocsCount(); InferenceResults inferenceResults = model.inferNoStats(featuresFromDoc(doc)); - bulkIndexer.addAndExecuteIfNeeded(createIndexRequest(doc, inferenceResults, config.getDest().getResultsField())); + IndexRequest indexRequest = createIndexRequest(doc, inferenceResults, config.getDest().getResultsField()); + try { + bulkIndexer.addAndExecuteIfNeeded(indexRequest); + } finally { + indexRequest.decRef(); + } processedDocCount++; int progressPercent = Math.min((int) (processedDocCount * 100.0 / totalDocCount), MAX_PROGRESS_BEFORE_COMPLETION); @@ -227,11 +232,16 @@ private IndexRequest createIndexRequest(SearchHit hit, InferenceResults results, Map source = new LinkedHashMap<>(hit.getSourceAsMap()); source.put(resultField, resultsMap); IndexRequest indexRequest = new IndexRequest(hit.getIndex()); - indexRequest.id(hit.getId()); - indexRequest.source(source); - indexRequest.opType(DocWriteRequest.OpType.INDEX); - indexRequest.setParentTask(parentTaskId); - return indexRequest; + try { + indexRequest.id(hit.getId()); + indexRequest.source(source); + indexRequest.opType(DocWriteRequest.OpType.INDEX); + indexRequest.setParentTask(parentTaskId); + return indexRequest; + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } private void executeBulkRequest(BulkRequest bulkRequest) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java index 5469c6a7a7d87..3f51ef67832c8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java @@ -235,30 +235,36 @@ private void index( ) { try (XContentBuilder builder = XContentFactory.jsonBuilder()) { config.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); - IndexRequest indexRequest = new IndexRequest(MlConfigIndex.indexName()).id(DataFrameAnalyticsConfig.documentId(config.getId())) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(builder); - if (getResponse == null) { - indexRequest.opType(DocWriteRequest.OpType.CREATE); - } else { - indexRequest.opType(DocWriteRequest.OpType.INDEX) - .setIfSeqNo(getResponse.getSeqNo()) - .setIfPrimaryTerm(getResponse.getPrimaryTerm()); - } + IndexRequest indexRequest = new IndexRequest(MlConfigIndex.indexName()); + try { + indexRequest.id(DataFrameAnalyticsConfig.documentId(config.getId())) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(builder); + if (getResponse == null) { + indexRequest.opType(DocWriteRequest.OpType.CREATE); + } else { + indexRequest.opType(DocWriteRequest.OpType.INDEX) + .setIfSeqNo(getResponse.getSeqNo()) + .setIfPrimaryTerm(getResponse.getPrimaryTerm()); + } - executeAsyncWithOrigin( - client, - ML_ORIGIN, - TransportIndexAction.TYPE, - indexRequest, - ActionListener.wrap(indexResponse -> listener.onResponse(config), e -> { - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - listener.onFailure(ExceptionsHelper.dataFrameAnalyticsAlreadyExists(config.getId())); - } else { - listener.onFailure(e); - } - }) - ); + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(ActionListener.wrap(indexResponse -> listener.onResponse(config), e -> { + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + listener.onFailure(ExceptionsHelper.dataFrameAnalyticsAlreadyExists(config.getId())); + } else { + listener.onFailure(e); + } + }), indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } catch (IOException e) { listener.onFailure( new ElasticsearchParseException("Failed to serialise data frame analytics with id [" + config.getId() + "]") diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java index ee91b0637bfc7..729d026bdb627 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java @@ -100,7 +100,12 @@ private void joinCurrentResults() { RowResults result = currentResults.pop(); DataFrameDataExtractor.Row row = dataFrameRowsIterator.next(); checkChecksumsMatch(row, result); - bulkIndexer.addAndExecuteIfNeeded(createIndexRequest(result, row.getHit())); + IndexRequest indexRequest = createIndexRequest(result, row.getHit()); + try { + bulkIndexer.addAndExecuteIfNeeded(indexRequest); + } finally { + indexRequest.decRef(); + } } } @@ -132,11 +137,16 @@ private IndexRequest createIndexRequest(RowResults result, SearchHit hit) { Map source = new LinkedHashMap<>(hit.getSourceAsMap()); source.putAll(result.getResults()); IndexRequest indexRequest = new IndexRequest(hit.getIndex()); - indexRequest.id(hit.getId()); - indexRequest.source(source); - indexRequest.opType(DocWriteRequest.OpType.INDEX); - indexRequest.setParentTask(parentTaskId); - return indexRequest; + try { + indexRequest.id(hit.getId()); + indexRequest.source(source); + indexRequest.opType(DocWriteRequest.OpType.INDEX); + indexRequest.setParentTask(parentTaskId); + return indexRequest; + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java index 4f06e9ace2887..3f0db26d25583 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java @@ -211,7 +211,13 @@ void updateStats() { } String jobPattern = ""; try (BulkRequest bulkRequest = new BulkRequest()) { - stats.stream().map(TrainedModelStatsService::buildUpdateRequest).filter(Objects::nonNull).forEach(bulkRequest::add); + stats.stream().map(TrainedModelStatsService::buildUpdateRequest).filter(Objects::nonNull).forEach(updateRequest -> { + try { + bulkRequest.add(updateRequest); + } finally { + updateRequest.decRef(); + } + }); bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); if (bulkRequest.requests().isEmpty()) { return; @@ -285,15 +291,20 @@ static UpdateRequest buildUpdateRequest(InferenceStats stats) { params.put(InferenceStats.CACHE_MISS_COUNT.getPreferredName(), stats.getCacheMissCount()); stats.toXContent(builder, FOR_INTERNAL_STORAGE_PARAMS); UpdateRequest updateRequest = new UpdateRequest(); - updateRequest.upsert(builder) - .index(MlStatsIndex.writeAlias()) - // Usually, there shouldn't be a conflict, but if there is, only around a single update should have happened - // out of band. If there is MANY more than that, something strange is happening and it should fail. - .retryOnConflict(3) - .id(InferenceStats.docId(stats.getModelId(), stats.getNodeId())) - .script(new Script(ScriptType.INLINE, "painless", STATS_UPDATE_SCRIPT, params)) - .setRequireAlias(true); - return updateRequest; + try { + updateRequest.upsert(builder) + .index(MlStatsIndex.writeAlias()) + // Usually, there shouldn't be a conflict, but if there is, only around a single update should have happened + // out of band. If there is MANY more than that, something strange is happening and it should fail. + .retryOnConflict(3) + .id(InferenceStats.docId(stats.getModelId(), stats.getNodeId())) + .script(new Script(ScriptType.INLINE, "painless", STATS_UPDATE_SCRIPT, params)) + .setRequireAlias(true); + return updateRequest; + } catch (Exception e) { + updateRequest.decRef(); + throw e; + } } catch (IOException ex) { logger.error(() -> format("[%s] [%s] failed to serialize stats for update.", stats.getModelId(), stats.getNodeId()), ex); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java index 4c7e88c6fac01..6d57664958eb4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java @@ -201,31 +201,36 @@ public void storeTrainedModelConfig(TrainedModelConfig trainedModelConfig, Actio trainedModelConfig, allowOverwriting ); - request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + try { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - executeAsyncWithOrigin( - client, - ML_ORIGIN, - TransportIndexAction.TYPE, - request, - ActionListener.wrap(indexResponse -> listener.onResponse(true), e -> { - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - listener.onFailure( - new ResourceAlreadyExistsException( - Messages.getMessage(Messages.INFERENCE_TRAINED_MODEL_EXISTS, trainedModelConfig.getModelId()) - ) - ); - } else { - listener.onFailure( - new ElasticsearchStatusException( - Messages.getMessage(Messages.INFERENCE_FAILED_TO_STORE_MODEL, trainedModelConfig.getModelId()), - RestStatus.INTERNAL_SERVER_ERROR, - e - ) - ); - } - }) - ); + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + request, + ActionListener.runAfter(ActionListener.wrap(indexResponse -> listener.onResponse(true), e -> { + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + listener.onFailure( + new ResourceAlreadyExistsException( + Messages.getMessage(Messages.INFERENCE_TRAINED_MODEL_EXISTS, trainedModelConfig.getModelId()) + ) + ); + } else { + listener.onFailure( + new ElasticsearchStatusException( + Messages.getMessage(Messages.INFERENCE_FAILED_TO_STORE_MODEL, trainedModelConfig.getModelId()), + RestStatus.INTERNAL_SERVER_ERROR, + e + ) + ); + } + }), request::decRef) + ); + } catch (Exception e) { + request.decRef(); + throw e; + } } public void storeTrainedModelDefinitionDoc(TrainedModelDefinitionDoc trainedModelDefinitionDoc, ActionListener listener) { @@ -252,29 +257,38 @@ public void storeTrainedModelVocabulary( listener.onFailure(new ResourceAlreadyExistsException(Messages.getMessage(Messages.INFERENCE_TRAINED_MODEL_EXISTS, modelId))); return; } - executeAsyncWithOrigin( - client, - ML_ORIGIN, - TransportIndexAction.TYPE, - createRequest(VocabularyConfig.docId(modelId), vocabularyConfig.getIndex(), vocabulary, allowOverwriting).setRefreshPolicy( - WriteRequest.RefreshPolicy.IMMEDIATE - ), - ActionListener.wrap(indexResponse -> listener.onResponse(null), e -> { - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - listener.onFailure( - new ResourceAlreadyExistsException(Messages.getMessage(Messages.INFERENCE_TRAINED_MODEL_VOCAB_EXISTS, modelId)) - ); - } else { - listener.onFailure( - new ElasticsearchStatusException( - Messages.getMessage(Messages.INFERENCE_FAILED_TO_STORE_MODEL_VOCAB, modelId), - RestStatus.INTERNAL_SERVER_ERROR, - e - ) - ); - } - }) - ); + IndexRequest indexRequest = createRequest( + VocabularyConfig.docId(modelId), + vocabularyConfig.getIndex(), + vocabulary, + allowOverwriting + ).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + try { + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(ActionListener.wrap(indexResponse -> listener.onResponse(null), e -> { + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + listener.onFailure( + new ResourceAlreadyExistsException(Messages.getMessage(Messages.INFERENCE_TRAINED_MODEL_VOCAB_EXISTS, modelId)) + ); + } else { + listener.onFailure( + new ElasticsearchStatusException( + Messages.getMessage(Messages.INFERENCE_FAILED_TO_STORE_MODEL_VOCAB, modelId), + RestStatus.INTERNAL_SERVER_ERROR, + e + ) + ); + } + }), indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } public void storeTrainedModelDefinitionDoc( @@ -300,37 +314,43 @@ public void storeTrainedModelDefinitionDoc( return; } - executeAsyncWithOrigin( - client, - ML_ORIGIN, - TransportIndexAction.TYPE, - createRequest(trainedModelDefinitionDoc.getDocId(), index, trainedModelDefinitionDoc, allowOverwriting), - ActionListener.wrap(indexResponse -> listener.onResponse(null), e -> { - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - listener.onFailure( - new ResourceAlreadyExistsException( - Messages.getMessage( - Messages.INFERENCE_TRAINED_MODEL_DOC_EXISTS, - trainedModelDefinitionDoc.getModelId(), - trainedModelDefinitionDoc.getDocNum() + IndexRequest indexRequest = createRequest(trainedModelDefinitionDoc.getDocId(), index, trainedModelDefinitionDoc, allowOverwriting); + try { + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(ActionListener.wrap(indexResponse -> listener.onResponse(null), e -> { + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + listener.onFailure( + new ResourceAlreadyExistsException( + Messages.getMessage( + Messages.INFERENCE_TRAINED_MODEL_DOC_EXISTS, + trainedModelDefinitionDoc.getModelId(), + trainedModelDefinitionDoc.getDocNum() + ) ) - ) - ); - } else { - listener.onFailure( - new ElasticsearchStatusException( - Messages.getMessage( - Messages.INFERENCE_FAILED_TO_STORE_MODEL_DEFINITION, - trainedModelDefinitionDoc.getModelId(), - trainedModelDefinitionDoc.getDocNum() - ), - RestStatus.INTERNAL_SERVER_ERROR, - e - ) - ); - } - }) - ); + ); + } else { + listener.onFailure( + new ElasticsearchStatusException( + Messages.getMessage( + Messages.INFERENCE_FAILED_TO_STORE_MODEL_DEFINITION, + trainedModelDefinitionDoc.getModelId(), + trainedModelDefinitionDoc.getDocNum() + ), + RestStatus.INTERNAL_SERVER_ERROR, + e + ) + ); + } + }), indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } public void storeTrainedModelMetadata(TrainedModelMetadata trainedModelMetadata, ActionListener listener) { @@ -350,34 +370,40 @@ public void storeTrainedModelMetadata( ); return; } - executeAsyncWithOrigin( - client, - ML_ORIGIN, - TransportIndexAction.TYPE, - createRequest( - trainedModelMetadata.getDocId(), - InferenceIndexConstants.LATEST_INDEX_NAME, - trainedModelMetadata, - allowOverwriting - ), - ActionListener.wrap(indexResponse -> listener.onResponse(null), e -> { - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - listener.onFailure( - new ResourceAlreadyExistsException( - Messages.getMessage(Messages.INFERENCE_TRAINED_MODEL_METADATA_EXISTS, trainedModelMetadata.getModelId()) - ) - ); - } else { - listener.onFailure( - new ElasticsearchStatusException( - Messages.getMessage(Messages.INFERENCE_FAILED_TO_STORE_MODEL_METADATA, trainedModelMetadata.getModelId()), - RestStatus.INTERNAL_SERVER_ERROR, - e - ) - ); - } - }) + IndexRequest indexRequest = createRequest( + trainedModelMetadata.getDocId(), + InferenceIndexConstants.LATEST_INDEX_NAME, + trainedModelMetadata, + allowOverwriting ); + try { + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(ActionListener.wrap(indexResponse -> listener.onResponse(null), e -> { + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + listener.onFailure( + new ResourceAlreadyExistsException( + Messages.getMessage(Messages.INFERENCE_TRAINED_MODEL_METADATA_EXISTS, trainedModelMetadata.getModelId()) + ) + ); + } else { + listener.onFailure( + new ElasticsearchStatusException( + Messages.getMessage(Messages.INFERENCE_FAILED_TO_STORE_MODEL_METADATA, trainedModelMetadata.getModelId()), + RestStatus.INTERNAL_SERVER_ERROR, + e + ) + ); + } + }), indexRequest::decRef) + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } public void getTrainedModelMetadata( @@ -473,64 +499,80 @@ private void storeTrainedModelAndDefinition( return; } - BulkRequestBuilder bulkRequest = client.prepareBulk(InferenceIndexConstants.LATEST_INDEX_NAME) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .add(createRequest(trainedModelConfig.getModelId(), trainedModelConfig, allowOverwriting)); - trainedModelDefinitionDocs.forEach(defDoc -> bulkRequest.add(createRequest(defDoc.getDocId(), defDoc, allowOverwriting))); + IndexRequest indexRequest = createRequest(trainedModelConfig.getModelId(), trainedModelConfig, allowOverwriting); + try { + BulkRequestBuilder bulkRequest = client.prepareBulk(InferenceIndexConstants.LATEST_INDEX_NAME); + try { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).add(indexRequest); + trainedModelDefinitionDocs.forEach(defDoc -> { + IndexRequest indexRequest1 = createRequest(defDoc.getDocId(), defDoc, allowOverwriting); + try { + bulkRequest.add(indexRequest1); + } finally { + indexRequest1.decRef(); + } + }); - ActionListener wrappedListener = ActionListener.wrap(listener::onResponse, e -> { - if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { - listener.onFailure( - new ResourceAlreadyExistsException( - Messages.getMessage(Messages.INFERENCE_TRAINED_MODEL_EXISTS, trainedModelConfig.getModelId()) - ) - ); - } else { - listener.onFailure( - new ElasticsearchStatusException( - Messages.getMessage(Messages.INFERENCE_FAILED_TO_STORE_MODEL, trainedModelConfig.getModelId()), - RestStatus.INTERNAL_SERVER_ERROR, - e - ) - ); - } - }); + ActionListener wrappedListener = ActionListener.wrap(listener::onResponse, e -> { + if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { + listener.onFailure( + new ResourceAlreadyExistsException( + Messages.getMessage(Messages.INFERENCE_TRAINED_MODEL_EXISTS, trainedModelConfig.getModelId()) + ) + ); + } else { + listener.onFailure( + new ElasticsearchStatusException( + Messages.getMessage(Messages.INFERENCE_FAILED_TO_STORE_MODEL, trainedModelConfig.getModelId()), + RestStatus.INTERNAL_SERVER_ERROR, + e + ) + ); + } + }); - ActionListener bulkResponseActionListener = ActionListener.wrap(r -> { - assert r.getItems().length == trainedModelDefinitionDocs.size() + 1; - if (r.getItems()[0].isFailed()) { - logger.error( - () -> "[" + trainedModelConfig.getModelId() + "] failed to store trained model config for inference", - r.getItems()[0].getFailure().getCause() - ); + ActionListener bulkResponseActionListener = ActionListener.wrap(r -> { + assert r.getItems().length == trainedModelDefinitionDocs.size() + 1; + if (r.getItems()[0].isFailed()) { + logger.error( + () -> "[" + trainedModelConfig.getModelId() + "] failed to store trained model config for inference", + r.getItems()[0].getFailure().getCause() + ); - wrappedListener.onFailure(r.getItems()[0].getFailure().getCause()); - return; - } - if (r.hasFailures()) { - Exception firstFailure = Arrays.stream(r.getItems()) - .filter(BulkItemResponse::isFailed) - .map(BulkItemResponse::getFailure) - .map(BulkItemResponse.Failure::getCause) - .findFirst() - .orElse(new Exception("unknown failure")); - logger.error( - () -> format("[%s] failed to store trained model definition for inference", trainedModelConfig.getModelId()), - firstFailure + wrappedListener.onFailure(r.getItems()[0].getFailure().getCause()); + return; + } + if (r.hasFailures()) { + Exception firstFailure = Arrays.stream(r.getItems()) + .filter(BulkItemResponse::isFailed) + .map(BulkItemResponse::getFailure) + .map(BulkItemResponse.Failure::getCause) + .findFirst() + .orElse(new Exception("unknown failure")); + logger.error( + () -> format("[%s] failed to store trained model definition for inference", trainedModelConfig.getModelId()), + firstFailure + ); + wrappedListener.onFailure(firstFailure); + return; + } + wrappedListener.onResponse(true); + }, wrappedListener::onFailure); + + executeAsyncWithOrigin( + client, + ML_ORIGIN, + BulkAction.INSTANCE, + bulkRequest.request(), + ActionListener.releaseAfter(bulkResponseActionListener, bulkRequest) ); - wrappedListener.onFailure(firstFailure); - return; + } catch (Exception e) { + bulkRequest.close(); + throw e; } - wrappedListener.onResponse(true); - }, wrappedListener::onFailure); - - executeAsyncWithOrigin( - client, - ML_ORIGIN, - BulkAction.INSTANCE, - bulkRequest.request(), - ActionListener.releaseAfter(bulkResponseActionListener, bulkRequest) - ); + } finally { + indexRequest.decRef(); + } } /** @@ -1363,11 +1405,23 @@ private XContentParser createParser(BytesReference source) throws IOException { } private static IndexRequest createRequest(String docId, String index, ToXContentObject body, boolean allowOverwriting) { - return createRequest(new IndexRequest(index), docId, body, allowOverwriting); + IndexRequest indexRequest = new IndexRequest(index); + try { + return createRequest(indexRequest, docId, body, allowOverwriting); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } private static IndexRequest createRequest(String docId, ToXContentObject body, boolean allowOverwriting) { - return createRequest(new IndexRequest(), docId, body, allowOverwriting); + IndexRequest indexRequest = new IndexRequest(); + try { + return createRequest(indexRequest, docId, body, allowOverwriting); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } private static IndexRequest createRequest(IndexRequest request, String docId, ToXContentObject body, boolean allowOverwriting) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 8493513f40bd6..073de6c46b826 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -131,14 +131,14 @@ public void putJob(Job job, ActionListener listener) { ML_ORIGIN, TransportIndexAction.TYPE, indexRequest, - ActionListener.wrap(listener::onResponse, e -> { + ActionListener.runAfter(ActionListener.wrap(listener::onResponse, e -> { if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { // the job already exists listener.onFailure(ExceptionsHelper.jobAlreadyExists(job.getId())); } else { listener.onFailure(e); } - }) + }), indexRequest::decRef) ); } catch (IOException e) { @@ -343,10 +343,16 @@ private void indexUpdatedJob(Job updatedJob, long seqNo, long primaryTerm, Actio indexRequest.setIfSeqNo(seqNo); indexRequest.setIfPrimaryTerm(primaryTerm); - executeAsyncWithOrigin(client, ML_ORIGIN, TransportIndexAction.TYPE, indexRequest, ActionListener.wrap(indexResponse -> { - assert indexResponse.getResult() == DocWriteResponse.Result.UPDATED; - updatedJobListener.onResponse(updatedJob); - }, updatedJobListener::onFailure)); + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + indexRequest, + ActionListener.runAfter(ActionListener.wrap(indexResponse -> { + assert indexResponse.getResult() == DocWriteResponse.Result.UPDATED; + updatedJobListener.onResponse(updatedJob); + }, updatedJobListener::onFailure), indexRequest::decRef) + ); } catch (IOException e) { updatedJobListener.onFailure( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java index 40951162045c3..355d6a3935fd6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java @@ -120,11 +120,23 @@ public boolean persistDataCounts(String jobId, DataCounts counts, boolean mustWa public void persistDataCountsAsync(String jobId, DataCounts counts, ActionListener listener) { counts.setLogTime(Instant.now()); try (XContentBuilder content = serialiseCounts(counts)) { - final IndexRequest request = new IndexRequest(AnomalyDetectorsIndex.resultsWriteAlias(jobId)).id(DataCounts.documentId(jobId)) - .setRequireAlias(true) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(content); - executeAsyncWithOrigin(client, ML_ORIGIN, TransportIndexAction.TYPE, request, listener.safeMap(r -> true)); + final IndexRequest request = new IndexRequest(AnomalyDetectorsIndex.resultsWriteAlias(jobId)); + try { + request.id(DataCounts.documentId(jobId)) + .setRequireAlias(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(content); + executeAsyncWithOrigin( + client, + ML_ORIGIN, + TransportIndexAction.TYPE, + request, + ActionListener.runAfter(listener.safeMap(r -> true), request::decRef) + ); + } catch (Exception e) { + request.decRef(); + throw e; + } } catch (IOException ioe) { String msg = "[" + jobId + "] Failed writing data_counts stats"; logger.error(msg, ioe); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java index 69344214524e0..5218cce9a9b56 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java @@ -76,7 +76,14 @@ public void updateResults(List normalizables) { public void updateResult(String id, String index, ToXContent resultDoc) { try (XContentBuilder content = toXContentBuilder(resultDoc)) { - bulkRequest.add(new IndexRequest(index).id(id).source(content)); + IndexRequest indexRequest = new IndexRequest(index); + try { + indexRequest.id(id).source(content); + bulkRequest.add(indexRequest); + } finally { + indexRequest.decRef(); + } + } catch (IOException e) { logger.error(() -> "[" + jobId + "] Error serialising result", e); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java index d38c479b74bd3..894324c68425d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java @@ -270,7 +270,16 @@ private void indexResult(String id, ToXContent resultDoc, String resultType) { private void indexResult(String id, ToXContent resultDoc, ToXContent.Params params, String resultType) { try (XContentBuilder content = toXContentBuilder(resultDoc, params)) { - items.put(id, new IndexRequest(indexName).id(id).source(content)); + IndexRequest indexRequest = new IndexRequest(indexName); + try { + IndexRequest previousRequest = items.put(id, indexRequest.id(id).source(content)); + if (previousRequest != null) { + previousRequest.decRef(); + } + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } catch (IOException e) { logger.error(() -> format("[%s] Error serialising %s", jobId, resultType), e); } @@ -308,6 +317,9 @@ private BulkRequest buildBulkRequest() { } public synchronized void clear() { + for (IndexRequest indexRequest : items.values()) { + indexRequest.decRef(); + } items.clear(); } @@ -584,11 +596,20 @@ void persistWithoutRetries(ActionListener listener, boolean re logCall(); try (XContentBuilder content = toXContentBuilder(object, params)) { - IndexRequest indexRequest = new IndexRequest(indexName).id(id) - .source(content) - .setRefreshPolicy(refreshPolicy) - .setRequireAlias(requireAlias); - executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, indexRequest, listener, client::index); + IndexRequest indexRequest = new IndexRequest(indexName).id(id); + try { + indexRequest.source(content).setRefreshPolicy(refreshPolicy).setRequireAlias(requireAlias); + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ML_ORIGIN, + indexRequest, + ActionListener.runAfter(listener, indexRequest::decRef), + client::index + ); + } catch (Exception e) { + indexRequest.decRef(); + throw e; + } } catch (IOException e) { logger.error(() -> format("[%s] Error writing [%s]", jobId, (id == null) ? "auto-generated ID" : id), e); IndexResponse.Builder notCreatedResponse = new IndexResponse.Builder(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java index 0575760cd269c..fe09bf573a461 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java @@ -1865,21 +1865,29 @@ public void updateCalendar( Calendar updatedCalendar = new Calendar(calendar.getId(), new ArrayList<>(currentJobs), calendar.getDescription()); UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.indexName(), updatedCalendar.documentId()); - updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + try { + updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - updateRequest.doc(updatedCalendar.toXContent(builder, ToXContent.EMPTY_PARAMS)); - } catch (IOException e) { - throw new IllegalStateException("Failed to serialise calendar with id [" + updatedCalendar.getId() + "]", e); - } + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + updateRequest.doc(updatedCalendar.toXContent(builder, ToXContent.EMPTY_PARAMS)); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialise calendar with id [" + updatedCalendar.getId() + "]", e); + } - executeAsyncWithOrigin( - client.threadPool().getThreadContext(), - ML_ORIGIN, - updateRequest, - ActionListener.wrap(response -> handler.accept(updatedCalendar), errorHandler), - client::update - ); + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ML_ORIGIN, + updateRequest, + ActionListener.runAfter( + ActionListener.wrap(response -> handler.accept(updatedCalendar), errorHandler), + updateRequest::decRef + ), + client::update + ); + } catch (Exception e) { + updateRequest.decRef(); + throw e; + } }, errorHandler); @@ -1931,13 +1939,19 @@ public void removeJobFromCalendars(String jobId, ActionListener listene List ids = calendar.getJobIds().stream().filter(jId -> jobId.equals(jId) == false).collect(Collectors.toList()); Calendar newCalendar = new Calendar(calendar.getId(), ids, calendar.getDescription()); UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.indexName(), newCalendar.documentId()); - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - updateRequest.doc(newCalendar.toXContent(builder, ToXContent.EMPTY_PARAMS)); - } catch (IOException e) { - listener.onFailure(new IllegalStateException("Failed to serialise calendar with id [" + newCalendar.getId() + "]", e)); - return; + try { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + updateRequest.doc(newCalendar.toXContent(builder, ToXContent.EMPTY_PARAMS)); + } catch (IOException e) { + listener.onFailure( + new IllegalStateException("Failed to serialise calendar with id [" + newCalendar.getId() + "]", e) + ); + return; + } + bulkUpdate.add(updateRequest); + } finally { + updateRequest.decRef(); } - bulkUpdate.add(updateRequest); } if (bulkUpdate.numberOfActions() > 0) { executeAsyncWithOrigin( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java index 08c52c9c51b74..4ceae3b306bc1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java @@ -149,7 +149,12 @@ public BulkResponse indexWithRetry( ) throws IOException { try (BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(refreshPolicy)) { try (XContentBuilder content = object.toXContent(XContentFactory.jsonBuilder(), params)) { - bulkRequest.add(new IndexRequest(indexName).id(id).source(content).setRequireAlias(requireAlias)); + IndexRequest indexRequest = new IndexRequest(indexName).id(id).source(content).setRequireAlias(requireAlias); + try { + bulkRequest.add(indexRequest); + } finally { + indexRequest.decRef(); + } } return bulkIndexWithRetry(bulkRequest, jobId, shouldRetry, retryMsgHandler); } @@ -169,7 +174,12 @@ public void indexWithRetry( ) throws IOException { BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(refreshPolicy); try (XContentBuilder content = object.toXContent(XContentFactory.jsonBuilder(), params)) { - bulkRequest.add(new IndexRequest(indexName).id(id).source(content).setRequireAlias(requireAlias)); + IndexRequest indexRequest = new IndexRequest(indexName).id(id).source(content).setRequireAlias(requireAlias); + try { + bulkRequest.add(indexRequest); + } finally { + indexRequest.decRef(); + } } bulkIndexWithRetry(bulkRequest, jobId, shouldRetry, retryMsgHandler, ActionListener.releaseAfter(finalListener, bulkRequest)); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java index 30c6ae7c65943..3e65bbf44f14d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java @@ -236,6 +236,7 @@ public void testBulkRequestExecutesWhenReachMaxDocs() { inOrder.verify(client, times(3)).threadPool(); inOrder.verify(client).execute(eq(BulkAction.INSTANCE), bulkRequestCaptor.capture(), any()); verifyNoMoreInteractions(client); + bulkBuilder.executeRequest(); // clean up by clearing out any remaining requests } public void testPersistTimingStats() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index 6aed34fdb59ea..9483c3f88bd6a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -303,26 +303,35 @@ public static void indexDocs(Logger logger, String index, long numDocs, long sta public static void indexDocs(Client client, Logger logger, String index, long numDocs, long start, long end) { int maxDelta = (int) (end - start - 1); - try (BulkRequestBuilder bulkRequestBuilder = client.prepareBulk()) { - for (int i = 0; i < numDocs; i++) { - IndexRequest indexRequest = new IndexRequest(index); - long timestamp = start + randomIntBetween(0, maxDelta); - assert timestamp >= start && timestamp < end; - indexRequest.source("time", timestamp, "@timestamp", timestamp).opType(DocWriteRequest.OpType.CREATE); - bulkRequestBuilder.add(indexRequest); - } - BulkResponse bulkResponse = bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); - if (bulkResponse.hasFailures()) { - int failures = 0; - for (BulkItemResponse itemResponse : bulkResponse) { - if (itemResponse.isFailed()) { - failures++; - logger.error("Item response failure [{}]", itemResponse.getFailureMessage()); + int maxBatchSize = 1000; + int docsIndexed = 0; + while (docsIndexed < numDocs) { + try (BulkRequestBuilder bulkRequestBuilder = client.prepareBulk()) { + for (int i = 0; i < maxBatchSize && docsIndexed < numDocs; i++) { + docsIndexed++; + IndexRequest indexRequest = new IndexRequest(index); + try { + long timestamp = start + randomIntBetween(0, maxDelta); + assert timestamp >= start && timestamp < end; + indexRequest.source("time", timestamp, "@timestamp", timestamp).opType(DocWriteRequest.OpType.CREATE); + bulkRequestBuilder.add(indexRequest); + } finally { + indexRequest.decRef(); + } + } + BulkResponse bulkResponse = bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + if (bulkResponse.hasFailures()) { + int failures = 0; + for (BulkItemResponse itemResponse : bulkResponse) { + if (itemResponse.isFailed()) { + failures++; + logger.error("Item response failure [{}]", itemResponse.getFailureMessage()); + } } + fail("Bulk response contained " + failures + " failures"); } - fail("Bulk response contained " + failures + " failures"); + logger.info("Indexed [{}] documents", numDocs); } - logger.info("Indexed [{}] documents", numDocs); } } diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index a0990330aecb7..20d0806f34535 100644 --- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; @@ -196,14 +197,12 @@ public void testMonitoringService() throws Exception { final boolean createAPMIndex = randomBoolean(); final String indexName = createAPMIndex ? "apm-2017.11.06" : "books"; + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setId("0"); assertThat( - prepareIndex(indexName).setId("0") - .setRefreshPolicy("true") - .setSource("{\"field\":\"value\"}", XContentType.JSON) - .get() - .status(), + indexRequestBuilder.setRefreshPolicy("true").setSource("{\"field\":\"value\"}", XContentType.JSON).get().status(), is(RestStatus.CREATED) ); + indexRequestBuilder.request().decRef(); final Settings settings = Settings.builder().put("cluster.metadata.display_name", "my cluster").build(); assertAcked(clusterAdmin().prepareUpdateSettings().setTransientSettings(settings)); diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java index a7dfecc6cce08..09c537e9c73a7 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java @@ -62,24 +62,28 @@ protected void doAdd(Collection docs) throws ExportException { final String index = MonitoringTemplateUtils.indexName(formatter, doc.getSystem(), doc.getTimestamp()); final IndexRequest request = new IndexRequest(index); - if (Strings.hasText(doc.getId())) { - request.id(doc.getId()); - } - - final BytesReference source = XContentHelper.toXContent(doc, XContentType.SMILE, false); - request.source(source, XContentType.SMILE); - - requestBuilder.add(request); - - if (logger.isTraceEnabled()) { - logger.trace( - "local exporter [{}] - added index request [index={}, id={}, pipeline={}, monitoring data type={}]", - name, - request.index(), - request.id(), - request.getPipeline(), - doc.getType() - ); + try { + if (Strings.hasText(doc.getId())) { + request.id(doc.getId()); + } + + final BytesReference source = XContentHelper.toXContent(doc, XContentType.SMILE, false); + request.source(source, XContentType.SMILE); + + requestBuilder.add(request); + + if (logger.isTraceEnabled()) { + logger.trace( + "local exporter [{}] - added index request [index={}, id={}, pipeline={}, monitoring data type={}]", + name, + request.index(), + request.id(), + request.getPipeline(), + doc.getType() + ); + } + } finally { + request.decRef(); } } catch (Exception e) { if (exception == null) { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java index ef4f22f852b37..d978826fade93 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java @@ -78,6 +78,9 @@ public void testExport() throws Exception { .setSource("title", "This is a random document"); } indexRandom(true, indexRequestBuilders); + for (IndexRequestBuilder builder : indexRequestBuilders) { + builder.request().decRef(); + } } // start the monitoring service so that /_monitoring/bulk is not ignored diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankCoordinatorCanMatchIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankCoordinatorCanMatchIT.java index 445aeaa375e11..15096fc85612e 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankCoordinatorCanMatchIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankCoordinatorCanMatchIT.java @@ -10,6 +10,7 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.PointValues; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexSettings; @@ -114,10 +115,14 @@ public void testCanMatchCoordinator() throws Exception { ensureGreen("time_index"); for (int i = 0; i < 500; i++) { - prepareIndex("time_index").setSource("@timestamp", i).setRouting("a").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("time_index").setSource("@timestamp", i).setRouting("a"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } for (int i = 500; i < 1000; i++) { - prepareIndex("time_index").setSource("@timestamp", i).setRouting("b").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("time_index").setSource("@timestamp", i).setRouting("b"); + indexRequestBuilder.get(); + indexRequestBuilder.request().decRef(); } client().admin().indices().prepareRefresh("time_index").get(); diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java index af465658a0b52..0b06c4fcb2a2b 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java @@ -77,9 +77,9 @@ public void setupSuiteScopeCluster() throws Exception { assertAcked(prepareCreate("tiny_index").setMapping(builder)); ensureGreen("tiny_index"); - prepareIndex("tiny_index").setSource("vector", new float[] { 0.0f }, "text", "term term").get(); - prepareIndex("tiny_index").setSource("vector", new float[] { 1.0f }, "text", "other").get(); - prepareIndex("tiny_index").setSource("vector", new float[] { 2.0f }, "text", "term").get(); + indexDoc("tiny_index", null, "vector", new float[] { 0.0f }, "text", "term term"); + indexDoc("tiny_index", null, "vector", new float[] { 1.0f }, "text", "other"); + indexDoc("tiny_index", null, "vector", new float[] { 2.0f }, "text", "term"); indicesAdmin().prepareRefresh("tiny_index").get(); @@ -117,7 +117,9 @@ public void setupSuiteScopeCluster() throws Exception { ensureGreen(TimeValue.timeValueSeconds(120), "nrd_index"); for (int doc = 0; doc < 1001; ++doc) { - prepareIndex("nrd_index").setSource( + indexDoc( + "nrd_index", + null, "vector_asc", new float[] { doc }, "vector_desc", @@ -128,7 +130,7 @@ public void setupSuiteScopeCluster() throws Exception { "term " + doc, "text1", "term " + (1000 - doc) - ).get(); + ); } indicesAdmin().prepareRefresh("nrd_index").get(); diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankShardCanMatchIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankShardCanMatchIT.java index 084ccc88bee33..4ba151b4f70a2 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankShardCanMatchIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankShardCanMatchIT.java @@ -9,6 +9,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -129,13 +130,17 @@ public void testCanMatchShard() throws IOException { int shardB = -1; for (int i = 0; i < 10; i++) { - DocWriteResponse ir = prepareIndex("value_index").setSource("value", "" + i).setRouting("a").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("value_index"); + DocWriteResponse ir = indexRequestBuilder.setSource("value", "" + i).setRouting("a").get(); + indexRequestBuilder.request().decRef(); int a = ir.getShardId().id(); assertTrue(shardA == a || shardA == -1); shardA = a; } for (int i = 10; i < 20; i++) { - DocWriteResponse ir = prepareIndex("value_index").setSource("value", "" + i).setRouting("b").get(); + IndexRequestBuilder indexRequestBuilder = prepareIndex("value_index"); + DocWriteResponse ir = indexRequestBuilder.setSource("value", "" + i).setRouting("b").get(); + indexRequestBuilder.request().decRef(); int b = ir.getShardId().id(); assertTrue(shardB == b || shardB == -1); shardB = b; diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java index 3a82f697acc9d..b8c13f0db1cf5 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.rank.rrf; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; @@ -63,9 +65,9 @@ public void setupIndices() throws Exception { createIndex("tiny_index", indexSettings, builder); ensureGreen("tiny_index"); - prepareIndex("tiny_index").setSource("vector", new float[] { 0.0f }, "text", "term term").get(); - prepareIndex("tiny_index").setSource("vector", new float[] { 1.0f }, "text", "other").get(); - prepareIndex("tiny_index").setSource("vector", new float[] { 2.0f }, "text", "term").get(); + indexDoc("tiny_index", null, "vector", new float[] { 0.0f }, "text", "term term"); + indexDoc("tiny_index", null, "vector", new float[] { 1.0f }, "text", "other"); + indexDoc("tiny_index", null, "vector", new float[] { 2.0f }, "text", "term"); client().admin().indices().prepareRefresh("tiny_index").get(); @@ -103,7 +105,9 @@ public void setupIndices() throws Exception { ensureGreen(TimeValue.timeValueSeconds(120), "nrd_index"); for (int doc = 0; doc < 1001; ++doc) { - prepareIndex("nrd_index").setSource( + indexDoc( + "nrd_index", + null, "vector_asc", new float[] { doc }, "vector_desc", @@ -114,7 +118,7 @@ public void setupIndices() throws Exception { "term " + doc, "text1", "term " + (1000 - doc) - ).get(); + ); } client().admin().indices().prepareRefresh("nrd_index").get(); @@ -973,4 +977,13 @@ public void testMultiBM25AndMultipleKnnWithAggregation() { } ); } + + private DocWriteResponse indexDoc(String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java b/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java index 9a28e11ee1cbb..d43e74da7ffbd 100644 --- a/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java +++ b/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java @@ -71,16 +71,16 @@ public void testPinnedPromotions() throws Exception { for (int i = 0; i < numRelevantDocs; i++) { if (i % 2 == 0) { // add lower-scoring text - prepareIndex("test").setId(Integer.toString(i)).setSource("field1", "the quick brown fox").get(); + indexDoc("test", Integer.toString(i), "field1", "the quick brown fox"); } else { // add higher-scoring text - prepareIndex("test").setId(Integer.toString(i)).setSource("field1", "red fox").get(); + indexDoc("test", Integer.toString(i), "field1", "red fox"); } } // Add docs with no relevance int numIrrelevantDocs = randomIntBetween(1, 10); for (int i = numRelevantDocs; i <= numRelevantDocs + numIrrelevantDocs; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field1", "irrelevant").get(); + indexDoc("test", Integer.toString(i), "field1", "irrelevant"); } refresh(); @@ -177,8 +177,8 @@ public void testExhaustiveScoring() throws Exception { ).setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 1)) ); - prepareIndex("test").setId("1").setSource("field1", "foo").get(); - prepareIndex("test").setId("2").setSource("field1", "foo", "field2", "foo").get(); + indexDoc("test", "1", "field1", "foo"); + indexDoc("test", "2", "field1", "foo", "field2", "foo"); refresh(); @@ -210,10 +210,10 @@ public void testExplain() throws Exception { ) ); ensureGreen(); - prepareIndex("test").setId("1").setSource("field1", "the quick brown fox").get(); - prepareIndex("test").setId("2").setSource("field1", "pinned").get(); - prepareIndex("test").setId("3").setSource("field1", "irrelevant").get(); - prepareIndex("test").setId("4").setSource("field1", "slow brown cat").get(); + indexDoc("test", "1", "field1", "the quick brown fox"); + indexDoc("test", "2", "field1", "pinned"); + indexDoc("test", "3", "field1", "irrelevant"); + indexDoc("test", "4", "field1", "slow brown cat"); refresh(); QueryBuilder organicQuery = QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR); @@ -254,7 +254,7 @@ public void testHighlight() throws Exception { ) ); ensureGreen(); - prepareIndex("test").setId("1").setSource("field1", "the quick brown fox").get(); + indexDoc("test", "1", "field1", "the quick brown fox"); refresh(); QueryBuilder organicQuery = QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR); @@ -309,12 +309,12 @@ public void testMultiIndexDocs() throws Exception { ).setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", randomIntBetween(2, 5))) ); - prepareIndex("test1").setId("a").setSource("field1", "1a bar").get(); - prepareIndex("test1").setId("b").setSource("field1", "1b bar").get(); - prepareIndex("test1").setId("c").setSource("field1", "1c bar").get(); - prepareIndex("test2").setId("a").setSource("field1", "2a bar").get(); - prepareIndex("test2").setId("b").setSource("field1", "2b bar").get(); - prepareIndex("test2").setId("c").setSource("field1", "2c foo").get(); + indexDoc("test1", "a", "field1", "1a bar"); + indexDoc("test1", "b", "field1", "1b bar"); + indexDoc("test1", "c", "field1", "1c bar"); + indexDoc("test2", "a", "field1", "2a bar"); + indexDoc("test2", "b", "field1", "2b bar"); + indexDoc("test2", "c", "field1", "2c foo"); refresh(); @@ -352,9 +352,9 @@ public void testMultiIndexWithAliases() throws Exception { .addAlias(new Alias("test-alias")) ); - prepareIndex("test").setId("a").setSource("field1", "document a").get(); - prepareIndex("test").setId("b").setSource("field1", "document b").get(); - prepareIndex("test").setId("c").setSource("field1", "document c").get(); + indexDoc("test", "a", "field1", "document a"); + indexDoc("test", "b", "field1", "document b"); + indexDoc("test", "c", "field1", "document c"); refresh(); @@ -406,11 +406,11 @@ public void testMultiIndexWithAliasesAndDuplicateIds() throws Exception { ).setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", randomIntBetween(2, 5))).addAlias(alias) ); - prepareIndex("test1").setId("a").setSource("field1", "document a").get(); - prepareIndex("test1").setId("b").setSource("field1", "document b").get(); - prepareIndex("test1").setId("c").setSource("field1", "document c").get(); + indexDoc("test1", "a", "field1", "document a"); + indexDoc("test1", "b", "field1", "document b"); + indexDoc("test1", "c", "field1", "document c"); - prepareIndex("test2").setId("a").setSource("field1", "document a").get(); + indexDoc("test2", "a", "field1", "document a"); refresh(); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java index ad5f57645aa84..c1feabb4067d4 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java @@ -213,6 +213,9 @@ protected void populateIndex(String indexName, int maxIndexRequests) throws Inte indexRequestBuilders.add(prepareIndex(indexName).setSource(key, randomBoolean() ? "bar" : "baz")); } indexRandom(true, true, indexRequestBuilders); + for (IndexRequestBuilder builder : indexRequestBuilders) { + builder.request().decRef(); + } refresh(indexName); if (randomBoolean()) { assertThat( diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java index 82ded22603ef1..6f793ec4b71bb 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java @@ -52,6 +52,9 @@ public void testRepositoriesServiceClusterStateApplierIsCalledBeforeIndicesClust indexRequestBuilders.add(prepareIndex(indexName).setSource("foo", randomBoolean() ? "bar" : "baz")); } indexRandom(true, true, indexRequestBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } refresh(indexName); final SnapshotInfo snapshotInfo = createFullSnapshot(fsRepoName, "snapshot"); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java index 18b4e6ed7cb31..31455c0767b8a 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -433,11 +434,11 @@ public void testRequestCacheOnFrozen() throws Exception { .setMapping("f", "type=date") .setSettings(indexSettings(1, 0).put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)) ); - indexRandom( - true, - prepareIndex("test-index").setSource("f", "2014-03-10T00:00:00.000Z"), - prepareIndex("test-index").setSource("f", "2014-05-13T00:00:00.000Z") - ); + IndexRequestBuilder indexRequestBuilder1 = prepareIndex("test-index").setSource("f", "2014-03-10T00:00:00.000Z"); + IndexRequestBuilder indexRequestBuilder2 = prepareIndex("test-index").setSource("f", "2014-05-13T00:00:00.000Z"); + indexRandom(true, indexRequestBuilder1, indexRequestBuilder2); + indexRequestBuilder1.request().decRef(); + indexRequestBuilder2.request().decRef(); ensureSearchable("test-index"); createRepository("repo", "fs", Settings.builder().put("location", randomRepoPath())); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/RetrySearchIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/RetrySearchIntegTests.java index e3b631ba69c8a..d41475c7a8225 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/RetrySearchIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/RetrySearchIntegTests.java @@ -51,6 +51,9 @@ public void testSearcherId() throws Exception { indexRequestBuilders.add(prepareIndex(indexName).setSource("created_date", "2011-02-02")); } indexRandom(true, false, indexRequestBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } assertThat( indicesAdmin().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), equalTo(0) @@ -118,6 +121,9 @@ public void testRetryPointInTime() throws Exception { indexRequestBuilders.add(prepareIndex(indexName).setSource("created_date", "2011-02-02")); } indexRandom(true, false, indexRequestBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } assertThat( indicesAdmin().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), equalTo(0) diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java index a7a3b8e461604..d7f4744666a2c 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java @@ -757,6 +757,9 @@ private void indexDocumentsWithTimestampWithinDate(String indexName, int docCoun ); } indexRandom(true, false, indexRequestBuilders); + for (IndexRequestBuilder indexRequestBuilder : indexRequestBuilders) { + indexRequestBuilder.request().decRef(); + } assertThat( indicesAdmin().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 38222f64b282b..07e3c85f14374 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -363,7 +363,7 @@ public void testCanMountSnapshotTakenWhileConcurrentlyIndexing() throws Exceptio } try { safeAwait(cyclicBarrier); - indexRandom(true, true, indexRequestBuilders); + indexRandomAndDecRefRequests(true, true, indexRequestBuilders); } catch (InterruptedException e) { throw new AssertionError(e); } @@ -417,7 +417,7 @@ public void testMaxRestoreBytesPerSecIsUsed() throws Exception { assertAcked(prepareCreate(indexName, indexSettings)); final int nbDocs = between(10, 50); - indexRandom( + indexRandomAndDecRefRequests( true, false, IntStream.range(0, nbDocs) @@ -706,7 +706,7 @@ public void testSnapshotMountedIndexWithTimestampsRecordsTimestampRangeInIndexMe ) ); } - indexRandom(true, false, indexRequestBuilders); + indexRandomAndDecRefRequests(true, false, indexRequestBuilders); assertThat( indicesAdmin().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), equalTo(0) @@ -1158,4 +1158,15 @@ private void assertSearchableSnapshotStats(String indexName, boolean cacheEnable private static long max(long... values) { return Arrays.stream(values).max().orElseThrow(() -> new AssertionError("no values")); } + + private void indexRandomAndDecRefRequests(boolean forceRefresh, boolean dummyDocuments, List builders) + throws InterruptedException { + try { + indexRandom(forceRefresh, dummyDocuments, builders); + } finally { + for (IndexRequestBuilder builder : builders) { + builder.request().decRef(); + } + } + } } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java index 56074f97650f0..a78b316278ef3 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java @@ -371,6 +371,9 @@ private Map> mountRandomIndicesWithCache(String re indexRequestBuilders.add(prepareIndex(indexName).setSource(builder)); } indexRandom(true, indexRequestBuilders); + for (IndexRequestBuilder builder : indexRequestBuilders) { + builder.request().decRef(); + } final String snapshot = "snapshot-" + i; createSnapshot(repositoryName, snapshot, List.of(indexName)); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java index 7bb7343cdb47f..c6e922b95dd2c 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.elasticsearch.cluster.service.ClusterService; @@ -138,7 +139,9 @@ public void testConcurrentPrewarming() throws Exception { if (nbDocs > 0) { try (BulkRequestBuilder bulkRequest = client().prepareBulk()) { for (int i = 0; i < nbDocs; i++) { - bulkRequest.add(prepareIndex(indexName).setSource("foo", randomBoolean() ? "bar" : "baz")); + IndexRequestBuilder indexRequestBuilder = prepareIndex(indexName).setSource("foo", randomBoolean() ? "bar" : "baz"); + bulkRequest.add(indexRequestBuilder); + indexRequestBuilder.request().decRef(); } final BulkResponse bulkResponse = bulkRequest.get(); assertThat(bulkResponse.hasFailures(), is(false)); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheService.java index a7ba0294d5c98..5d094962116ac 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheService.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheService.java @@ -250,10 +250,6 @@ public final void putAsync( bytes, range.start() ); - final IndexRequest request = new IndexRequest(index).id(id); - try (XContentBuilder builder = jsonBuilder()) { - request.source(cachedBlob.toXContent(builder, ToXContent.EMPTY_PARAMS)); - } final RunOnce release = new RunOnce(() -> { final int availablePermits = inFlightCacheFills.availablePermits(); @@ -263,8 +259,15 @@ public final void putAsync( boolean submitted = false; inFlightCacheFills.acquire(); + final IndexRequest request = new IndexRequest(index).id(id); try { - final ActionListener wrappedListener = ActionListener.runAfter(listener, release); + try (XContentBuilder builder = jsonBuilder()) { + request.source(cachedBlob.toXContent(builder, ToXContent.EMPTY_PARAMS)); + } + final ActionListener wrappedListener = ActionListener.runAfter( + ActionListener.runAfter(listener, release), + request::decRef + ); innerPut(request, new ActionListener<>() { @Override public void onResponse(DocWriteResponse indexResponse) { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/BulkUpdateTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/BulkUpdateTests.java index c882396e53b37..49e4ac8c0d573 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/BulkUpdateTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/BulkUpdateTests.java @@ -11,6 +11,8 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.common.settings.SecureString; @@ -42,10 +44,7 @@ public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { } public void testThatBulkUpdateDoesNotLoseFields() { - assertEquals( - DocWriteResponse.Result.CREATED, - prepareIndex("index1").setSource("{\"test\": \"test\"}", XContentType.JSON).setId("1").get().getResult() - ); + assertEquals(DocWriteResponse.Result.CREATED, index("index1", "1", "{\"test\": \"test\"}", XContentType.JSON).getResult()); GetResponse getResponse = client().prepareGet("index1", "1").get(); assertEquals("test", getResponse.getSource().get("test")); @@ -54,10 +53,12 @@ public void testThatBulkUpdateDoesNotLoseFields() { } // update with a new field + UpdateRequestBuilder updateRequestBuilder1 = client().prepareUpdate("index1", "1"); assertEquals( DocWriteResponse.Result.UPDATED, - client().prepareUpdate("index1", "1").setDoc("{\"not test\": \"not test\"}", XContentType.JSON).get().getResult() + updateRequestBuilder1.setDoc("{\"not test\": \"not test\"}", XContentType.JSON).get().getResult() ); + updateRequestBuilder1.request().decRef(); getResponse = client().prepareGet("index1", "1").get(); assertEquals("test", getResponse.getSource().get("test")); assertEquals("not test", getResponse.getSource().get("not test")); @@ -68,9 +69,11 @@ public void testThatBulkUpdateDoesNotLoseFields() { // do it in a bulk try (BulkRequestBuilder bulkRequestBuilder = client().prepareBulk()) { + UpdateRequestBuilder updateRequestBuilder2 = client().prepareUpdate("index1", "1"); BulkResponse response = bulkRequestBuilder.add( - client().prepareUpdate("index1", "1").setDoc("{\"bulk updated\": \"bulk updated\"}", XContentType.JSON) + updateRequestBuilder2.setDoc("{\"bulk updated\": \"bulk updated\"}", XContentType.JSON) ).get(); + updateRequestBuilder2.request().decRef(); assertEquals(DocWriteResponse.Result.UPDATED, response.getItems()[0].getResponse().getResult()); getResponse = client().prepareGet("index1", "1").get(); assertEquals("test", getResponse.getSource().get("test")); @@ -131,4 +134,13 @@ public void testThatBulkUpdateDoesNotLoseFieldsHttp() throws IOException { assertThat(afterBulk, containsString("\"not test\":\"not test\"")); assertThat(afterBulk, containsString("\"bulk updated\":\"bulk updated\"")); } + + private DocWriteResponse index(String index, String id, String source, XContentType contentType) { + IndexRequestBuilder indexRequestBuilder = prepareIndex(index); + try { + return indexRequestBuilder.setId(id).setSource(source, contentType).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index 766e3b6055a87..1150994000a6b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -383,10 +383,15 @@ private void bulkIndex(Client client, String dataStreamName, Supplier assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L)) ); - UpdateResponse updateResponse = client.prepareUpdate(expression, response.getId()) - .setDoc(Requests.INDEX_CONTENT_TYPE, "new", "field") + UpdateRequestBuilder updateRequestBuilder = client.prepareUpdate(expression, response.getId()); + UpdateResponse updateResponse = updateRequestBuilder.setDoc(Requests.INDEX_CONTENT_TYPE, "new", "field") .setRefreshPolicy(refeshOnOperation ? IMMEDIATE : NONE) .get(); + updateRequestBuilder.request().decRef(); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); if (refeshOnOperation == false) { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java index 3fbcd00690e82..136bc66a36152 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.client.internal.Client; @@ -352,22 +353,22 @@ private void prepareIndices() { {"script":{"source":"{\\"match\\":{\\"username\\":\\"{{_user.username}}\\"}}","lang":"mustache"}}"""), XContentType.JSON)); assertAcked(indicesAdmin().prepareCreate(DLS_INDEX).addAlias(new Alias("dls-alias")).get()); - client.prepareIndex(DLS_INDEX).setId("101").setSource("number", 101, "letter", "A").get(); - client.prepareIndex(DLS_INDEX).setId("102").setSource("number", 102, "letter", "B").get(); + indexDoc(client, DLS_INDEX, "101", "number", 101, "letter", "A"); + indexDoc(client, DLS_INDEX, "102", "number", 102, "letter", "B"); assertAcked(indicesAdmin().prepareCreate(FLS_INDEX).addAlias(new Alias("fls-alias")).get()); - client.prepareIndex(FLS_INDEX).setId("201").setSource("public", "X", "private", "x").get(); - client.prepareIndex(FLS_INDEX).setId("202").setSource("public", "Y", "private", "y").get(); + indexDoc(client, FLS_INDEX, "201", "public", "X", "private", "x"); + indexDoc(client, FLS_INDEX, "202", "public", "Y", "private", "y"); assertAcked( indicesAdmin().prepareCreate(INDEX).addAlias(new Alias(ALIAS1)).addAlias(new Alias(ALIAS2)).addAlias(new Alias(ALL_ALIAS)) ); - client.prepareIndex(INDEX).setId("1").setSource("number", 1, "letter", "a", "private", "sesame_1", "public", "door_1").get(); - client.prepareIndex(INDEX).setId("2").setSource("number", 2, "letter", "b", "private", "sesame_2", "public", "door_2").get(); + indexDoc(client, INDEX, "1", "number", 1, "letter", "a", "private", "sesame_1", "public", "door_1"); + indexDoc(client, INDEX, "2", "number", 2, "letter", "b", "private", "sesame_2", "public", "door_2"); assertAcked(indicesAdmin().prepareCreate(DLS_TEMPLATE_ROLE_QUERY_INDEX).addAlias(new Alias(DLS_TEMPLATE_ROLE_QUERY_ALIAS)).get()); - client.prepareIndex(DLS_TEMPLATE_ROLE_QUERY_INDEX).setId("1").setSource("username", DLS_TEMPLATE_ROLE_QUERY_USER_1).get(); - client.prepareIndex(DLS_TEMPLATE_ROLE_QUERY_INDEX).setId("2").setSource("username", DLS_TEMPLATE_ROLE_QUERY_USER_2).get(); + indexDoc(client, DLS_TEMPLATE_ROLE_QUERY_INDEX, "1", "username", DLS_TEMPLATE_ROLE_QUERY_USER_1); + indexDoc(client, DLS_TEMPLATE_ROLE_QUERY_INDEX, "2", "username", DLS_TEMPLATE_ROLE_QUERY_USER_2); ensureGreen(DLS_INDEX, FLS_INDEX, INDEX, DLS_TEMPLATE_ROLE_QUERY_INDEX); assertCacheState(DLS_INDEX, 0, 0); @@ -449,4 +450,13 @@ private void assertCacheState(String index, long expectedHits, long expectedMiss ); } + private void indexDoc(Client client, String index, String id, Object... source) { + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(index); + try { + indexRequestBuilder.setId(id).setSource(source).get(); + } finally { + indexRequestBuilder.request().decRef(); + } + } + } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index 741752a8c42c8..0bd0188ba70cd 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.integration; import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; @@ -15,6 +16,8 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.internal.Requests; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.settings.SecureString; @@ -111,8 +114,8 @@ public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { public void testSimpleQuery() { assertAcked(indicesAdmin().prepareCreate("test").setMapping("id", "type=keyword", "field1", "type=text", "field2", "type=text")); - prepareIndex("test").setId("1").setSource("id", "1", "field1", "value1").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("test").setId("2").setSource("id", "2", "field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "id", "1", "field1", "value1"); + indexDocImmediate("test", "2", "id", "2", "field2", "value2"); assertResponse( client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD))) @@ -158,27 +161,32 @@ public void testUpdatesAreRejected() { .setMapping("id", "type=keyword", "field1", "type=text", "field2", "type=text") .setSettings(indexSettings(1, 0)) ); - prepareIndex(indexName).setId("1").setSource("id", "1", "field1", "value1").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate(indexName, "1", "id", "1", "field1", "value1"); - ElasticsearchSecurityException exception = expectThrows(ElasticsearchSecurityException.class, () -> { - client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD))) - .prepareUpdate(indexName, "1") - .setDoc(Requests.INDEX_CONTENT_TYPE, "field2", "value2") - .get(); - }); - assertThat( - exception.getDetailedMessage(), - containsString("Can't execute an update request if field or document level " + "security") - ); + UpdateRequestBuilder updateRequestBuilder1 = client().filterWithHeader( + Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)) + ).prepareUpdate(indexName, "1"); + try { + ElasticsearchSecurityException exception = expectThrows(ElasticsearchSecurityException.class, () -> { + updateRequestBuilder1.setDoc(Requests.INDEX_CONTENT_TYPE, "field2", "value2").get(); + }); + assertThat( + exception.getDetailedMessage(), + containsString("Can't execute an update request if field or document level " + "security") + ); + } finally { + updateRequestBuilder1.request().decRef(); + } try ( BulkRequestBuilder bulkRequestBuilder = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)) ).prepareBulk() ) { - BulkResponse bulkResponse = bulkRequestBuilder.add( - client().prepareUpdate(indexName, "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field2", "value2") - ).get(); + UpdateRequestBuilder updateRequestBuilder2 = client().prepareUpdate(indexName, "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field2", "value2"); + BulkResponse bulkResponse = bulkRequestBuilder.add(updateRequestBuilder2).get(); + updateRequestBuilder2.request().decRef(); assertThat(bulkResponse.getItems().length, is(1)); assertThat( bulkResponse.getItems()[0].getFailureMessage(), @@ -194,8 +202,8 @@ public void testUpdatesAreRejected() { public void testDLSIsAppliedBeforeFLS() { assertAcked(indicesAdmin().prepareCreate("test").setMapping("field1", "type=text", "field2", "type=text")); - prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value1").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("test").setId("2").setSource("field1", "value2", "field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1", "field2", "value1"); + indexDocImmediate("test", "2", "field1", "value2", "field2", "value2"); assertResponse( client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user5", USERS_PASSWD))) @@ -223,8 +231,8 @@ public void testQueryCache() { .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)) .setMapping("id", "type=keyword", "field1", "type=text", "field2", "type=text") ); - prepareIndex("test").setId("1").setSource("id", "1", "field1", "value1").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("test").setId("2").setSource("id", "2", "field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "id", "1", "field1", "value1"); + indexDocImmediate("test", "2", "id", "2", "field2", "value2"); // Both users have the same role query, but user3 has access to field2 and not field1, which should result in zero hits: int max = scaledRandomIntBetween(4, 32); @@ -288,8 +296,8 @@ public void testQueryCache() { public void testGetMappingsIsFiltered() { assertAcked(indicesAdmin().prepareCreate("test").setMapping("field1", "type=text", "field2", "type=text")); - prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("test").setId("2").setSource("field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1"); + indexDocImmediate("test", "2", "field2", "value2"); { GetMappingsResponse getMappingsResponse = client().filterWithHeader( @@ -322,8 +330,8 @@ public void testGetMappingsIsFiltered() { public void testGetIndexMappingsIsFiltered() { assertAcked(indicesAdmin().prepareCreate("test").setMapping("field1", "type=text", "field2", "type=text")); - prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("test").setId("2").setSource("field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1"); + indexDocImmediate("test", "2", "field2", "value2"); { GetIndexResponse getIndexResponse = client().filterWithHeader( @@ -353,8 +361,8 @@ public void testGetIndexMappingsIsFiltered() { public void testGetFieldMappingsIsFiltered() { assertAcked(indicesAdmin().prepareCreate("test").setMapping("field1", "type=text", "field2", "type=text")); - prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("test").setId("2").setSource("field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1"); + indexDocImmediate("test", "2", "field2", "value2"); { GetFieldMappingsResponse getFieldMappingsResponse = client().filterWithHeader( @@ -396,8 +404,8 @@ public void testGetFieldMappingsIsFiltered() { public void testFieldCapabilitiesIsFiltered() { assertAcked(indicesAdmin().prepareCreate("test").setMapping("field1", "type=text", "field2", "type=text")); - prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex("test").setId("2").setSource("field2", "value2").setRefreshPolicy(IMMEDIATE).get(); + indexDocImmediate("test", "1", "field1", "value1"); + indexDocImmediate("test", "2", "field2", "value2"); { FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest().fields("*").indices("test"); @@ -468,4 +476,13 @@ private static void assertExpectedFields(Map