Skip to content

Commit 5f52bf3

Browse files
gregkalaposgregkalapos
authored
Merge branch 'main' into otel_profiling_integration
2 parents 34cb3a6 + 24fa8ea commit 5f52bf3

File tree

22 files changed

+445
-109
lines changed

22 files changed

+445
-109
lines changed

‎libs/entitlement/README.md‎

Lines changed: 187 additions & 6 deletions
Large diffs are not rendered by default.

‎modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@
8080
import java.util.Objects;
8181
import java.util.OptionalInt;
8282
import java.util.Set;
83-
import java.util.concurrent.atomic.AtomicBoolean;
8483
import java.util.concurrent.atomic.AtomicInteger;
8584
import java.util.concurrent.atomic.AtomicLong;
8685
import java.util.function.IntConsumer;
@@ -114,19 +113,19 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes
114113

115114
private static final int MAX_NUMBER_SNAPSHOT_DELETE_RETRIES = 10;
116115
private S3Service service;
117-
private AtomicBoolean shouldErrorOnDns;
116+
private volatile boolean shouldErrorOnDns;
118117
private RecordingMeterRegistry recordingMeterRegistry;
119118

120119
@Before
121120
public void setUp() throws Exception {
122-
shouldErrorOnDns = new AtomicBoolean(false);
121+
shouldErrorOnDns = false;
123122
service = new S3Service(Mockito.mock(Environment.class), Settings.EMPTY, Mockito.mock(ResourceWatcherService.class)) {
124123
@Override
125124
protected AmazonS3ClientBuilder buildClientBuilder(S3ClientSettings clientSettings) {
126125
final AmazonS3ClientBuilder builder = super.buildClientBuilder(clientSettings);
127126
final DnsResolver defaultDnsResolver = builder.getClientConfiguration().getDnsResolver();
128127
builder.getClientConfiguration().setDnsResolver(host -> {
129-
if (shouldErrorOnDns.get() && randomBoolean() && randomBoolean()) {
128+
if (shouldErrorOnDns && randomBoolean() && randomBoolean()) {
130129
throw new UnknownHostException(host);
131130
}
132131
return defaultDnsResolver.resolve(host);
@@ -665,7 +664,7 @@ public void testReadWithIndicesPurposeRetriesForever() throws IOException {
665664

666665
final byte[] bytes = randomBlobContent(512);
667666

668-
shouldErrorOnDns.set(true);
667+
shouldErrorOnDns = true;
669668
final AtomicInteger failures = new AtomicInteger();
670669
@SuppressForbidden(reason = "use a http server")
671670
class FlakyReadHandler implements HttpHandler {
@@ -893,7 +892,7 @@ private int expectedNumberOfBatches(int blobsToDelete) {
893892
}
894893

895894
@SuppressForbidden(reason = "use a http server")
896-
private class ThrottlingDeleteHandler extends S3HttpHandler {
895+
private static class ThrottlingDeleteHandler extends S3HttpHandler {
897896

898897
private static final String THROTTLING_ERROR_CODE = "SlowDown";
899898

@@ -918,7 +917,7 @@ private class ThrottlingDeleteHandler extends S3HttpHandler {
918917

919918
@Override
920919
public void handle(HttpExchange exchange) throws IOException {
921-
if (exchange.getRequestMethod().equals("POST") && exchange.getRequestURI().toString().startsWith("/bucket/?delete")) {
920+
if (isMultiDeleteRequest(exchange)) {
922921
onAttemptCallback.accept(numberOfDeleteAttempts.get());
923922
numberOfDeleteAttempts.incrementAndGet();
924923
if (throttleTimesBeforeSuccess.getAndDecrement() > 0) {
@@ -1022,7 +1021,7 @@ public void testSuppressedDeletionErrorsAreCapped() {
10221021
int maxBulkDeleteSize = randomIntBetween(1, 10);
10231022
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize);
10241023
httpServer.createContext("/", exchange -> {
1025-
if (exchange.getRequestMethod().equals("POST") && exchange.getRequestURI().toString().startsWith("/bucket/?delete")) {
1024+
if (isMultiDeleteRequest(exchange)) {
10261025
exchange.sendResponseHeaders(
10271026
randomFrom(
10281027
HttpStatus.SC_INTERNAL_SERVER_ERROR,
@@ -1056,7 +1055,7 @@ public void testTrimmedLogAndCappedSuppressedErrorOnMultiObjectDeletionException
10561055

10571056
final Pattern pattern = Pattern.compile("<Key>(.+?)</Key>");
10581057
httpServer.createContext("/", exchange -> {
1059-
if (exchange.getRequestMethod().equals("POST") && exchange.getRequestURI().toString().startsWith("/bucket/?delete")) {
1058+
if (isMultiDeleteRequest(exchange)) {
10601059
final String requestBody = Streams.copyToString(new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8));
10611060
final var matcher = pattern.matcher(requestBody);
10621061
final StringBuilder deletes = new StringBuilder();
@@ -1208,6 +1207,10 @@ private Map<String, Object> metricAttributes(String action) {
12081207
return Map.of("repo_type", "s3", "repo_name", "repository", "operation", "GetObject", "purpose", "Indices", "action", action);
12091208
}
12101209

1210+
private static boolean isMultiDeleteRequest(HttpExchange exchange) {
1211+
return new S3HttpHandler("bucket").parseRequest(exchange).isMultiObjectDeleteRequest();
1212+
}
1213+
12111214
/**
12121215
* Asserts that an InputStream is fully consumed, or aborted, when it is closed
12131216
*/

‎muted-tests.yml‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,6 @@ tests:
219219
- class: org.elasticsearch.blocks.SimpleBlocksIT
220220
method: testConcurrentAddBlock
221221
issue: https://github.com/elastic/elasticsearch/issues/122324
222-
- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
223-
method: testEnrichExplosionManyMatches
224-
issue: https://github.com/elastic/elasticsearch/issues/122913
225222
- class: org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT
226223
method: testHistoryIsWrittenWithFailure
227224
issue: https://github.com/elastic/elasticsearch/issues/123203
@@ -243,9 +240,6 @@ tests:
243240
- class: org.elasticsearch.indices.recovery.IndexRecoveryIT
244241
method: testSourceThrottling
245242
issue: https://github.com/elastic/elasticsearch/issues/123680
246-
- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
247-
method: testLookupExplosionNoFetch
248-
issue: https://github.com/elastic/elasticsearch/issues/123432
249243
- class: org.elasticsearch.entitlement.runtime.policy.FileAccessTreeTests
250244
method: testDuplicatePrunedPaths
251245
issue: https://github.com/elastic/elasticsearch/issues/124006
@@ -276,9 +270,6 @@ tests:
276270
- class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
277271
method: test {yaml=data_stream/190_failure_store_redirection/Redirect ingest failure in data stream to failure store}
278272
issue: https://github.com/elastic/elasticsearch/issues/124518
279-
- class: org.elasticsearch.xpack.esql.expression.function.aggregate.ValuesTests
280-
method: "testGroupingAggregate {TestCase=<<no alt geo_shape>s> #2}"
281-
issue: https://github.com/elastic/elasticsearch/issues/124571
282273
- class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
283274
method: test {yaml=data_stream/150_tsdb/created the data stream}
284275
issue: https://github.com/elastic/elasticsearch/issues/124575
@@ -309,9 +300,6 @@ tests:
309300
- class: org.elasticsearch.index.shard.StoreRecoveryTests
310301
method: testAddIndices
311302
issue: https://github.com/elastic/elasticsearch/issues/124104
312-
- class: org.elasticsearch.xpack.esql.parser.StatementParserTests
313-
method: testInvalidJoinPatterns
314-
issue: https://github.com/elastic/elasticsearch/issues/125536
315303
- class: org.elasticsearch.smoketest.MlWithSecurityIT
316304
method: test {yaml=ml/data_frame_analytics_crud/Test get stats on newly created config}
317305
issue: https://github.com/elastic/elasticsearch/issues/121726
@@ -348,9 +336,9 @@ tests:
348336
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
349337
method: test {p0=ml/start_data_frame_analytics/Test start given dest index is not empty}
350338
issue: https://github.com/elastic/elasticsearch/issues/125909
351-
- class: org.elasticsearch.xpack.esql.action.ManyShardsIT
352-
method: testCancelUnnecessaryRequests
353-
issue: https://github.com/elastic/elasticsearch/issues/125947
339+
- class: org.elasticsearch.indices.stats.IndexStatsIT
340+
method: testThrottleStats
341+
issue: https://github.com/elastic/elasticsearch/issues/125910
354342
- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT
355343
method: testResumingSearchableSnapshotFromPartialToFull
356344
issue: https://github.com/elastic/elasticsearch/issues/125789

‎test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -207,10 +207,6 @@ public void handle(final HttpExchange exchange) throws IOException {
207207
exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1);
208208

209209
} else if (request.isListObjectsRequest()) {
210-
if (request.queryParameters().containsKey("list-type")) {
211-
throw new AssertionError("Test must be adapted for GET Bucket (List Objects) Version 2");
212-
}
213-
214210
final StringBuilder list = new StringBuilder();
215211
list.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
216212
list.append("<ListBucketResult>");
@@ -223,6 +219,9 @@ public void handle(final HttpExchange exchange) throws IOException {
223219
if (delimiter != null) {
224220
list.append("<Delimiter>").append(delimiter).append("</Delimiter>");
225221
}
222+
// Would be good to test pagination here (the only real difference between ListObjects and ListObjectsV2) but for now
223+
// we return all the results at once.
224+
list.append("<IsTruncated>false</IsTruncated>");
226225
for (Map.Entry<String, BytesReference> blob : blobs.entrySet()) {
227226
if (prefix != null && blob.getKey().startsWith("/" + bucket + "/" + prefix) == false) {
228227
continue;

‎test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java‎

Lines changed: 51 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import com.sun.net.httpserver.HttpExchange;
1515
import com.sun.net.httpserver.HttpPrincipal;
1616

17+
import org.elasticsearch.common.Randomness;
1718
import org.elasticsearch.common.Strings;
1819
import org.elasticsearch.common.bytes.BytesArray;
1920
import org.elasticsearch.common.bytes.BytesReference;
@@ -28,6 +29,7 @@
2829
import java.net.InetSocketAddress;
2930
import java.net.URI;
3031
import java.nio.charset.StandardCharsets;
32+
import java.util.ArrayList;
3133
import java.util.List;
3234
import java.util.Objects;
3335

@@ -45,63 +47,81 @@ public void testRejectsBadUri() {
4547
);
4648
}
4749

50+
private static void assertListObjectsResponse(
51+
S3HttpHandler handler,
52+
@Nullable String prefix,
53+
@Nullable String delimiter,
54+
String expectedResponse
55+
) {
56+
final var queryParts = new ArrayList<String>(3);
57+
if (prefix != null) {
58+
queryParts.add("prefix=" + prefix);
59+
}
60+
if (delimiter != null) {
61+
queryParts.add("delimiter=" + delimiter);
62+
}
63+
if (randomBoolean()) {
64+
// test both ListObjects and ListObjectsV2 - they only differ in terms of pagination but S3HttpHandler doesn't do that
65+
queryParts.add("list-type=2");
66+
}
67+
Randomness.shuffle(queryParts);
68+
69+
final var requestUri = "/bucket" + randomFrom("", "/") + (queryParts.isEmpty() ? "" : "?" + String.join("&", queryParts));
70+
assertEquals("GET " + requestUri, new TestHttpResponse(RestStatus.OK, expectedResponse), handleRequest(handler, "GET", requestUri));
71+
}
72+
4873
public void testSimpleObjectOperations() {
4974
final var handler = new S3HttpHandler("bucket", "path");
5075

5176
assertEquals(RestStatus.NOT_FOUND, handleRequest(handler, "GET", "/bucket/path/blob").status());
5277

53-
assertEquals(
54-
new TestHttpResponse(RestStatus.OK, """
55-
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix></ListBucketResult>"""),
56-
handleRequest(handler, "GET", "/bucket/?prefix=")
57-
);
78+
assertListObjectsResponse(handler, "", null, """
79+
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
80+
</ListBucketResult>""");
5881

5982
final var body = randomAlphaOfLength(50);
6083
assertEquals(RestStatus.OK, handleRequest(handler, "PUT", "/bucket/path/blob", body).status());
6184
assertEquals(new TestHttpResponse(RestStatus.OK, body), handleRequest(handler, "GET", "/bucket/path/blob"));
6285

63-
assertEquals(new TestHttpResponse(RestStatus.OK, """
64-
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix>\
86+
assertListObjectsResponse(handler, "", null, """
87+
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
6588
<Contents><Key>path/blob</Key><Size>50</Size></Contents>\
66-
</ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?prefix="));
89+
</ListBucketResult>""");
6790

68-
assertEquals(new TestHttpResponse(RestStatus.OK, """
69-
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/</Prefix>\
91+
assertListObjectsResponse(handler, "path/", null, """
92+
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/</Prefix><IsTruncated>false</IsTruncated>\
7093
<Contents><Key>path/blob</Key><Size>50</Size></Contents>\
71-
</ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?prefix=path/"));
94+
</ListBucketResult>""");
7295

73-
assertEquals(
74-
new TestHttpResponse(RestStatus.OK, """
75-
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/other</Prefix></ListBucketResult>"""),
76-
handleRequest(handler, "GET", "/bucket/?prefix=path/other")
77-
);
96+
assertListObjectsResponse(handler, "path/other", null, """
97+
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/other</Prefix><IsTruncated>false</IsTruncated>\
98+
</ListBucketResult>""");
7899

79100
assertEquals(RestStatus.OK, handleRequest(handler, "PUT", "/bucket/path/subpath1/blob", randomAlphaOfLength(50)).status());
80101
assertEquals(RestStatus.OK, handleRequest(handler, "PUT", "/bucket/path/subpath2/blob", randomAlphaOfLength(50)).status());
81-
assertEquals(new TestHttpResponse(RestStatus.OK, """
82-
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/</Prefix>\
83-
<Delimiter>/</Delimiter><Contents><Key>path/blob</Key><Size>50</Size></Contents>\
102+
assertListObjectsResponse(handler, "path/", "/", """
103+
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult>\
104+
<Prefix>path/</Prefix><Delimiter>/</Delimiter><IsTruncated>false</IsTruncated>\
105+
<Contents><Key>path/blob</Key><Size>50</Size></Contents>\
84106
<CommonPrefixes><Prefix>path/subpath1/</Prefix></CommonPrefixes>\
85107
<CommonPrefixes><Prefix>path/subpath2/</Prefix></CommonPrefixes>\
86-
</ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?delimiter=/&prefix=path/"));
108+
</ListBucketResult>""");
87109

88110
assertEquals(RestStatus.OK, handleRequest(handler, "DELETE", "/bucket/path/blob").status());
89111
assertEquals(RestStatus.NO_CONTENT, handleRequest(handler, "DELETE", "/bucket/path/blob").status());
90112

91-
assertEquals(new TestHttpResponse(RestStatus.OK, """
92-
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix>\
113+
assertListObjectsResponse(handler, "", null, """
114+
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
93115
<Contents><Key>path/subpath1/blob</Key><Size>50</Size></Contents>\
94116
<Contents><Key>path/subpath2/blob</Key><Size>50</Size></Contents>\
95-
</ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?prefix="));
117+
</ListBucketResult>""");
96118

97119
assertEquals(RestStatus.OK, handleRequest(handler, "DELETE", "/bucket/path/subpath1/blob").status());
98120
assertEquals(RestStatus.OK, handleRequest(handler, "DELETE", "/bucket/path/subpath2/blob").status());
99121

100-
assertEquals(
101-
new TestHttpResponse(RestStatus.OK, """
102-
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix></ListBucketResult>"""),
103-
handleRequest(handler, "GET", "/bucket/?prefix=")
104-
);
122+
assertListObjectsResponse(handler, "", null, """
123+
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
124+
</ListBucketResult>""");
105125
}
106126

107127
public void testGetWithBytesRange() {
@@ -216,10 +236,10 @@ public void testSingleMultipartUpload() {
216236
</CompleteMultipartUpload>""", part1Etag, part2Etag))
217237
);
218238

219-
assertEquals(new TestHttpResponse(RestStatus.OK, """
220-
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix>\
239+
assertListObjectsResponse(handler, "", null, """
240+
<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
221241
<Contents><Key>path/blob</Key><Size>100</Size></Contents>\
222-
</ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?prefix="));
242+
</ListBucketResult>""");
223243

224244
assertEquals(new TestHttpResponse(RestStatus.OK, part1 + part2), handleRequest(handler, "GET", "/bucket/path/blob"));
225245

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperator.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
package org.elasticsearch.compute.operator;
99

10+
import org.elasticsearch.common.Rounding;
1011
import org.elasticsearch.compute.Describable;
1112
import org.elasticsearch.compute.aggregation.AggregatorMode;
1213
import org.elasticsearch.compute.aggregation.GroupingAggregator;
@@ -23,6 +24,7 @@
2324
public class TimeSeriesAggregationOperator extends HashAggregationOperator {
2425

2526
public record Factory(
27+
Rounding.Prepared timeBucket,
2628
List<BlockHash.GroupSpec> groups,
2729
AggregatorMode aggregatorMode,
2830
List<GroupingAggregator.Factory> aggregators,
@@ -32,6 +34,7 @@ public record Factory(
3234
public Operator get(DriverContext driverContext) {
3335
// TODO: use TimeSeriesBlockHash when possible
3436
return new TimeSeriesAggregationOperator(
37+
timeBucket,
3538
aggregators,
3639
() -> BlockHash.build(
3740
groups,
@@ -53,11 +56,15 @@ public String describe() {
5356
}
5457
}
5558

59+
private final Rounding.Prepared timeBucket;
60+
5661
public TimeSeriesAggregationOperator(
62+
Rounding.Prepared timeBucket,
5763
List<GroupingAggregator.Factory> aggregators,
5864
Supplier<BlockHash> blockHash,
5965
DriverContext driverContext
6066
) {
6167
super(aggregators, blockHash, driverContext);
68+
this.timeBucket = timeBucket;
6269
}
6370
}

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperatorFactories.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
package org.elasticsearch.compute.operator;
99

10+
import org.elasticsearch.common.Rounding;
1011
import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier;
1112
import org.elasticsearch.compute.aggregation.AggregatorMode;
1213
import org.elasticsearch.compute.aggregation.GroupingAggregator;
@@ -46,6 +47,7 @@ public record SupplierWithChannels(AggregatorFunctionSupplier supplier, List<Int
4647
public record Initial(
4748
int tsHashChannel,
4849
int timeBucketChannel,
50+
Rounding.Prepared timeBucket,
4951
List<BlockHash.GroupSpec> groupings,
5052
List<SupplierWithChannels> rates,
5153
List<SupplierWithChannels> nonRates,
@@ -62,6 +64,7 @@ public Operator get(DriverContext driverContext) {
6264
}
6365
aggregators.addAll(valuesAggregatorForGroupings(groupings, timeBucketChannel));
6466
return new TimeSeriesAggregationOperator(
67+
timeBucket,
6568
aggregators,
6669
() -> new TimeSeriesBlockHash(tsHashChannel, timeBucketChannel, driverContext.blockFactory()),
6770
driverContext
@@ -77,6 +80,7 @@ public String describe() {
7780
public record Intermediate(
7881
int tsHashChannel,
7982
int timeBucketChannel,
83+
Rounding.Prepared timeBucket,
8084
List<BlockHash.GroupSpec> groupings,
8185
List<SupplierWithChannels> rates,
8286
List<SupplierWithChannels> nonRates,
@@ -97,6 +101,7 @@ public Operator get(DriverContext driverContext) {
97101
new BlockHash.GroupSpec(timeBucketChannel, ElementType.LONG)
98102
);
99103
return new TimeSeriesAggregationOperator(
104+
timeBucket,
100105
aggregators,
101106
() -> BlockHash.build(hashGroups, driverContext.blockFactory(), maxPageSize, true),
102107
driverContext

0 commit comments

Comments
 (0)