Skip to content

Commit 0e88f21

Browse files
davidkyledavidkyle
authored
Merge branch 'main' into no-conflicts
2 parents 81e4979 + 953b9fb commit 0e88f21

File tree

93 files changed

+2675
-455
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+2675
-455
lines changed

‎docs/changelog/124832.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 124832
2+
summary: List/get query API
3+
area: ES|QL
4+
type: feature
5+
issues:
6+
- 124827

‎docs/changelog/126409.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 126409
2+
summary: System data streams are not being upgraded in the feature migration API
3+
area: Infra/Core
4+
type: bug
5+
issues:
6+
- 122949

‎docs/reference/elasticsearch/security-privileges.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,9 @@ This section lists the privileges that you can assign to a role.
194194
`monitor_enrich`
195195
: All read-only operations related to managing and executing enrich policies.
196196

197+
`monitor_esql`
198+
: All read-only operations related to ES|QL queries.
199+
197200
`monitor_inference`
198201
: All read-only operations related to {{infer}}.
199202

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
326326
.build(),
327327
Map.of(),
328328
List.of("product"),
329+
"product",
329330
ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
330331
)
331332
);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
275275
.build(),
276276
Map.of(),
277277
Collections.singletonList("test"),
278+
"test",
278279
new ExecutorNames(ThreadPool.Names.SYSTEM_CRITICAL_READ, ThreadPool.Names.SYSTEM_READ, ThreadPool.Names.SYSTEM_WRITE)
279280
)
280281
);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
213213
.build(),
214214
Map.of(),
215215
List.of("product"),
216+
"product",
216217
ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
217218
)
218219
);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1094,6 +1094,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
10941094
.build(),
10951095
Map.of(),
10961096
List.of(),
1097+
"test",
10971098
ExecutorNames.DEFAULT_SYSTEM_INDEX_THREAD_POOLS
10981099
)
10991100
);

‎modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ public void testShouldRetryOnUnresolvableHost() {
229229

230230
private void executeListBlobsAndAssertRetries() {
231231
final int maxRetries = randomIntBetween(3, 5);
232-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null);
232+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
233233
expectThrows(StorageException.class, () -> blobContainer.listBlobs(randomPurpose()));
234234
assertEquals(maxRetries + 1, requestCounters.get("/storage/v1/b/bucket/o").get());
235235
}
@@ -238,7 +238,7 @@ public void testReadLargeBlobWithRetries() throws Exception {
238238
final int maxRetries = randomIntBetween(2, 10);
239239
final AtomicInteger countDown = new AtomicInteger(maxRetries);
240240

241-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null);
241+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
242242

243243
// SDK reads in 2 MB chunks so we use twice that to simulate 2 chunks
244244
final byte[] bytes = randomBytes(1 << 22);
@@ -267,7 +267,7 @@ public void testWriteBlobWithRetries() throws Exception {
267267
final int maxRetries = randomIntBetween(2, 10);
268268
final CountDown countDown = new CountDown(maxRetries);
269269

270-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null);
270+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
271271
final byte[] bytes = randomBlobContent();
272272
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
273273
assertThat(exchange.getRequestURI().getQuery(), containsString("uploadType=multipart"));
@@ -309,7 +309,7 @@ public void testWriteBlobWithRetries() throws Exception {
309309
public void testWriteBlobWithReadTimeouts() {
310310
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
311311
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
312-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, null, null, null, null, null);
312+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1).readTimeout(readTimeout).build();
313313

314314
// HTTP server does not send a response
315315
httpServer.createContext("/upload/storage/v1/b/bucket/o", exchange -> {
@@ -361,7 +361,7 @@ public void testWriteLargeBlob() throws IOException {
361361
logger.debug("starting with resumable upload id [{}]", sessionUploadId.get());
362362

363363
final TimeValue readTimeout = allowReadTimeout.get() ? TimeValue.timeValueSeconds(3) : null;
364-
final BlobContainer blobContainer = createBlobContainer(nbErrors + 1, readTimeout, null, null, null, null, null);
364+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(nbErrors + 1).readTimeout(readTimeout).build();
365365

366366
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
367367
final BytesReference requestBody = Streams.readFully(exchange.getRequestBody());
@@ -508,7 +508,7 @@ public String next() {
508508
return Integer.toString(totalDeletesSent++);
509509
}
510510
};
511-
final BlobContainer blobContainer = createBlobContainer(1, null, null, null, null, null, null);
511+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1).build();
512512
httpServer.createContext("/batch/storage/v1", safeHandler(exchange -> {
513513
assert pendingDeletes.get() <= MAX_DELETES_PER_BATCH;
514514

@@ -544,7 +544,7 @@ public void testCompareAndExchangeWhenThrottled() throws IOException {
544544
httpServer.createContext("/", new ResponseInjectingHttpHandler(requestHandlers, new GoogleCloudStorageHttpHandler("bucket")));
545545

546546
final int maxRetries = randomIntBetween(1, 3);
547-
final BlobContainer container = createBlobContainer(maxRetries, null, null, null, null, null, null);
547+
final BlobContainer container = blobContainerBuilder().maxRetries(maxRetries).build();
548548
final byte[] data = randomBytes(randomIntBetween(1, BlobContainerUtils.MAX_REGISTER_CONTENT_LENGTH));
549549
final String key = randomIdentifier();
550550

‎modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java‎

Lines changed: 57 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ public void testWriteBlobWithRetries() throws Exception {
270270
final int maxRetries = randomInt(5);
271271
final CountDown countDown = new CountDown(maxRetries + 1);
272272

273-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null, null);
273+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).disableChunkedEncoding(true).build();
274274

275275
final byte[] bytes = randomBlobContent();
276276
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_max_retries"), exchange -> {
@@ -318,7 +318,10 @@ public void testWriteBlobWithRetries() throws Exception {
318318
public void testWriteBlobWithReadTimeouts() {
319319
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
320320
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
321-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, null, null);
321+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1)
322+
.readTimeout(readTimeout)
323+
.disableChunkedEncoding(true)
324+
.build();
322325

323326
// HTTP server does not send a response
324327
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_timeout"), exchange -> {
@@ -352,7 +355,10 @@ public void testWriteLargeBlob() throws Exception {
352355
final boolean useTimeout = rarely();
353356
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
354357
final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB);
355-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, null, bufferSize, null, null);
358+
final BlobContainer blobContainer = blobContainerBuilder().readTimeout(readTimeout)
359+
.disableChunkedEncoding(true)
360+
.bufferSize(bufferSize)
361+
.build();
356362

357363
final int parts = randomIntBetween(1, 5);
358364
final long lastPartSize = randomLongBetween(10, 512);
@@ -448,7 +454,10 @@ public void testWriteLargeBlobStreaming() throws Exception {
448454
final boolean useTimeout = rarely();
449455
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
450456
final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB);
451-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, null, bufferSize, null, null);
457+
final BlobContainer blobContainer = blobContainerBuilder().readTimeout(readTimeout)
458+
.disableChunkedEncoding(true)
459+
.bufferSize(bufferSize)
460+
.build();
452461

453462
final int parts = randomIntBetween(1, 5);
454463
final long lastPartSize = randomLongBetween(10, 512);
@@ -557,15 +566,10 @@ public void testReadRetriesAfterMeaningfulProgress() throws Exception {
557566
0,
558567
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
559568
);
560-
final BlobContainer blobContainer = createBlobContainer(
561-
maxRetries,
562-
null,
563-
true,
564-
null,
565-
ByteSizeValue.ofBytes(bufferSizeBytes),
566-
null,
567-
null
568-
);
569+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries)
570+
.disableChunkedEncoding(true)
571+
.bufferSize(ByteSizeValue.ofBytes(bufferSizeBytes))
572+
.build();
569573
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
570574

571575
final byte[] bytes = randomBlobContent();
@@ -638,15 +642,10 @@ public void testReadDoesNotRetryForRepositoryAnalysis() {
638642
0,
639643
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
640644
);
641-
final BlobContainer blobContainer = createBlobContainer(
642-
maxRetries,
643-
null,
644-
true,
645-
null,
646-
ByteSizeValue.ofBytes(bufferSizeBytes),
647-
null,
648-
null
649-
);
645+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries)
646+
.disableChunkedEncoding(true)
647+
.bufferSize(ByteSizeValue.ofBytes(bufferSizeBytes))
648+
.build();
650649

651650
final byte[] bytes = randomBlobContent();
652651

@@ -684,15 +683,10 @@ public void testReadWithIndicesPurposeRetriesForever() throws IOException {
684683
0,
685684
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
686685
);
687-
final BlobContainer blobContainer = createBlobContainer(
688-
maxRetries,
689-
null,
690-
true,
691-
null,
692-
ByteSizeValue.ofBytes(bufferSizeBytes),
693-
null,
694-
null
695-
);
686+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries)
687+
.disableChunkedEncoding(true)
688+
.bufferSize(ByteSizeValue.ofBytes(bufferSizeBytes))
689+
.build();
696690
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
697691

698692
final byte[] bytes = randomBlobContent(512);
@@ -785,7 +779,7 @@ public void handle(HttpExchange exchange) throws IOException {
785779

786780
public void testDoesNotRetryOnNotFound() {
787781
final int maxRetries = between(3, 5);
788-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null, null);
782+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).disableChunkedEncoding(true).build();
789783

790784
final AtomicInteger numberOfReads = new AtomicInteger(0);
791785
@SuppressForbidden(reason = "use a http server")
@@ -816,8 +810,11 @@ public void handle(HttpExchange exchange) throws IOException {
816810
}
817811

818812
public void testSnapshotDeletesRetryOnThrottlingError() throws IOException {
819-
// disable AWS-client retries
820-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null, null);
813+
final BlobContainer blobContainer = blobContainerBuilder()
814+
// disable AWS-client retries
815+
.maxRetries(0)
816+
.disableChunkedEncoding(true)
817+
.build();
821818

822819
int numBlobsToDelete = randomIntBetween(500, 3000);
823820
List<String> blobsToDelete = new ArrayList<>();
@@ -836,8 +833,11 @@ public void testSnapshotDeletesRetryOnThrottlingError() throws IOException {
836833
}
837834

838835
public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() {
839-
// disable AWS-client retries
840-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null, null);
836+
final BlobContainer blobContainer = blobContainerBuilder()
837+
// disable AWS-client retries
838+
.maxRetries(0)
839+
.disableChunkedEncoding(true)
840+
.build();
841841

842842
int numBlobsToDelete = randomIntBetween(500, 3000);
843843
List<String> blobsToDelete = new ArrayList<>();
@@ -873,8 +873,11 @@ public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() {
873873
}
874874

875875
public void testNonSnapshotDeletesAreNotRetried() {
876-
// disable AWS-client retries
877-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null, null);
876+
final BlobContainer blobContainer = blobContainerBuilder()
877+
// disable AWS-client retries
878+
.maxRetries(0)
879+
.disableChunkedEncoding(true)
880+
.build();
878881

879882
int numBlobsToDelete = randomIntBetween(500, 3000);
880883
List<String> blobsToDelete = new ArrayList<>();
@@ -902,8 +905,11 @@ public void testNonSnapshotDeletesAreNotRetried() {
902905
}
903906

904907
public void testNonThrottlingErrorsAreNotRetried() {
905-
// disable AWS-client retries
906-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null, null);
908+
final BlobContainer blobContainer = blobContainerBuilder()
909+
// disable AWS-client retries
910+
.maxRetries(0)
911+
.disableChunkedEncoding(true)
912+
.build();
907913

908914
int numBlobsToDelete = randomIntBetween(500, 3000);
909915
List<String> blobsToDelete = new ArrayList<>();
@@ -982,7 +988,7 @@ private Set<OperationPurpose> operationPurposesThatRetryOnDelete() {
982988

983989
public void testGetRegisterRetries() {
984990
final var maxRetries = between(0, 3);
985-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null);
991+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
986992

987993
interface FailingHandlerFactory {
988994
void addHandler(String blobName, Integer... responseCodes);
@@ -1052,7 +1058,11 @@ interface FailingHandlerFactory {
10521058
public void testSuppressedDeletionErrorsAreCapped() {
10531059
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
10541060
int maxBulkDeleteSize = randomIntBetween(1, 10);
1055-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, maxBulkDeleteSize, null);
1061+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1)
1062+
.readTimeout(readTimeout)
1063+
.disableChunkedEncoding(true)
1064+
.maxBulkDeletes(maxBulkDeleteSize)
1065+
.build();
10561066
httpServer.createContext("/", exchange -> {
10571067
if (isMultiDeleteRequest(exchange)) {
10581068
exchange.sendResponseHeaders(
@@ -1084,7 +1094,11 @@ public void testSuppressedDeletionErrorsAreCapped() {
10841094
public void testTrimmedLogAndCappedSuppressedErrorOnMultiObjectDeletionException() {
10851095
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
10861096
int maxBulkDeleteSize = randomIntBetween(10, 30);
1087-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, maxBulkDeleteSize, null);
1097+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1)
1098+
.readTimeout(readTimeout)
1099+
.disableChunkedEncoding(true)
1100+
.maxBulkDeletes(maxBulkDeleteSize)
1101+
.build();
10881102

10891103
final Pattern pattern = Pattern.compile("<Key>(.+?)</Key>");
10901104
httpServer.createContext("/", exchange -> {

‎rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_query.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"esql.get_query": {
3+
"documentation": {
4+
"url": null,
5+
"description": "Executes a get ESQL query request"
6+
},
7+
"stability": "experimental",
8+
"visibility": "public",
9+
"headers": {
10+
"accept": [],
11+
"content_type": [
12+
"application/json"
13+
]
14+
},
15+
"url": {
16+
"paths": [
17+
{
18+
"path": "/_query/queries/{id}",
19+
"methods": [
20+
"GET"
21+
],
22+
"parts": {
23+
"id": {
24+
"type": "string",
25+
"description": "The query ID"
26+
}
27+
}
28+
}
29+
]
30+
}
31+
}
32+
}

0 commit comments

Comments
 (0)