Skip to content

Commit 84c8772

Browse files
nik9000nik9000
committed
Merge branch 'main' into took_telemetry
2 parents 05ff40d + 6d86b20 commit 84c8772

File tree

51 files changed

+1747
-719
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+1747
-719
lines changed

‎docs/changelog/126529.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 126529
2+
summary: "Batch ILM policy cluster state updates [#122917]"
3+
area: ILM+SLM
4+
type: enhancement
5+
issues:
6+
- 122917

‎docs/changelog/126537.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126537
2+
summary: Fix ELAND endpoints not updating dimensions
3+
area: Machine Learning
4+
type: bug
5+
issues: []

‎docs/reference/elasticsearch/configuration-reference/enrich-settings.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ applies_to:
77

88
# Enrich settings [enrich_settings]
99

10-
You can configure these enrich settings in the `elasticsearch.yml` file. For more information, see [Set up an enrich processor](docs-content:///manage-data/ingest/transform-enrich/set-up-an-enrich-processor.md).
10+
You can configure these enrich settings in the `elasticsearch.yml` file. For more information, see [Set up an enrich processor](docs-content://manage-data/ingest/transform-enrich/set-up-an-enrich-processor.md).
1111

1212
`enrich.cache_size` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
1313
: Maximum number of searches to cache for enriching documents. Defaults to 1000. There is a single cache for all enrich processors in the cluster. This setting determines the size of that cache.

‎libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java‎

Lines changed: 41 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,8 @@
2929
import java.util.Map;
3030
import java.util.Map.Entry;
3131
import java.util.Set;
32-
import java.util.function.Function;
32+
import java.util.function.Predicate;
3333
import java.util.stream.Collectors;
34-
import java.util.stream.Stream;
3534

3635
import static java.util.Map.entry;
3736
import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_ALLOWED;
@@ -49,27 +48,34 @@ record CheckAction(
4948
Integer fromJavaVersion
5049
) {}
5150

52-
private static final Map<String, CheckAction> checkActions = Stream.of(
53-
getTestEntries(FileCheckActions.class),
54-
getTestEntries(FileStoreActions.class),
55-
getTestEntries(JvmActions.class),
56-
getTestEntries(LoadNativeLibrariesCheckActions.class),
57-
getTestEntries(ManageThreadsActions.class),
58-
getTestEntries(NativeActions.class),
59-
getTestEntries(NetworkAccessCheckActions.class),
60-
getTestEntries(NioChannelsActions.class),
61-
getTestEntries(NioFilesActions.class),
62-
getTestEntries(NioFileSystemActions.class),
63-
getTestEntries(OperatingSystemActions.class),
64-
getTestEntries(PathActions.class),
65-
getTestEntries(SpiActions.class),
66-
getTestEntries(SystemActions.class),
67-
getTestEntries(URLConnectionFileActions.class),
68-
getTestEntries(URLConnectionNetworkActions.class)
69-
)
70-
.flatMap(Function.identity())
71-
.filter(entry -> entry.getValue().fromJavaVersion() == null || Runtime.version().feature() >= entry.getValue().fromJavaVersion())
72-
.collect(Collectors.toUnmodifiableMap(Entry::getKey, Entry::getValue));
51+
private static final Map<String, CheckAction> checkActions = collectTests(
52+
FileCheckActions.class,
53+
FileStoreActions.class,
54+
JvmActions.class,
55+
LoadNativeLibrariesCheckActions.class,
56+
ManageThreadsActions.class,
57+
NativeActions.class,
58+
NetworkAccessCheckActions.class,
59+
NioChannelsActions.class,
60+
NioFilesActions.class,
61+
NioFileSystemActions.class,
62+
OperatingSystemActions.class,
63+
PathActions.class,
64+
SpiActions.class,
65+
SystemActions.class,
66+
URLConnectionFileActions.class,
67+
URLConnectionNetworkActions.class
68+
);
69+
70+
private static Map<String, CheckAction> collectTests(Class<?>... testClasses) {
71+
List<Entry<String, CheckAction>> entries = new ArrayList<>();
72+
for (Class<?> testClass : testClasses) {
73+
getTestEntries(entries, testClass, a -> a.fromJavaVersion() == null || Runtime.version().feature() >= a.fromJavaVersion());
74+
}
75+
@SuppressWarnings({ "unchecked", "rawtypes" })
76+
Entry<String, CheckAction>[] entriesArray = entries.toArray(new Entry[0]);
77+
return Map.ofEntries(entriesArray);
78+
}
7379

7480
private final Environment environment;
7581

@@ -82,8 +88,7 @@ private static Method[] getDeclaredMethods(Class<?> clazz) {
8288
return clazz.getDeclaredMethods();
8389
}
8490

85-
private static Stream<Entry<String, CheckAction>> getTestEntries(Class<?> actionsClass) {
86-
List<Entry<String, CheckAction>> entries = new ArrayList<>();
91+
private static void getTestEntries(List<Entry<String, CheckAction>> entries, Class<?> actionsClass, Predicate<CheckAction> filter) {
8792
for (var method : getDeclaredMethods(actionsClass)) {
8893
var testAnnotation = method.getAnnotation(EntitlementTest.class);
8994
if (testAnnotation == null) {
@@ -92,6 +97,9 @@ private static Stream<Entry<String, CheckAction>> getTestEntries(Class<?> action
9297
if (Modifier.isStatic(method.getModifiers()) == false) {
9398
throw new AssertionError("Entitlement test method [" + method + "] must be static");
9499
}
100+
if (Modifier.isPrivate(method.getModifiers())) {
101+
throw new AssertionError("Entitlement test method [" + method + "] must not be private");
102+
}
95103
final CheckedConsumer<Environment, Exception> call = createConsumerForMethod(method);
96104
CheckedConsumer<Environment, Exception> runnable = env -> {
97105
try {
@@ -107,14 +115,16 @@ private static Stream<Entry<String, CheckAction>> getTestEntries(Class<?> action
107115
}
108116
};
109117
Integer fromJavaVersion = testAnnotation.fromJavaVersion() == -1 ? null : testAnnotation.fromJavaVersion();
110-
entries.add(
111-
entry(
112-
method.getName(),
113-
new CheckAction(runnable, testAnnotation.expectedAccess(), testAnnotation.expectedExceptionIfDenied(), fromJavaVersion)
114-
)
118+
var checkAction = new CheckAction(
119+
runnable,
120+
testAnnotation.expectedAccess(),
121+
testAnnotation.expectedExceptionIfDenied(),
122+
fromJavaVersion
115123
);
124+
if (filter.test(checkAction)) {
125+
entries.add(entry(method.getName(), checkAction));
126+
}
116127
}
117-
return entries.stream();
118128
}
119129

120130
private static CheckedConsumer<Environment, Exception> createConsumerForMethod(Method method) {

‎modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import java.nio.charset.StandardCharsets;
4949
import java.nio.file.NoSuchFileException;
5050
import java.util.ArrayList;
51+
import java.util.Arrays;
5152
import java.util.Base64;
5253
import java.util.Collection;
5354
import java.util.Collections;
@@ -241,7 +242,7 @@ protected String requestUniqueId(final HttpExchange exchange) {
241242
private static class AzureHTTPStatsCollectorHandler extends HttpStatsCollectorHandler {
242243

243244
private AzureHTTPStatsCollectorHandler(HttpHandler delegate) {
244-
super(delegate);
245+
super(delegate, Arrays.stream(AzureBlobStore.Operation.values()).map(AzureBlobStore.Operation::getKey).toArray(String[]::new));
245246
}
246247

247248
@Override

‎modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ protected boolean canFailRequest(final HttpExchange exchange) {
364364
private static class GoogleCloudStorageStatsCollectorHttpHandler extends HttpStatsCollectorHandler {
365365

366366
GoogleCloudStorageStatsCollectorHttpHandler(final HttpHandler delegate) {
367-
super(delegate);
367+
super(delegate, Arrays.stream(StorageOperation.values()).map(StorageOperation::key).toArray(String[]::new));
368368
}
369369

370370
@Override

‎modules/repository-s3/qa/third-party/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,4 +228,56 @@ public void testReadFromPositionLargerThanBlobLength() {
228228
e -> asInstanceOf(AmazonS3Exception.class, e.getCause()).getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()
229229
);
230230
}
231+
232+
public void testCopy() {
233+
final var sourceBlobName = randomIdentifier();
234+
final var blobBytes = randomBytesReference(randomIntBetween(100, 2_000));
235+
final var destinationBlobName = randomIdentifier();
236+
237+
final var repository = getRepository();
238+
239+
final var targetBytes = executeOnBlobStore(repository, sourceBlobContainer -> {
240+
sourceBlobContainer.writeBlob(randomPurpose(), sourceBlobName, blobBytes, true);
241+
242+
final var destinationBlobContainer = repository.blobStore().blobContainer(repository.basePath().add("target"));
243+
destinationBlobContainer.copyBlob(
244+
randomPurpose(),
245+
sourceBlobContainer,
246+
sourceBlobName,
247+
destinationBlobName,
248+
blobBytes.length()
249+
);
250+
251+
return destinationBlobContainer.readBlob(randomPurpose(), destinationBlobName).readAllBytes();
252+
});
253+
254+
assertArrayEquals(BytesReference.toBytes(blobBytes), targetBytes);
255+
}
256+
257+
public void testMultipartCopy() {
258+
final var sourceBlobName = randomIdentifier();
259+
// executeMultipart requires a minimum part size of 5 MiB
260+
final var blobBytes = randomBytesReference(randomIntBetween(5 * 1024 * 1024, 10 * 1024 * 1024));
261+
final var destinationBlobName = randomIdentifier();
262+
263+
final var repository = getRepository();
264+
265+
final var targetBytes = executeOnBlobStore(repository, sourceBlobContainer -> {
266+
sourceBlobContainer.writeBlob(randomPurpose(), sourceBlobName, blobBytes, true);
267+
268+
final S3BlobContainer destinationBlobContainer = (S3BlobContainer) repository.blobStore()
269+
.blobContainer(repository.basePath().add("target"));
270+
destinationBlobContainer.executeMultipartCopy(
271+
randomPurpose(),
272+
(S3BlobContainer) sourceBlobContainer,
273+
sourceBlobName,
274+
destinationBlobName,
275+
blobBytes.length()
276+
);
277+
278+
return destinationBlobContainer.readBlob(randomPurpose(), destinationBlobName).readAllBytes();
279+
});
280+
281+
assertArrayEquals(BytesReference.toBytes(blobBytes), targetBytes);
282+
}
231283
}

‎modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import java.io.IOException;
7272
import java.io.InputStream;
7373
import java.util.ArrayList;
74+
import java.util.Arrays;
7475
import java.util.Collection;
7576
import java.util.Collections;
7677
import java.util.EnumSet;
@@ -621,6 +622,12 @@ long getLargeBlobThresholdInBytes() {
621622
return ByteSizeUnit.MB.toBytes(1L);
622623
}
623624

625+
@Override
626+
long getMaxCopySizeBeforeMultipart() {
627+
// on my laptop 10K exercises this better but larger values should be fine for nightlies
628+
return ByteSizeUnit.MB.toBytes(1L);
629+
}
630+
624631
@Override
625632
void ensureMultiPartUploadSize(long blobSize) {}
626633
};
@@ -688,7 +695,7 @@ protected class S3StatsCollectorHttpHandler extends HttpStatsCollectorHandler {
688695
private final Map<S3BlobStore.StatsKey, AtomicLong> metricsCount = ConcurrentCollections.newConcurrentMap();
689696

690697
S3StatsCollectorHttpHandler(final HttpHandler delegate) {
691-
super(delegate);
698+
super(delegate, Arrays.stream(S3BlobStore.Operation.values()).map(S3BlobStore.Operation::getKey).toArray(String[]::new));
692699
}
693700

694701
private S3HttpHandler.S3Request parseRequest(HttpExchange exchange) {
@@ -736,9 +743,17 @@ public void maybeTrack(HttpExchange exchange) {
736743
k -> new AtomicLong()
737744
).incrementAndGet();
738745
} else if (request.isPutObjectRequest()) {
739-
trackRequest("PutObject");
740-
metricsCount.computeIfAbsent(new S3BlobStore.StatsKey(S3BlobStore.Operation.PUT_OBJECT, purpose), k -> new AtomicLong())
741-
.incrementAndGet();
746+
if (exchange.getRequestHeaders().containsKey(S3BlobStore.CUSTOM_QUERY_PARAMETER_COPY_SOURCE)) {
747+
trackRequest("CopyObject");
748+
metricsCount.computeIfAbsent(
749+
new S3BlobStore.StatsKey(S3BlobStore.Operation.COPY_OBJECT, purpose),
750+
k -> new AtomicLong()
751+
).incrementAndGet();
752+
} else {
753+
trackRequest("PutObject");
754+
metricsCount.computeIfAbsent(new S3BlobStore.StatsKey(S3BlobStore.Operation.PUT_OBJECT, purpose), k -> new AtomicLong())
755+
.incrementAndGet();
756+
}
742757
} else if (request.isMultiObjectDeleteRequest()) {
743758
trackRequest("DeleteObjects");
744759
metricsCount.computeIfAbsent(new S3BlobStore.StatsKey(S3BlobStore.Operation.DELETE_OBJECTS, purpose), k -> new AtomicLong())

0 commit comments

Comments
 (0)