Skip to content

Commit 12908fa

Browse files
n1v0lgn1v0lg
authored
Merge branch 'main' into uiam-cloud-api-key-authentication
2 parents 3be47f0 + e352d2c commit 12908fa

File tree

72 files changed

+1442
-219
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+1442
-219
lines changed

‎docs/changelog/126581.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
pr: 126581
2+
summary: "Optimize shared blob cache evictions on shard removal
3+
Shared blob cache evictions occur on the cluster applier thread when shards are
4+
removed from a node. These can be expensive if a large number of shards are
5+
being removed. This change uses the context of the removal to avoid unnecessary
6+
evictions that might hold up the applier thread.
7+
"
8+
area: Snapshot/Restore
9+
type: enhancement
10+
issues: []

‎modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,6 @@
2121

2222
import java.util.Locale;
2323

24-
import static org.elasticsearch.repositories.s3.S3Service.REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING;
25-
import static org.elasticsearch.repositories.s3.S3Service.REPOSITORY_S3_CAS_TTL_SETTING;
26-
2724
@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
2825
@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
2926
public class RepositoryS3MinioBasicCredentialsRestIT extends AbstractRepositoryS3RestTestCase {
@@ -42,10 +39,6 @@ public class RepositoryS3MinioBasicCredentialsRestIT extends AbstractRepositoryS
4239
.keystore("s3.client." + CLIENT + ".access_key", ACCESS_KEY)
4340
.keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY)
4441
.setting("s3.client." + CLIENT + ".endpoint", minioFixture::getAddress)
45-
// Skip listing of pre-existing uploads during a CAS because MinIO sometimes leaks them; also reduce the delay before proceeding
46-
// TODO do not set these if running a MinIO version in which https://github.com/minio/minio/issues/21189 is fixed
47-
.setting(REPOSITORY_S3_CAS_TTL_SETTING.getKey(), "-1")
48-
.setting(REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING.getKey(), "100ms")
4942
.build();
5043

5144
@ClassRule

‎muted-tests.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -426,9 +426,6 @@ tests:
426426
- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithPartialResultsIT
427427
method: testOneRemoteClusterPartial
428428
issue: https://github.com/elastic/elasticsearch/issues/124055
429-
- class: org.elasticsearch.compute.aggregation.SampleDoubleAggregatorFunctionTests
430-
method: testSimpleWithCranky
431-
issue: https://github.com/elastic/elasticsearch/issues/128024
432429
- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
433430
method: test {lookup-join.MvJoinKeyOnTheLookupIndex ASYNC}
434431
issue: https://github.com/elastic/elasticsearch/issues/128030

‎server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.index.IndexSettings;
2323
import org.elasticsearch.index.shard.ShardId;
2424
import org.elasticsearch.indices.IndicesService;
25+
import org.elasticsearch.indices.cluster.IndexRemovalReason;
2526
import org.elasticsearch.test.ESIntegTestCase;
2627
import org.elasticsearch.test.junit.annotations.TestLogging;
2728

@@ -345,12 +346,22 @@ public static class IndexFoldersDeletionListenerPlugin extends Plugin implements
345346
public List<IndexFoldersDeletionListener> getIndexFoldersDeletionListeners() {
346347
return List.of(new IndexFoldersDeletionListener() {
347348
@Override
348-
public void beforeIndexFoldersDeleted(Index index, IndexSettings indexSettings, Path[] indexPaths) {
349+
public void beforeIndexFoldersDeleted(
350+
Index index,
351+
IndexSettings indexSettings,
352+
Path[] indexPaths,
353+
IndexRemovalReason reason
354+
) {
349355
deletedIndices.add(index);
350356
}
351357

352358
@Override
353-
public void beforeShardFoldersDeleted(ShardId shardId, IndexSettings indexSettings, Path[] shardPaths) {
359+
public void beforeShardFoldersDeleted(
360+
ShardId shardId,
361+
IndexSettings indexSettings,
362+
Path[] shardPaths,
363+
IndexRemovalReason reason
364+
) {
354365
deletedShards.computeIfAbsent(shardId.getIndex(), i -> Collections.synchronizedList(new ArrayList<>())).add(shardId);
355366
}
356367
});

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848

4949
import static java.util.Collections.emptyList;
5050
import static java.util.Collections.emptyMap;
51-
import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
51+
import static org.elasticsearch.indices.cluster.IndexRemovalReason.NO_LONGER_ASSIGNED;
5252

5353
/**
5454
* Service responsible for submitting add and remove aliases requests

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@
8383

8484
import static java.util.Collections.emptyMap;
8585
import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping;
86-
import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
86+
import static org.elasticsearch.indices.cluster.IndexRemovalReason.NO_LONGER_ASSIGNED;
8787

8888
/**
8989
* Service responsible for submitting index templates updates

‎server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import org.elasticsearch.index.shard.IndexShard;
2222
import org.elasticsearch.index.shard.IndexShardState;
2323
import org.elasticsearch.index.shard.ShardId;
24-
import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
24+
import org.elasticsearch.indices.cluster.IndexRemovalReason;
2525
import org.elasticsearch.threadpool.ThreadPool;
2626

2727
import java.util.Collection;

‎server/src/main/java/org/elasticsearch/index/IndexService.java‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
import org.elasticsearch.index.store.Store;
8484
import org.elasticsearch.index.translog.Translog;
8585
import org.elasticsearch.indices.breaker.CircuitBreakerService;
86+
import org.elasticsearch.indices.cluster.IndexRemovalReason;
8687
import org.elasticsearch.indices.cluster.IndicesClusterStateService;
8788
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
8889
import org.elasticsearch.indices.recovery.RecoveryState;
@@ -494,7 +495,12 @@ public synchronized IndexShard createShard(
494495
nodeEnv,
495496
lock,
496497
this.indexSettings,
497-
shardPaths -> indexFoldersDeletionListener.beforeShardFoldersDeleted(shardId, this.indexSettings, shardPaths)
498+
shardPaths -> indexFoldersDeletionListener.beforeShardFoldersDeleted(
499+
shardId,
500+
this.indexSettings,
501+
shardPaths,
502+
IndexRemovalReason.FAILURE
503+
)
498504
);
499505
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings.customDataPath());
500506
} catch (Exception inner) {
@@ -704,11 +710,11 @@ private void onShardClose(ShardLock lock) {
704710
try {
705711
eventListener.beforeIndexShardDeleted(lock.getShardId(), indexSettings.getSettings());
706712
} finally {
707-
shardStoreDeleter.deleteShardStore("delete index", lock, indexSettings);
713+
shardStoreDeleter.deleteShardStore("delete index", lock, indexSettings, IndexRemovalReason.DELETED);
708714
eventListener.afterIndexShardDeleted(lock.getShardId(), indexSettings.getSettings());
709715
}
710716
} catch (IOException e) {
711-
shardStoreDeleter.addPendingDelete(lock.getShardId(), indexSettings);
717+
shardStoreDeleter.addPendingDelete(lock.getShardId(), indexSettings, IndexRemovalReason.DELETED);
712718
logger.debug(() -> "[" + lock.getShardId().id() + "] failed to delete shard content - scheduled a retry", e);
713719
}
714720
}
@@ -1062,9 +1068,9 @@ public static Function<String, String> dateMathExpressionResolverAt(long instant
10621068
}
10631069

10641070
public interface ShardStoreDeleter {
1065-
void deleteShardStore(String reason, ShardLock lock, IndexSettings indexSettings) throws IOException;
1071+
void deleteShardStore(String reasonText, ShardLock lock, IndexSettings indexSettings, IndexRemovalReason reason) throws IOException;
10661072

1067-
void addPendingDelete(ShardId shardId, IndexSettings indexSettings);
1073+
void addPendingDelete(ShardId shardId, IndexSettings indexSettings, IndexRemovalReason reason);
10681074
}
10691075

10701076
public final EngineFactory getEngineFactory() {

‎server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,16 @@
1111

1212
import org.apache.lucene.index.BinaryDocValues;
1313
import org.apache.lucene.index.DocValues;
14+
import org.apache.lucene.index.FloatVectorValues;
15+
import org.apache.lucene.index.KnnVectorValues;
1416
import org.apache.lucene.index.LeafReaderContext;
1517
import org.apache.lucene.index.NumericDocValues;
1618
import org.apache.lucene.index.SortedDocValues;
1719
import org.apache.lucene.index.SortedNumericDocValues;
1820
import org.apache.lucene.index.SortedSetDocValues;
1921
import org.apache.lucene.util.BytesRef;
2022
import org.elasticsearch.common.io.stream.ByteArrayStreamInput;
23+
import org.elasticsearch.index.IndexVersion;
2124
import org.elasticsearch.index.mapper.BlockLoader.BlockFactory;
2225
import org.elasticsearch.index.mapper.BlockLoader.BooleanBuilder;
2326
import org.elasticsearch.index.mapper.BlockLoader.Builder;
@@ -26,6 +29,7 @@
2629
import org.elasticsearch.index.mapper.BlockLoader.DoubleBuilder;
2730
import org.elasticsearch.index.mapper.BlockLoader.IntBuilder;
2831
import org.elasticsearch.index.mapper.BlockLoader.LongBuilder;
32+
import org.elasticsearch.index.mapper.vectors.VectorEncoderDecoder;
2933
import org.elasticsearch.search.fetch.StoredFieldsSpec;
3034

3135
import java.io.IOException;
@@ -504,6 +508,87 @@ public String toString() {
504508
}
505509
}
506510

511+
public static class DenseVectorBlockLoader extends DocValuesBlockLoader {
512+
private final String fieldName;
513+
private final int dimensions;
514+
515+
public DenseVectorBlockLoader(String fieldName, int dimensions) {
516+
this.fieldName = fieldName;
517+
this.dimensions = dimensions;
518+
}
519+
520+
@Override
521+
public Builder builder(BlockFactory factory, int expectedCount) {
522+
return factory.denseVectors(expectedCount, dimensions);
523+
}
524+
525+
@Override
526+
public AllReader reader(LeafReaderContext context) throws IOException {
527+
FloatVectorValues floatVectorValues = context.reader().getFloatVectorValues(fieldName);
528+
if (floatVectorValues != null) {
529+
return new DenseVectorValuesBlockReader(floatVectorValues, dimensions);
530+
}
531+
return new ConstantNullsReader();
532+
}
533+
}
534+
535+
private static class DenseVectorValuesBlockReader extends BlockDocValuesReader {
536+
private final FloatVectorValues floatVectorValues;
537+
private final KnnVectorValues.DocIndexIterator iterator;
538+
private final int dimensions;
539+
540+
DenseVectorValuesBlockReader(FloatVectorValues floatVectorValues, int dimensions) {
541+
this.floatVectorValues = floatVectorValues;
542+
iterator = floatVectorValues.iterator();
543+
this.dimensions = dimensions;
544+
}
545+
546+
@Override
547+
public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException {
548+
// Doubles from doc values ensures that the values are in order
549+
try (BlockLoader.FloatBuilder builder = factory.denseVectors(docs.count(), dimensions)) {
550+
for (int i = 0; i < docs.count(); i++) {
551+
int doc = docs.get(i);
552+
if (doc < iterator.docID()) {
553+
throw new IllegalStateException("docs within same block must be in order");
554+
}
555+
read(doc, builder);
556+
}
557+
return builder.build();
558+
}
559+
}
560+
561+
@Override
562+
public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException {
563+
read(docId, (BlockLoader.FloatBuilder) builder);
564+
}
565+
566+
private void read(int doc, BlockLoader.FloatBuilder builder) throws IOException {
567+
if (iterator.advance(doc) == doc) {
568+
builder.beginPositionEntry();
569+
float[] floats = floatVectorValues.vectorValue(iterator.index());
570+
assert floats.length == dimensions
571+
: "unexpected dimensions for vector value; expected " + dimensions + " but got " + floats.length;
572+
for (float aFloat : floats) {
573+
builder.appendFloat(aFloat);
574+
}
575+
builder.endPositionEntry();
576+
} else {
577+
builder.appendNull();
578+
}
579+
}
580+
581+
@Override
582+
public int docId() {
583+
return iterator.docID();
584+
}
585+
586+
@Override
587+
public String toString() {
588+
return "BlockDocValuesReader.FloatVectorValuesBlockReader";
589+
}
590+
}
591+
507592
public static class BytesRefsFromOrdsBlockLoader extends DocValuesBlockLoader {
508593
private final String fieldName;
509594

@@ -752,6 +837,94 @@ public String toString() {
752837
}
753838
}
754839

840+
public static class DenseVectorFromBinaryBlockLoader extends DocValuesBlockLoader {
841+
private final String fieldName;
842+
private final int dims;
843+
private final IndexVersion indexVersion;
844+
845+
public DenseVectorFromBinaryBlockLoader(String fieldName, int dims, IndexVersion indexVersion) {
846+
this.fieldName = fieldName;
847+
this.dims = dims;
848+
this.indexVersion = indexVersion;
849+
}
850+
851+
@Override
852+
public Builder builder(BlockFactory factory, int expectedCount) {
853+
return factory.denseVectors(expectedCount, dims);
854+
}
855+
856+
@Override
857+
public AllReader reader(LeafReaderContext context) throws IOException {
858+
BinaryDocValues docValues = context.reader().getBinaryDocValues(fieldName);
859+
if (docValues == null) {
860+
return new ConstantNullsReader();
861+
}
862+
return new DenseVectorFromBinary(docValues, dims, indexVersion);
863+
}
864+
}
865+
866+
private static class DenseVectorFromBinary extends BlockDocValuesReader {
867+
private final BinaryDocValues docValues;
868+
private final IndexVersion indexVersion;
869+
private final int dimensions;
870+
private final float[] scratch;
871+
872+
private int docID = -1;
873+
874+
DenseVectorFromBinary(BinaryDocValues docValues, int dims, IndexVersion indexVersion) {
875+
this.docValues = docValues;
876+
this.scratch = new float[dims];
877+
this.indexVersion = indexVersion;
878+
this.dimensions = dims;
879+
}
880+
881+
@Override
882+
public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException {
883+
try (BlockLoader.FloatBuilder builder = factory.denseVectors(docs.count(), dimensions)) {
884+
for (int i = 0; i < docs.count(); i++) {
885+
int doc = docs.get(i);
886+
if (doc < docID) {
887+
throw new IllegalStateException("docs within same block must be in order");
888+
}
889+
read(doc, builder);
890+
}
891+
return builder.build();
892+
}
893+
}
894+
895+
@Override
896+
public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException {
897+
read(docId, (BlockLoader.FloatBuilder) builder);
898+
}
899+
900+
private void read(int doc, BlockLoader.FloatBuilder builder) throws IOException {
901+
this.docID = doc;
902+
if (false == docValues.advanceExact(doc)) {
903+
builder.appendNull();
904+
return;
905+
}
906+
BytesRef bytesRef = docValues.binaryValue();
907+
assert bytesRef.length > 0;
908+
VectorEncoderDecoder.decodeDenseVector(indexVersion, bytesRef, scratch);
909+
910+
builder.beginPositionEntry();
911+
for (float value : scratch) {
912+
builder.appendFloat(value);
913+
}
914+
builder.endPositionEntry();
915+
}
916+
917+
@Override
918+
public int docId() {
919+
return docID;
920+
}
921+
922+
@Override
923+
public String toString() {
924+
return "DenseVectorFromBinary.Bytes";
925+
}
926+
}
927+
755928
public static class BooleansBlockLoader extends DocValuesBlockLoader {
756929
private final String fieldName;
757930

‎server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,11 @@ interface BlockFactory {
373373
*/
374374
DoubleBuilder doubles(int expectedCount);
375375

376+
/**
377+
* Build a builder to load dense vectors without any loading constraints.
378+
*/
379+
FloatBuilder denseVectors(int expectedVectorsCount, int dimensions);
380+
376381
/**
377382
* Build a builder to load ints as loaded from doc values.
378383
* Doc values load ints in sorted order.

0 commit comments

Comments
 (0)