Skip to content

Commit 1d38a6a

Browse files
cbueschercbuescher
committed
Merge branch 'main' into lucene_snapshot
2 parents 40d0b72 + e8bf344 commit 1d38a6a

File tree

59 files changed

+1460
-369
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+1460
-369
lines changed

‎muted-tests.yml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,6 @@ tests:
9898
- class: org.elasticsearch.xpack.sql.qa.single_node.JdbcSqlSpecIT
9999
method: test {case-functions.testUcaseInline3}
100100
issue: https://github.com/elastic/elasticsearch/issues/112643
101-
- class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT
102-
issue: https://github.com/elastic/elasticsearch/issues/112889
103101
- class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT
104102
method: test {case-functions.testUcaseInline1}
105103
issue: https://github.com/elastic/elasticsearch/issues/112641
@@ -274,8 +272,6 @@ tests:
274272
- class: org.elasticsearch.action.search.PointInTimeIT
275273
method: testPITTiebreak
276274
issue: https://github.com/elastic/elasticsearch/issues/115810
277-
- class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT
278-
issue: https://github.com/elastic/elasticsearch/issues/115995
279275
- class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT
280276
method: testReindexWithShutdown
281277
issue: https://github.com/elastic/elasticsearch/issues/115996
@@ -288,6 +284,15 @@ tests:
288284
- class: org.elasticsearch.action.admin.HotThreadsIT
289285
method: testHotThreadsDontFail
290286
issue: https://github.com/elastic/elasticsearch/issues/115754
287+
- class: org.elasticsearch.search.functionscore.QueryRescorerIT
288+
method: testScoring
289+
issue: https://github.com/elastic/elasticsearch/issues/116050
290+
- class: org.elasticsearch.indexing.IndexActionIT
291+
method: testAutoGenerateIdNoDuplicates
292+
issue: https://github.com/elastic/elasticsearch/issues/115716
293+
- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
294+
method: test {yaml=logsdb/10_settings/logsdb with default ignore dynamic beyond limit and subobjects false}
295+
issue: https://github.com/elastic/elasticsearch/issues/116054
291296

292297
# Examples:
293298
#

‎server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java‎

Lines changed: 213 additions & 193 deletions
Large diffs are not rendered by default.

‎server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java‎

Lines changed: 14 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1796,7 +1796,6 @@ public static class Builder {
17961796
private DiffableStringMap hashesOfConsistentSettings = DiffableStringMap.EMPTY;
17971797

17981798
private final ImmutableOpenMap.Builder<String, IndexMetadata> indices;
1799-
private final ImmutableOpenMap.Builder<String, Set<Index>> aliasedIndices;
18001799
private final ImmutableOpenMap.Builder<String, IndexTemplateMetadata> templates;
18011800
private final ImmutableOpenMap.Builder<String, Custom> customs;
18021801

@@ -1826,7 +1825,6 @@ public Builder() {
18261825
this.hashesOfConsistentSettings = metadata.hashesOfConsistentSettings;
18271826
this.version = metadata.version;
18281827
this.indices = ImmutableOpenMap.builder(metadata.indices);
1829-
this.aliasedIndices = ImmutableOpenMap.builder(metadata.aliasedIndices);
18301828
this.templates = ImmutableOpenMap.builder(metadata.templates);
18311829
this.customs = ImmutableOpenMap.builder(metadata.customs);
18321830
this.previousIndicesLookup = metadata.indicesLookup;
@@ -1839,7 +1837,6 @@ public Builder() {
18391837
private Builder(Map<String, MappingMetadata> mappingsByHash, int indexCountHint) {
18401838
clusterUUID = UNKNOWN_CLUSTER_UUID;
18411839
indices = ImmutableOpenMap.builder(indexCountHint);
1842-
aliasedIndices = ImmutableOpenMap.builder();
18431840
templates = ImmutableOpenMap.builder();
18441841
customs = ImmutableOpenMap.builder();
18451842
reservedStateMetadata = new HashMap<>();
@@ -1854,7 +1851,6 @@ public Builder put(IndexMetadata.Builder indexMetadataBuilder) {
18541851
dedupeMapping(indexMetadataBuilder);
18551852
IndexMetadata indexMetadata = indexMetadataBuilder.build();
18561853
IndexMetadata previous = indices.put(indexMetadata.getIndex().getName(), indexMetadata);
1857-
updateAliases(previous, indexMetadata);
18581854
if (unsetPreviousIndicesLookup(previous, indexMetadata)) {
18591855
previousIndicesLookup = null;
18601856
}
@@ -1879,7 +1875,6 @@ public Builder put(IndexMetadata indexMetadata, boolean incrementVersion) {
18791875
return this;
18801876
}
18811877
}
1882-
updateAliases(previous, indexMetadata);
18831878
if (unsetPreviousIndicesLookup(previous, indexMetadata)) {
18841879
previousIndicesLookup = null;
18851880
}
@@ -1954,8 +1949,7 @@ public IndexMetadata getSafe(Index index) {
19541949
public Builder remove(String index) {
19551950
previousIndicesLookup = null;
19561951
checkForUnusedMappings = true;
1957-
IndexMetadata previous = indices.remove(index);
1958-
updateAliases(previous, null);
1952+
indices.remove(index);
19591953
return this;
19601954
}
19611955

@@ -1965,7 +1959,6 @@ public Builder removeAllIndices() {
19651959

19661960
indices.clear();
19671961
mappingsByHash.clear();
1968-
aliasedIndices.clear();
19691962
return this;
19701963
}
19711964

@@ -1976,67 +1969,6 @@ public Builder indices(Map<String, IndexMetadata> indices) {
19761969
return this;
19771970
}
19781971

1979-
void updateAliases(IndexMetadata previous, IndexMetadata current) {
1980-
if (previous == null && current != null) {
1981-
for (var key : current.getAliases().keySet()) {
1982-
putAlias(key, current.getIndex());
1983-
}
1984-
} else if (previous != null && current == null) {
1985-
for (var key : previous.getAliases().keySet()) {
1986-
removeAlias(key, previous.getIndex());
1987-
}
1988-
} else if (previous != null && current != null) {
1989-
if (Objects.equals(previous.getAliases(), current.getAliases())) {
1990-
return;
1991-
}
1992-
1993-
for (var key : current.getAliases().keySet()) {
1994-
if (previous.getAliases().containsKey(key) == false) {
1995-
putAlias(key, current.getIndex());
1996-
}
1997-
}
1998-
for (var key : previous.getAliases().keySet()) {
1999-
if (current.getAliases().containsKey(key) == false) {
2000-
removeAlias(key, current.getIndex());
2001-
}
2002-
}
2003-
}
2004-
}
2005-
2006-
private Builder putAlias(String alias, Index index) {
2007-
Objects.requireNonNull(alias);
2008-
Objects.requireNonNull(index);
2009-
2010-
Set<Index> indices = new HashSet<>(aliasedIndices.getOrDefault(alias, Set.of()));
2011-
if (indices.add(index) == false) {
2012-
return this; // indices already contained this index
2013-
}
2014-
aliasedIndices.put(alias, Collections.unmodifiableSet(indices));
2015-
return this;
2016-
}
2017-
2018-
private Builder removeAlias(String alias, Index index) {
2019-
Objects.requireNonNull(alias);
2020-
Objects.requireNonNull(index);
2021-
2022-
Set<Index> indices = aliasedIndices.get(alias);
2023-
if (indices == null || indices.isEmpty()) {
2024-
throw new IllegalStateException("Cannot remove non-existent alias [" + alias + "] for index [" + index.getName() + "]");
2025-
}
2026-
2027-
indices = new HashSet<>(indices);
2028-
if (indices.remove(index) == false) {
2029-
throw new IllegalStateException("Cannot remove non-existent alias [" + alias + "] for index [" + index.getName() + "]");
2030-
}
2031-
2032-
if (indices.isEmpty()) {
2033-
aliasedIndices.remove(alias); // for consistency, we don't store empty sets, so null it out
2034-
} else {
2035-
aliasedIndices.put(alias, Collections.unmodifiableSet(indices));
2036-
}
2037-
return this;
2038-
}
2039-
20401972
public Builder put(IndexTemplateMetadata.Builder template) {
20411973
return put(template.build());
20421974
}
@@ -2358,6 +2290,7 @@ public Metadata build(boolean skipNameCollisionChecks) {
23582290
int totalNumberOfShards = 0;
23592291
int totalOpenIndexShards = 0;
23602292

2293+
ImmutableOpenMap.Builder<String, Set<Index>> aliasedIndicesBuilder = ImmutableOpenMap.builder();
23612294
final String[] allIndicesArray = new String[indicesMap.size()];
23622295
int i = 0;
23632296
final Set<String> sha256HashesInUse = checkForUnusedMappings ? Sets.newHashSetWithExpectedSize(mappingsByHash.size()) : null;
@@ -2389,9 +2322,19 @@ public Metadata build(boolean skipNameCollisionChecks) {
23892322
sha256HashesInUse.add(mapping.getSha256());
23902323
}
23912324
}
2325+
for (var alias : indexMetadata.getAliases().keySet()) {
2326+
var indices = aliasedIndicesBuilder.get(alias);
2327+
if (indices == null) {
2328+
indices = new HashSet<>();
2329+
aliasedIndicesBuilder.put(alias, indices);
2330+
}
2331+
indices.add(indexMetadata.getIndex());
2332+
}
23922333
}
2393-
2394-
var aliasedIndices = this.aliasedIndices.build();
2334+
for (String alias : aliasedIndicesBuilder.keys()) {
2335+
aliasedIndicesBuilder.put(alias, Collections.unmodifiableSet(aliasedIndicesBuilder.get(alias)));
2336+
}
2337+
var aliasedIndices = aliasedIndicesBuilder.build();
23952338
for (var entry : aliasedIndices.entrySet()) {
23962339
List<IndexMetadata> aliasIndices = entry.getValue().stream().map(idx -> indicesMap.get(idx.getName())).toList();
23972340
validateAlias(entry.getKey(), aliasIndices);

‎server/src/main/java/org/elasticsearch/index/shard/IndexShard.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3860,8 +3860,7 @@ public int getActiveOperationsCount() {
38603860
* listener handles all exception cases internally.
38613861
*/
38623862
public final void syncAfterWrite(Translog.Location location, Consumer<Exception> syncListener) {
3863-
// TODO AwaitsFix https://github.com/elastic/elasticsearch/issues/97183
3864-
// assert indexShardOperationPermits.getActiveOperationsCount() != 0;
3863+
assert indexShardOperationPermits.getActiveOperationsCount() != 0;
38653864
verifyNotClosed();
38663865
getEngine().asyncEnsureTranslogSynced(location, syncListener);
38673866
}

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,9 @@ public static boolean isUnsupported(DataType from) {
370370
}
371371

372372
public static boolean isString(DataType t) {
373+
if (EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG.isEnabled() && t == SEMANTIC_TEXT) {
374+
return true;
375+
}
373376
return t == KEYWORD || t == TEXT;
374377
}
375378

@@ -585,7 +588,7 @@ static Builder builder() {
585588
}
586589

587590
public DataType noText() {
588-
return this == TEXT ? KEYWORD : this;
591+
return isString(this) ? KEYWORD : this;
589592
}
590593

591594
/**

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@
6868
},
6969
"value": {
7070
"type": "long"
71+
},
72+
"st_base64": {
73+
"type": "semantic_text",
74+
"inference_id": "test_sparse_inference"
7175
}
7276
}
7377
}

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long
2-
1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001
3-
2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002
4-
3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003
1+
_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text
2+
1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw==
3+
2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8=
4+
3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003,

0 commit comments

Comments
 (0)