Skip to content

Commit dc2148b

Browse files
Merge branch 'main' into entitlements/fix
2 parents d6cbd40 + 9b7ee21 commit dc2148b

File tree

20 files changed

+1134
-447
lines changed

20 files changed

+1134
-447
lines changed

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.cluster.metadata.IndexMetadata;
1515
import org.elasticsearch.cluster.metadata.Metadata;
1616
import org.elasticsearch.cluster.node.DiscoveryNodes;
17+
import org.elasticsearch.cluster.project.DefaultProjectResolver;
1718
import org.elasticsearch.cluster.routing.IndexRoutingTable;
1819
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
1920
import org.elasticsearch.cluster.routing.RecoverySource;
@@ -178,7 +179,12 @@ public void setUp() throws Exception {
178179
new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet())
179180
);
180181
clusterService.getClusterApplierService().setInitialState(initialClusterState);
181-
indicatorService = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, new SystemIndices(List.of()));
182+
indicatorService = new ShardsAvailabilityHealthIndicatorService(
183+
clusterService,
184+
allocationService,
185+
new SystemIndices(List.of()),
186+
DefaultProjectResolver.INSTANCE
187+
);
182188
}
183189

184190
private int toInt(String v) {

docs/changelog/106953.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 106953
2+
summary: Optimize usage calculation in ILM policies retrieval API
3+
area: ILM+SLM
4+
type: enhancement
5+
issues:
6+
- 105773

modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,12 @@ public ShardsAvailabilityPlugin() {}
2727
@Override
2828
public Collection<?> createComponents(PluginServices services) {
2929
this.shardHealthService.set(
30-
new ShardsAvailabilityHealthIndicatorService(services.clusterService(), services.allocationService(), services.systemIndices())
30+
new ShardsAvailabilityHealthIndicatorService(
31+
services.clusterService(),
32+
services.allocationService(),
33+
services.systemIndices(),
34+
services.projectResolver()
35+
)
3136
);
3237
return Set.of(this.shardHealthService.get());
3338
}

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.elasticsearch.reindex.ReindexPlugin;
3636
import org.elasticsearch.rest.RestStatus;
3737
import org.elasticsearch.test.ESIntegTestCase;
38+
import org.elasticsearch.test.junit.annotations.TestLogging;
3839
import org.elasticsearch.transport.RemoteTransportException;
3940
import org.elasticsearch.xcontent.XContentType;
4041
import org.junit.ClassRule;
@@ -87,6 +88,10 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
8788
}
8889

8990
@SuppressWarnings("unchecked")
91+
@TestLogging(
92+
reason = "understanding why ipinfo asn database sometimes is not loaded",
93+
value = "org.elasticsearch.ingest.geoip.DatabaseNodeService:TRACE"
94+
)
9095
public void testEnterpriseDownloaderTask() throws Exception {
9196
/*
9297
* This test starts the enterprise geoip downloader task, and creates a database configuration. Then it creates an ingest
@@ -121,17 +126,17 @@ public void testEnterpriseDownloaderTask() throws Exception {
121126
assertNotNull(returnedSource);
122127
Object targetFieldValue = returnedSource.get(targetField);
123128
assertNotNull(targetFieldValue);
124-
assertThat(((Map<String, Object>) targetFieldValue).get("organization_name"), equalTo("Bredband2 AB"));
129+
assertThat(((Map<String, Object>) targetFieldValue).get("city_name"), equalTo("Linköping"));
125130
});
126131
assertBusy(() -> {
127132
logger.info("Ingesting another test document");
128-
String documentId = ingestDocument(indexName, iplocationPipelineName, sourceField, "12.10.66.1");
133+
String documentId = ingestDocument(indexName, iplocationPipelineName, sourceField, "103.134.48.0");
129134
GetResponse getResponse = client().get(new GetRequest(indexName, documentId)).actionGet();
130135
Map<String, Object> returnedSource = getResponse.getSource();
131136
assertNotNull(returnedSource);
132137
Object targetFieldValue = returnedSource.get(targetField);
133138
assertNotNull(targetFieldValue);
134-
assertThat(((Map<String, Object>) targetFieldValue).get("organization_name"), equalTo("OAKLAWN JOCKEY CLUB, INC."));
139+
assertThat(((Map<String, Object>) targetFieldValue).get("organization_name"), equalTo("PT Nevigate Telekomunikasi Indonesia"));
135140
});
136141
}
137142

muted-tests.yml

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,6 @@ tests:
8686
- class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT
8787
method: test {yaml=/10_apm/Test template reinstallation}
8888
issue: https://github.com/elastic/elasticsearch/issues/116445
89-
- class: org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderIT
90-
method: testEnterpriseDownloaderTask
91-
issue: https://github.com/elastic/elasticsearch/issues/115163
9289
- class: org.elasticsearch.versioning.ConcurrentSeqNoVersioningIT
9390
method: testSeqNoCASLinearizability
9491
issue: https://github.com/elastic/elasticsearch/issues/117249
@@ -302,15 +299,9 @@ tests:
302299
- class: org.elasticsearch.packaging.test.BootstrapCheckTests
303300
method: test20RunWithBootstrapChecks
304301
issue: https://github.com/elastic/elasticsearch/issues/124940
305-
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
306-
method: testStopQueryLocalNoRemotes
307-
issue: https://github.com/elastic/elasticsearch/issues/124959
308302
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
309303
method: testStopQueryLocal
310304
issue: https://github.com/elastic/elasticsearch/issues/121672
311-
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
312-
method: testStopQuery
313-
issue: https://github.com/elastic/elasticsearch/issues/124977
314305
- class: org.elasticsearch.integration.DataStreamLifecycleServiceRuntimeSecurityIT
315306
method: testRolloverLifecycleAndForceMergeAuthorized
316307
issue: https://github.com/elastic/elasticsearch/issues/124978
@@ -401,6 +392,9 @@ tests:
401392
- class: org.elasticsearch.xpack.esql.action.ManyShardsIT
402393
method: testCancelUnnecessaryRequests
403394
issue: https://github.com/elastic/elasticsearch/issues/125947
395+
- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT
396+
method: testResumingSearchableSnapshotFromPartialToFull
397+
issue: https://github.com/elastic/elasticsearch/issues/125789
404398

405399
# Examples:
406400
#

server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.cluster.ClusterChangedEvent;
1414
import org.elasticsearch.cluster.ClusterStateListener;
1515
import org.elasticsearch.cluster.metadata.IndexMetadata;
16+
import org.elasticsearch.cluster.project.ProjectResolver;
1617
import org.elasticsearch.cluster.routing.RoutingNodes;
1718
import org.elasticsearch.cluster.routing.allocation.AllocationService;
1819
import org.elasticsearch.cluster.service.ClusterService;
@@ -133,8 +134,9 @@ private void assertHealthDuring(Matcher<HealthStatus> statusMatcher, Runnable ac
133134
var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
134135
var allocationService = internalCluster().getCurrentMasterNodeInstance(AllocationService.class);
135136
var systemIndices = internalCluster().getCurrentMasterNodeInstance(SystemIndices.class);
137+
var projectResolver = internalCluster().getCurrentMasterNodeInstance(ProjectResolver.class);
136138

137-
var service = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, systemIndices);
139+
var service = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, systemIndices, projectResolver);
138140
var states = new ArrayList<RoutingNodesAndHealth>();
139141
var listener = new ClusterStateListener() {
140142
@Override

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

Lines changed: 74 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
import java.time.Instant;
6363
import java.util.ArrayList;
6464
import java.util.Arrays;
65+
import java.util.Collection;
6566
import java.util.Collections;
6667
import java.util.Comparator;
6768
import java.util.HashMap;
@@ -1087,9 +1088,8 @@ static Set<String> dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr
10871088
.map(templateName -> projectMetadata.templatesV2().get(templateName))
10881089
.filter(Objects::nonNull)
10891090
.map(ComposableIndexTemplate::indexPatterns)
1090-
.map(Set::copyOf)
1091-
.reduce(Sets::union)
1092-
.orElse(Set.of());
1091+
.flatMap(List::stream)
1092+
.collect(Collectors.toSet());
10931093

10941094
return projectMetadata.dataStreams()
10951095
.values()
@@ -1099,9 +1099,10 @@ static Set<String> dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr
10991099
.filter(ds -> {
11001100
// Retrieve the templates that match the data stream name ordered by priority
11011101
List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(
1102-
projectMetadata,
1102+
projectMetadata.templatesV2().entrySet(),
11031103
ds.getName(),
1104-
ds.isHidden()
1104+
ds.isHidden(),
1105+
false
11051106
);
11061107
if (candidates.isEmpty()) {
11071108
throw new IllegalStateException("Data stream " + ds.getName() + " did not match any composable index templates.");
@@ -1110,13 +1111,10 @@ static Set<String> dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr
11101111
// Limit data streams that can ONLY use any of the specified templates, we do this by filtering
11111112
// the matching templates that are others than the ones requested and could be a valid template to use.
11121113
return candidates.stream()
1113-
.filter(
1114+
.noneMatch(
11141115
template -> templateNames.contains(template.v1()) == false
11151116
&& isGlobalAndHasIndexHiddenSetting(projectMetadata, template.v2(), template.v1()) == false
1116-
)
1117-
.map(Tuple::v1)
1118-
.toList()
1119-
.isEmpty();
1117+
);
11201118
})
11211119
.map(DataStream::getName)
11221120
.collect(Collectors.toSet());
@@ -1357,7 +1355,41 @@ public static List<IndexTemplateMetadata> findV1Templates(
13571355
*/
13581356
@Nullable
13591357
public static String findV2Template(ProjectMetadata projectMetadata, String indexName, boolean isHidden) {
1360-
final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(projectMetadata, indexName, isHidden);
1358+
return findV2Template(projectMetadata, projectMetadata.templatesV2().entrySet(), indexName, isHidden, false);
1359+
}
1360+
1361+
/**
1362+
* Return the name (id) of the highest matching index template out of the provided templates (that <i>need</i> to be sorted descending
1363+
* on priority beforehand), or the given index name. In the event that no templates are matched, {@code null} is returned.
1364+
*/
1365+
@Nullable
1366+
public static String findV2TemplateFromSortedList(
1367+
ProjectMetadata projectMetadata,
1368+
Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
1369+
String indexName,
1370+
boolean isHidden
1371+
) {
1372+
return findV2Template(projectMetadata, templates, indexName, isHidden, true);
1373+
}
1374+
1375+
/**
1376+
* Return the name (id) of the highest matching index template, out of the provided templates, for the given index name. In
1377+
* the event that no templates are matched, {@code null} is returned.
1378+
*/
1379+
@Nullable
1380+
private static String findV2Template(
1381+
ProjectMetadata projectMetadata,
1382+
Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
1383+
String indexName,
1384+
boolean isHidden,
1385+
boolean exitOnFirstMatch
1386+
) {
1387+
final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(
1388+
templates,
1389+
indexName,
1390+
isHidden,
1391+
exitOnFirstMatch
1392+
);
13611393
if (candidates.isEmpty()) {
13621394
return null;
13631395
}
@@ -1386,33 +1418,39 @@ public static String findV2Template(ProjectMetadata projectMetadata, String inde
13861418
* Return an ordered list of the name (id) and composable index templates that would apply to an index. The first
13871419
* one is the winner template that is applied to this index. In the event that no templates are matched,
13881420
* an empty list is returned.
1421+
* @param templates a list of template entries (name, template) - needs to be sorted when {@code exitOnFirstMatch} is {@code true}
1422+
* @param indexName the index (or data stream) name that should be used for matching the index patterns on the templates
1423+
* @param isHidden whether {@code indexName} belongs to a hidden index - this option is redundant for data streams, as backing indices
1424+
* of data streams will always be returned, regardless of whether the data stream is hidden or not
1425+
* @param exitOnFirstMatch if true, we return immediately after finding a match. That means that the <code>templates</code>
1426+
* parameter needs to be sorted based on priority (descending) for this method to return a sensible result,
1427+
* otherwise this method would just return the first template that matches the name, in an unspecified order
13891428
*/
1390-
static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
1391-
ProjectMetadata projectMetadata,
1429+
private static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
1430+
Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
13921431
String indexName,
1393-
boolean isHidden
1432+
boolean isHidden,
1433+
boolean exitOnFirstMatch
13941434
) {
1435+
assert exitOnFirstMatch == false || areTemplatesSorted(templates) : "Expected templates to be sorted";
13951436
final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName);
13961437
final Predicate<String> patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, resolvedIndexName);
13971438
final List<Tuple<String, ComposableIndexTemplate>> candidates = new ArrayList<>();
1398-
for (Map.Entry<String, ComposableIndexTemplate> entry : projectMetadata.templatesV2().entrySet()) {
1439+
for (Map.Entry<String, ComposableIndexTemplate> entry : templates) {
13991440
final String name = entry.getKey();
14001441
final ComposableIndexTemplate template = entry.getValue();
14011442
/*
14021443
* We do not ordinarily return match-all templates for hidden indices. But all backing indices for data streams are hidden,
14031444
* and we do want to return even match-all templates for those. Not doing so can result in a situation where a data stream is
14041445
* built with a template that none of its indices match.
14051446
*/
1406-
if (isHidden == false || template.getDataStreamTemplate() != null) {
1407-
if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
1408-
candidates.add(Tuple.tuple(name, template));
1409-
}
1410-
} else {
1411-
final boolean isNotMatchAllTemplate = noneMatch(template.indexPatterns(), Regex::isMatchAllPattern);
1412-
if (isNotMatchAllTemplate) {
1413-
if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
1414-
candidates.add(Tuple.tuple(name, template));
1415-
}
1447+
if (anyMatch(template.indexPatterns(), Regex::isMatchAllPattern) && isHidden && template.getDataStreamTemplate() == null) {
1448+
continue;
1449+
}
1450+
if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
1451+
candidates.add(Tuple.tuple(name, template));
1452+
if (exitOnFirstMatch) {
1453+
return candidates;
14161454
}
14171455
}
14181456
}
@@ -1421,6 +1459,17 @@ static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
14211459
return candidates;
14221460
}
14231461

1462+
private static boolean areTemplatesSorted(Collection<Map.Entry<String, ComposableIndexTemplate>> templates) {
1463+
ComposableIndexTemplate previousTemplate = null;
1464+
for (Map.Entry<String, ComposableIndexTemplate> template : templates) {
1465+
if (previousTemplate != null && template.getValue().priorityOrZero() > previousTemplate.priorityOrZero()) {
1466+
return false;
1467+
}
1468+
previousTemplate = template.getValue();
1469+
}
1470+
return true;
1471+
}
1472+
14241473
// Checks if a global template specifies the `index.hidden` setting. This check is important because a global
14251474
// template shouldn't specify the `index.hidden` setting, we leave it up to the caller to handle this situation.
14261475
private static boolean isGlobalAndHasIndexHiddenSetting(

0 commit comments

Comments
 (0)