Skip to content

Commit 791e394

Browse files
idegtiarenkoidegtiarenko
committed
Attempt to properly fail partial resolutions
1 parent 926b705 commit 791e394

File tree

9 files changed

+131
-40
lines changed

9 files changed

+131
-40
lines changed

‎server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.elasticsearch.core.Nullable;
3939
import org.elasticsearch.core.Releasable;
4040
import org.elasticsearch.core.Tuple;
41+
import org.elasticsearch.index.IndexNotFoundException;
4142
import org.elasticsearch.index.shard.ShardId;
4243
import org.elasticsearch.indices.IndicesService;
4344
import org.elasticsearch.injection.guice.Inject;
@@ -157,21 +158,23 @@ private void doExecuteForked(
157158
final Map<String, OriginalIndices> remoteClusterIndices = transportService.getRemoteClusterService()
158159
.groupIndices(request.indicesOptions(), request.indices());
159160
final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
160-
final String[] concreteIndices;
161-
if (localIndices == null) {
161+
final FailureCollector indexFailures = new FailureCollector();
162+
final String[] concreteIndices = localIndices == null
162163
// in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices
163-
concreteIndices = Strings.EMPTY_ARRAY;
164-
} else {
165-
concreteIndices = indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), localIndices);
166-
}
164+
? Strings.EMPTY_ARRAY
165+
: indexNameExpressionResolver.concreteIndexNames(
166+
projectState.metadata(),
167+
localIndices,
168+
expr -> indexFailures.collect(expr, new IndexNotFoundException(expr))
169+
);
167170

168171
if (concreteIndices.length == 0 && remoteClusterIndices.isEmpty()) {
169172
listener.onResponse(new FieldCapabilitiesResponse(new String[0], Collections.emptyMap()));
170173
return;
171174
}
172175

173176
checkIndexBlocks(projectState, concreteIndices);
174-
final FailureCollector indexFailures = new FailureCollector();
177+
175178
final Map<String, FieldCapabilitiesIndexResponse> indexResponses = new HashMap<>();
176179
// This map is used to share the index response for indices which have the same index mapping hash to reduce the memory usage.
177180
final Map<String, FieldCapabilitiesIndexResponse> indexMappingHashToResponses = new HashMap<>();

‎server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,10 @@ public static IndicesOptions.Builder builder(IndicesOptions indicesOptions) {
6464
* Controls the way the target indices will be handled.
6565
* @param allowUnavailableTargets, if false when any of the concrete targets requested does not exist, throw an error
6666
*/
67-
public record ConcreteTargetOptions(boolean allowUnavailableTargets) implements ToXContentFragment {
67+
public record ConcreteTargetOptions(boolean allowUnavailableTargets, boolean reportFailuresToResolve) implements ToXContentFragment {
6868
public static final String IGNORE_UNAVAILABLE = "ignore_unavailable";
69-
public static final ConcreteTargetOptions ALLOW_UNAVAILABLE_TARGETS = new ConcreteTargetOptions(true);
70-
public static final ConcreteTargetOptions ERROR_WHEN_UNAVAILABLE_TARGETS = new ConcreteTargetOptions(false);
69+
public static final ConcreteTargetOptions ALLOW_UNAVAILABLE_TARGETS = new ConcreteTargetOptions(true, false);
70+
public static final ConcreteTargetOptions ERROR_WHEN_UNAVAILABLE_TARGETS = new ConcreteTargetOptions(false, false);
7171

7272
public static ConcreteTargetOptions fromParameter(Object ignoreUnavailableString, ConcreteTargetOptions defaultOption) {
7373
if (ignoreUnavailableString == null && defaultOption != null) {
@@ -1295,7 +1295,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic
12951295
);
12961296
}
12971297
return IndicesOptions.builder()
1298-
.concreteTargetOptions(new ConcreteTargetOptions(ignoreUnavailable))
1298+
.concreteTargetOptions(new ConcreteTargetOptions(ignoreUnavailable, false))
12991299
.wildcardOptions(wildcards)
13001300
.gatekeeperOptions(generalOptions)
13011301
.build();

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import java.util.SortedMap;
6161
import java.util.function.BiFunction;
6262
import java.util.function.BiPredicate;
63+
import java.util.function.Consumer;
6364
import java.util.function.Function;
6465
import java.util.function.LongSupplier;
6566
import java.util.function.Predicate;
@@ -151,6 +152,23 @@ public String[] concreteIndexNames(ProjectMetadata project, IndicesRequest reque
151152
return concreteIndexNames(context, request.indices());
152153
}
153154

155+
public String[] concreteIndexNames(ProjectMetadata project, IndicesRequest request, Consumer<String> onResolutionFailure) {
156+
Context context = new Context(
157+
project,
158+
request.indicesOptions(),
159+
System.currentTimeMillis(),
160+
false,
161+
false,
162+
request.includeDataStreams(),
163+
false,
164+
getSystemIndexAccessLevel(),
165+
getSystemIndexAccessPredicate(),
166+
getNetNewSystemIndexPredicate(),
167+
onResolutionFailure
168+
);
169+
return concreteIndexNames(context, request.indices());
170+
}
171+
154172
/**
155173
* Same as {@link #concreteIndexNames(ClusterState, IndicesRequest)}, but access to system indices is always allowed.
156174
*/
@@ -450,7 +468,10 @@ protected static Collection<ResolvedExpression> resolveExpressionsToResources(Co
450468
resources.remove(new ResolvedExpression(baseExpression, selector));
451469
} else if (ensureAliasOrIndexExists(context, baseExpression, selector)) {
452470
resources.add(new ResolvedExpression(baseExpression, selector));
453-
}
471+
} else if (context.getOptions().concreteTargetOptions().reportFailuresToResolve()
472+
&& context.onResolutionFailure() != null) {
473+
context.onResolutionFailure().accept(baseExpression);
474+
}
454475
}
455476
}
456477
return resources;
@@ -592,7 +613,8 @@ public Index[] concreteIndices(ProjectMetadata project, IndicesRequest request,
592613
false,
593614
getSystemIndexAccessLevel(),
594615
getSystemIndexAccessPredicate(),
595-
getNetNewSystemIndexPredicate()
616+
getNetNewSystemIndexPredicate(),
617+
null
596618
);
597619
return concreteIndices(context, request.indices());
598620
}
@@ -1542,6 +1564,7 @@ public static class Context {
15421564
private final SystemIndexAccessLevel systemIndexAccessLevel;
15431565
private final Predicate<String> systemIndexAccessPredicate;
15441566
private final Predicate<String> netNewSystemIndexPredicate;
1567+
private final Consumer<String> onResolutionFailure;
15451568

15461569
Context(ProjectMetadata project, IndicesOptions options, SystemIndexAccessLevel systemIndexAccessLevel) {
15471570
this(project, options, systemIndexAccessLevel, Predicates.always(), Predicates.never());
@@ -1584,7 +1607,8 @@ public static class Context {
15841607
false,
15851608
systemIndexAccessLevel,
15861609
systemIndexAccessPredicate,
1587-
netNewSystemIndexPredicate
1610+
netNewSystemIndexPredicate,
1611+
null
15881612
);
15891613
}
15901614

@@ -1609,7 +1633,8 @@ public static class Context {
16091633
preserveDataStreams,
16101634
systemIndexAccessLevel,
16111635
systemIndexAccessPredicate,
1612-
netNewSystemIndexPredicate
1636+
netNewSystemIndexPredicate,
1637+
null
16131638
);
16141639
}
16151640

@@ -1631,7 +1656,8 @@ public static class Context {
16311656
false,
16321657
systemIndexAccessLevel,
16331658
systemIndexAccessPredicate,
1634-
netNewSystemIndexPredicate
1659+
netNewSystemIndexPredicate,
1660+
null
16351661
);
16361662
}
16371663

@@ -1645,7 +1671,8 @@ protected Context(
16451671
boolean preserveDataStreams,
16461672
SystemIndexAccessLevel systemIndexAccessLevel,
16471673
Predicate<String> systemIndexAccessPredicate,
1648-
Predicate<String> netNewSystemIndexPredicate
1674+
Predicate<String> netNewSystemIndexPredicate,
1675+
Consumer<String> onResolutionFailure
16491676
) {
16501677
this.project = project;
16511678
this.options = options;
@@ -1657,6 +1684,7 @@ protected Context(
16571684
this.systemIndexAccessLevel = systemIndexAccessLevel;
16581685
this.systemIndexAccessPredicate = systemIndexAccessPredicate;
16591686
this.netNewSystemIndexPredicate = netNewSystemIndexPredicate;
1687+
this.onResolutionFailure = onResolutionFailure;
16601688
}
16611689

16621690
public ProjectMetadata getProject() {
@@ -1702,6 +1730,10 @@ public boolean isPreserveDataStreams() {
17021730
public Predicate<String> getSystemIndexAccessPredicate() {
17031731
return systemIndexAccessPredicate;
17041732
}
1733+
1734+
public Consumer<String> onResolutionFailure() {
1735+
return onResolutionFailure;
1736+
}
17051737
}
17061738

17071739
/**

‎server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,7 @@ public void testFromMap() {
333333
}
334334

335335
public void testToXContent() throws IOException {
336-
ConcreteTargetOptions concreteTargetOptions = new ConcreteTargetOptions(randomBoolean());
336+
ConcreteTargetOptions concreteTargetOptions = new ConcreteTargetOptions(randomBoolean(), randomBoolean());
337337
WildcardOptions wildcardOptions = new WildcardOptions(
338338
randomBoolean(),
339339
randomBoolean(),
@@ -374,7 +374,7 @@ public void testFromXContent() throws IOException {
374374
randomBoolean(),
375375
randomBoolean()
376376
);
377-
ConcreteTargetOptions concreteTargetOptions = new ConcreteTargetOptions(randomBoolean());
377+
ConcreteTargetOptions concreteTargetOptions = new ConcreteTargetOptions(randomBoolean(), randomBoolean());
378378

379379
IndicesOptions indicesOptions = IndicesOptions.builder()
380380
.concreteTargetOptions(concreteTargetOptions)

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1184,9 +1184,7 @@ private static IndexRequest createDoc(String dsName, String id, String ts, Objec
11841184
return new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE).id(id).source("@timestamp", ts, "count", count);
11851185
}
11861186

1187-
public void testOverlappingIndexPatterns() throws Exception {
1188-
String[] indexNames = { "test_overlapping_index_patterns_1", "test_overlapping_index_patterns_2" };
1189-
1187+
public void testOverlappingIndexPatterns() {
11901188
assertAcked(
11911189
client().admin()
11921190
.indices()
@@ -1228,15 +1226,15 @@ public void testErrorMessageForUnknownIndex() {
12281226
expectThrows(
12291227
VerificationException.class,
12301228
containsString("Unknown index [no-such-index]"),
1231-
() -> run("from no-such-index", randomPragmas(), null, randomBoolean())
1229+
() -> run("from no-such-index", randomPragmas(), null, false)
12321230
);
12331231
}
12341232

12351233
public void testErrorMessageForUnknownIndexInPatternList() {
12361234
expectThrows(
12371235
VerificationException.class,
12381236
containsString("Unknown index [no-such-index]"),
1239-
() -> run("from test,no-such-index", randomPragmas(), null, randomBoolean())
1237+
() -> run("from test,no-such-index", randomPragmas(), null, false)
12401238
);
12411239
}
12421240

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import org.elasticsearch.action.NoShardAvailableActionException;
1010
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
1111
import org.elasticsearch.core.Nullable;
12+
import org.elasticsearch.index.IndexNotFoundException;
1213

1314
import java.util.Map;
1415
import java.util.Objects;
@@ -27,25 +28,27 @@ public static IndexResolution valid(
2728
EsIndex index,
2829
Set<String> resolvedIndices,
2930
Set<NoShardAvailableActionException> unavailableShards,
31+
Set<IndexNotFoundException> unavailableIndices,
3032
Map<String, FieldCapabilitiesFailure> unavailableClusters
3133
) {
3234
Objects.requireNonNull(index, "index must not be null if it was found");
3335
Objects.requireNonNull(resolvedIndices, "resolvedIndices must not be null");
3436
Objects.requireNonNull(unavailableShards, "unavailableShards must not be null");
37+
Objects.requireNonNull(unavailableIndices, "unavailableIndices must not be null");
3538
Objects.requireNonNull(unavailableClusters, "unavailableClusters must not be null");
36-
return new IndexResolution(index, null, resolvedIndices, unavailableShards, unavailableClusters);
39+
return new IndexResolution(index, null, resolvedIndices, unavailableShards, unavailableIndices, unavailableClusters);
3740
}
3841

3942
/**
4043
* Use this method only if the set of concrete resolved indices is the same as EsIndex#concreteIndices().
4144
*/
4245
public static IndexResolution valid(EsIndex index) {
43-
return valid(index, index.concreteIndices(), Set.of(), Map.of());
46+
return valid(index, index.concreteIndices(), Set.of(), Set.of(), Map.of());
4447
}
4548

4649
public static IndexResolution invalid(String invalid) {
4750
Objects.requireNonNull(invalid, "invalid must not be null to signal that the index is invalid");
48-
return new IndexResolution(null, invalid, Set.of(), Set.of(), Map.of());
51+
return new IndexResolution(null, invalid, Set.of(), Set.of(), Set.of(), Map.of());
4952
}
5053

5154
public static IndexResolution notFound(String name) {
@@ -60,6 +63,7 @@ public static IndexResolution notFound(String name) {
6063
// all indices found by field-caps
6164
private final Set<String> resolvedIndices;
6265
private final Set<NoShardAvailableActionException> unavailableShards;
66+
private final Set<IndexNotFoundException> unavailableIndices;
6367
// remote clusters included in the user's index expression that could not be connected to
6468
private final Map<String, FieldCapabilitiesFailure> unavailableClusters;
6569

@@ -68,12 +72,14 @@ private IndexResolution(
6872
@Nullable String invalid,
6973
Set<String> resolvedIndices,
7074
Set<NoShardAvailableActionException> unavailableShards,
75+
Set<IndexNotFoundException> unavailableIndices,
7176
Map<String, FieldCapabilitiesFailure> unavailableClusters
7277
) {
7378
this.index = index;
7479
this.invalid = invalid;
7580
this.resolvedIndices = resolvedIndices;
7681
this.unavailableShards = unavailableShards;
82+
this.unavailableIndices = unavailableIndices;
7783
this.unavailableClusters = unavailableClusters;
7884
}
7985

@@ -122,6 +128,13 @@ public Set<NoShardAvailableActionException> getUnavailableShards() {
122128
return unavailableShards;
123129
}
124130

131+
/**
132+
* @return set of unavailable indices during index resolution
133+
*/
134+
public Set<IndexNotFoundException> getUnavailableIndices() {
135+
return unavailableIndices;
136+
}
137+
125138
@Override
126139
public boolean equals(Object obj) {
127140
if (obj == null || obj.getClass() != getClass()) {

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
import java.util.function.Function;
105105
import java.util.stream.Collectors;
106106

107+
import static java.util.stream.Collectors.joining;
107108
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
108109
import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
109110

@@ -421,6 +422,7 @@ private void initializeClusterData(List<IndexPattern> indices, EsqlExecutionInfo
421422
}
422423
}
423424

425+
@SuppressWarnings("checkstyle:LineLength")
424426
private void preAnalyzeMainIndices(
425427
PreAnalyzer.PreAnalysis preAnalysis,
426428
EsqlExecutionInfo executionInfo,
@@ -460,11 +462,25 @@ private void preAnalyzeMainIndices(
460462
result.fieldNames,
461463
requestFilter,
462464
listener.delegateFailure((l, indexResolution) -> {
463-
if (configuration.allowPartialResults() == false && indexResolution.getUnavailableShards().isEmpty() == false) {
464-
l.onFailure(indexResolution.getUnavailableShards().iterator().next());
465-
} else {
466-
l.onResponse(result.withIndexResolution(indexResolution));
465+
if (configuration.allowPartialResults() == false) {
466+
if (indexResolution.getUnavailableShards().isEmpty() == false) {
467+
l.onFailure(indexResolution.getUnavailableShards().iterator().next());
468+
return;
469+
}
470+
if (indexResolution.getUnavailableIndices().isEmpty() == false) {
471+
l.onFailure(
472+
new VerificationException(
473+
"Unknown index {}",
474+
indexResolution.getUnavailableIndices()
475+
.stream()
476+
.map(it -> it.getIndex().getName())
477+
.collect(joining(", ", "[", "]"))
478+
)
479+
);
480+
return;
481+
}
467482
}
483+
l.onResponse(result.withIndexResolution(indexResolution));
468484
})
469485
);
470486
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.common.Strings;
1919
import org.elasticsearch.common.util.Maps;
2020
import org.elasticsearch.index.IndexMode;
21+
import org.elasticsearch.index.IndexNotFoundException;
2122
import org.elasticsearch.index.mapper.TimeSeriesParams;
2223
import org.elasticsearch.index.query.QueryBuilder;
2324
import org.elasticsearch.threadpool.ThreadPool;
@@ -56,7 +57,7 @@ public class IndexResolver {
5657
public static final String UNMAPPED = "unmapped";
5758

5859
public static final IndicesOptions FIELD_CAPS_INDICES_OPTIONS = IndicesOptions.builder()
59-
.concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
60+
.concreteTargetOptions(new IndicesOptions.ConcreteTargetOptions(true, true))
6061
.wildcardOptions(
6162
IndicesOptions.WildcardOptions.builder()
6263
.matchOpen(true)
@@ -154,9 +155,13 @@ public static IndexResolution mergedMappings(String indexPattern, FieldCapabilit
154155
);
155156

156157
Set<NoShardAvailableActionException> unavailableShards = new HashSet<>();
158+
Set<IndexNotFoundException> unavailableIndices = new HashSet<>();
157159
for (FieldCapabilitiesFailure failure : fieldCapsResponse.getFailures()) {
158-
if (failure.getException() instanceof NoShardAvailableActionException e) {
159-
unavailableShards.add(e);
160+
switch (failure.getException()) {
161+
case NoShardAvailableActionException nsaae -> unavailableShards.add(nsaae);
162+
case IndexNotFoundException infe -> unavailableIndices.add(infe);
163+
default -> {
164+
/* do nothing */}
160165
}
161166
}
162167

@@ -171,7 +176,7 @@ public static IndexResolution mergedMappings(String indexPattern, FieldCapabilit
171176
}
172177
// If all the mappings are empty we return an empty set of resolved indices to line up with QL
173178
var index = new EsIndex(indexPattern, rootFields, allEmpty ? Map.of() : concreteIndices, partiallyUnmappedFields);
174-
return IndexResolution.valid(index, concreteIndices.keySet(), unavailableShards, unavailableRemotes);
179+
return IndexResolution.valid(index, concreteIndices.keySet(), unavailableShards, unavailableIndices, unavailableRemotes);
175180
}
176181

177182
private static Map<String, List<IndexFieldCapabilities>> collectFieldCaps(FieldCapabilitiesResponse fieldCapsResponse) {

0 commit comments

Comments
 (0)