Skip to content

Fix ESQL index resolution #135826

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 11 commits into
base: main
Choose a base branch
from
+262 −262
Draft

Fix ESQL index resolution #135826

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
5bf94ba
Fix ESQL index resolution
idegtiarenko Oct 2, 2025
4f4f327
fix some tests
idegtiarenko Oct 2, 2025
d3cd665
fix some tests
idegtiarenko Oct 2, 2025
675b7c2
fix some tests
idegtiarenko Oct 2, 2025
b0a563c
Merge branch 'main' into index_resolution
idegtiarenko Oct 13, 2025
555c294
upd
idegtiarenko Oct 13, 2025
82459d9
[CI] Auto commit changes from spotless
Oct 13, 2025
d30da96
upd
idegtiarenko Oct 13, 2025
cc5fec5
fix skip
idegtiarenko Oct 13, 2025
f4d565f
fix test
idegtiarenko Oct 13, 2025
237b470
fix CrossClusterQueryIT
idegtiarenko Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 76 additions & 47 deletions ...n/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
import org.hamcrest.Matcher;
import org.junit.Before;
import org.junit.ClassRule;

Expand Down Expand Up @@ -299,32 +300,38 @@ public void testAliasFilter() throws Exception {
}

public void testUnauthorizedIndices() throws IOException {
ResponseException error;
error = expectThrows(ResponseException.class, () -> runESQLCommand("user1", "from index-user2 | stats sum(value)"));
assertThat(error.getMessage(), containsString("Unknown index [index-user2]"));
assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));

error = expectThrows(ResponseException.class, () -> runESQLCommand("user2", "from index-user1 | stats sum(value)"));
assertThat(error.getMessage(), containsString("Unknown index [index-user1]"));
assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));

error = expectThrows(ResponseException.class, () -> runESQLCommand("alias_user2", "from index-user2 | stats sum(value)"));
assertThat(error.getMessage(), containsString("Unknown index [index-user2]"));
assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));

error = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", "from index-user1 | stats sum(value)"));
assertThat(error.getMessage(), containsString("Unknown index [index-user1]"));
assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
expectThrowsResponseException(
() -> runESQLCommand("user1", "from index-user2 | stats sum(value)"),
HttpStatus.SC_FORBIDDEN,
containsString("unauthorized for user [test-admin] run as [user1] with effective roles [user1] on indices [index-user2]")
);
expectThrowsResponseException(
() -> runESQLCommand("user2", "from index-user1 | stats sum(value)"),
HttpStatus.SC_FORBIDDEN,
containsString("unauthorized for user [test-admin] run as [user2] with effective roles [user2] on indices [index-user1]")
);
expectThrowsResponseException(
() -> runESQLCommand("alias_user2", "from index-user2 | stats sum(value)"),
HttpStatus.SC_FORBIDDEN,
containsString(
"unauthorized for user [test-admin] run as [alias_user2] with effective roles [alias_user2] on indices [index-user2]"
)
);
expectThrowsResponseException(
() -> runESQLCommand("metadata1_read2", "from index-user1 | stats sum(value)"),
HttpStatus.SC_FORBIDDEN,
containsString(
"unauthorized for user [test-admin] run as [metadata1_read2] with effective roles [metadata1_read2] on indices [index-user1]"
)
);
}

public void testInsufficientPrivilege() {
ResponseException error = expectThrows(
ResponseException.class,
() -> runESQLCommand("metadata1_read2", "FROM index-user1 | STATS sum=sum(value)")
expectThrowsResponseException(
() -> runESQLCommand("metadata1_read2", "FROM index-user1 | STATS sum=sum(value)"),
HttpStatus.SC_FORBIDDEN,
containsString(unauthorizedErrorMessage("metadata1_read2", "index-user1"))
);
logger.info("error", error);
assertThat(error.getMessage(), containsString("Unknown index [index-user1]"));
assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
}

public void testIndexPatternErrorMessageComparison_ESQL_SearchDSL() throws Exception {
Expand Down Expand Up @@ -917,7 +924,7 @@ public void testLookupJoinIndexForbidden() throws Exception {
testLookupJoinIndexForbiddenHelper(true);
}

private void testLookupJoinIndexForbiddenHelper(boolean useExpressionJoin) throws Exception {
private void testLookupJoinIndexForbiddenHelper(boolean useExpressionJoin) {
assumeTrue(
"Requires LOOKUP JOIN capability",
hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
Expand All @@ -926,44 +933,56 @@ private void testLookupJoinIndexForbiddenHelper(boolean useExpressionJoin) throw
String query1 = useExpressionJoin
? "FROM lookup-user2 | EVAL value_left = 10.0 | LOOKUP JOIN lookup-user1 ON value_left == value | KEEP x"
: "FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN lookup-user1 ON value | KEEP x";
var resp = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", query1));
assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
expectThrowsResponseException(
() -> runESQLCommand("metadata1_read2", query1),
HttpStatus.SC_FORBIDDEN,
containsString(unauthorizedErrorMessage("metadata1_read2", "lookup-user1"))
);

String query2 = useExpressionJoin
? "FROM lookup-user2 | EVAL value_left = 10.0 | LOOKUP JOIN lookup-first-alias ON value_left == value | KEEP x"
: "FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN lookup-first-alias ON value | KEEP x";
resp = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", query2));
assertThat(resp.getMessage(), containsString("Unknown index [lookup-first-alias]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
expectThrowsResponseException(
() -> runESQLCommand("metadata1_read2", query2),
HttpStatus.SC_FORBIDDEN,
containsString(unauthorizedErrorMessage("metadata1_read2", "lookup-first-alias"))
);

String query3 = useExpressionJoin
? "ROW x = 10.0 | EVAL value_left = x | LOOKUP JOIN lookup-user1 ON value_left == value | KEEP x"
: "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN lookup-user1 ON value | KEEP x";
resp = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", query3));
assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
expectThrowsResponseException(
() -> runESQLCommand("metadata1_read2", query3),
HttpStatus.SC_FORBIDDEN,
containsString(unauthorizedErrorMessage("metadata1_read2", "lookup-user1"))
);

String query4 = useExpressionJoin
? "ROW x = 10.0 | EVAL value_left = x | LOOKUP JOIN lookup-user1 ON value_left == value | KEEP x"
: "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN lookup-user1 ON value | KEEP x";
resp = expectThrows(ResponseException.class, () -> runESQLCommand("alias_user1", query4));
assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
expectThrowsResponseException(
() -> runESQLCommand("alias_user1", query4),
HttpStatus.SC_FORBIDDEN,
containsString(unauthorizedErrorMessage("alias_user1", "lookup-user1"))
);
}

public void testFromLookupIndexForbidden() throws Exception {
var resp = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", "FROM lookup-user1"));
assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));

resp = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", "FROM lookup-first-alias"));
assertThat(resp.getMessage(), containsString("Unknown index [lookup-first-alias]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));

resp = expectThrows(ResponseException.class, () -> runESQLCommand("alias_user1", "FROM lookup-user1"));
assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
public void testFromLookupIndexForbidden() {
expectThrowsResponseException(
() -> runESQLCommand("metadata1_read2", "FROM lookup-user1"),
HttpStatus.SC_FORBIDDEN,
containsString(unauthorizedErrorMessage("metadata1_read2", "lookup-user1"))
);
expectThrowsResponseException(
() -> runESQLCommand("metadata1_read2", "FROM lookup-first-alias"),
HttpStatus.SC_FORBIDDEN,
containsString(unauthorizedErrorMessage("metadata1_read2", "lookup-first-alias"))
);
expectThrowsResponseException(
() -> runESQLCommand("alias_user1", "FROM lookup-user1"),
HttpStatus.SC_FORBIDDEN,
containsString(unauthorizedErrorMessage("alias_user1", "lookup-user1"))
);
}

public void testListQueryAllowed() throws Exception {
Expand Down Expand Up @@ -1215,4 +1234,14 @@ private void createDataStreamAlias() throws IOException {
}""");
assertMap(entityAsMap(client().performRequest(request)), matchesMap().extraOk().entry("errors", false));
}

private static void expectThrowsResponseException(ThrowingRunnable runnable, int status, Matcher<String> messageMatcher) {
var exception = expectThrows(ResponseException.class, runnable);
assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(status));
assertThat(exception.getMessage(), messageMatcher);
}

private static String unauthorizedErrorMessage(String user, String index) {
return String.format("unauthorized for user [test-admin] run as [%s] with effective roles [%s] on indices [%s]", user, user, index);
}
}
28 changes: 10 additions & 18 deletions ...server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ public abstract class EsqlRestValidationTestCase extends ESRestTestCase {
aliasName + "*,inexistent*",
"inexistent*," + aliasName };
private static final String[] existentAliasWithoutWildcard = new String[] { aliasName + ",inexistent", "inexistent," + aliasName };
private static final String[] inexistentIndexNameWithWildcard = new String[] { "inexistent*", "inexistent1*,inexistent2*" };
private static final String[] inexistentIndexNameWithoutWildcard = new String[] { "inexistent", "inexistent1,inexistent2" };
private static final String createAlias = "{\"actions\":[{\"add\":{\"index\":\"" + indexName + "\",\"alias\":\"" + aliasName + "\"}}]}";
private static final String removeAlias = "{\"actions\":[{\"remove\":{\"index\":\""
Expand Down Expand Up @@ -73,21 +72,20 @@ public void wipeTestData() throws IOException {
}
}

private String getInexistentIndexErrorMessage() {
return "\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index ";
}

public void testInexistentIndexNameWithWildcard() throws IOException {
assertErrorMessages(inexistentIndexNameWithWildcard, getInexistentIndexErrorMessage(), 400);
}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now wildcards are allowed to expand to empty result set


public void testInexistentIndexNameWithoutWildcard() throws IOException {
assertErrorMessages(inexistentIndexNameWithoutWildcard, getInexistentIndexErrorMessage(), 400);
for (String indexName : inexistentIndexNameWithoutWildcard) {
// TODO ES-13095 all from indexName should be reported as missing.
assertErrorMessage(
indexName,
"\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index [" + clusterSpecificIndexName("inexistent"),
400
);
}
}

public void testExistentIndexWithoutWildcard() throws IOException {
for (String indexName : existentIndexWithoutWildcard) {
assertErrorMessage(indexName, "\"reason\" : \"no such index [inexistent]\"", 404);
assertErrorMessage(indexName, "\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index [inexistent]\"", 400);
}
}

Expand All @@ -99,19 +97,13 @@ public void testAlias() throws IOException {
createAlias();

for (String indexName : existentAliasWithoutWildcard) {
assertErrorMessage(indexName, "\"reason\" : \"no such index [inexistent]\"", 404);
assertErrorMessage(indexName, "\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index [inexistent]\"", 400);
}
assertValidRequestOnIndices(existentAliasWithWildcard);

deleteAlias();
}

private void assertErrorMessages(String[] indices, String errorMessage, int statusCode) throws IOException {
for (String indexName : indices) {
assertErrorMessage(indexName, errorMessage + "[" + clusterSpecificIndexName(indexName) + "]", statusCode);
}
}

protected String clusterSpecificIndexName(String indexName) {
return indexName;
}
Expand Down
24 changes: 4 additions & 20 deletions ...ver/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,15 +85,11 @@ public void testTimestampFilterFromQuery() throws IOException {
allOf(instanceOf(List.class), hasSize(docsTest1))
);

// filter excludes both indices (no rows); the first analysis step fails because there are no columns, a second attempt succeeds
// after eliminating the index filter. All columns are returned.
// filter excludes both indices (no rows); Empty result set is derived.
builder = timestampFilter("gte", "2025-01-01").query(from("test*"));
assertQueryResult(
runEsql(builder),
matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
.item(matchesMap().entry("name", "id1").entry("type", "integer"))
.item(matchesMap().entry("name", "id2").entry("type", "integer"))
.item(matchesMap().entry("name", "value").entry("type", "long")),
matchesList().item(matchesMap().entry("name", "<no-fields>").entry("type", "null")),
allOf(instanceOf(List.class), hasSize(0))
);
}
Expand Down Expand Up @@ -213,22 +209,10 @@ public void testIndicesDontExist() throws IOException {
)
);

e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*"))));
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The same, foo* can now expand to empty result set

e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo,test1")));
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
assertThat(e.getMessage(), containsString("verification_exception"));
assertThat(
e.getMessage(),
anyOf(
containsString("Unknown index [foo*]"),
containsString("Unknown index [*:foo*]"),
containsString("Unknown index [remote_cluster:foo*]")
)
);

e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo, test1")));
assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
assertThat(e.getMessage(), containsString("index_not_found_exception"));
assertThat(e.getMessage(), containsString("no such index [foo]"));
assertThat(e.getMessage(), containsString("Unknown index [foo]"));

// Don't test remote patterns here, we'll test them in the multi-cluster tests
if (EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()) {
Expand Down
Loading