Skip to content

Commit 87b0f72

Browse files
nik9000nik9000
authored
ESQL: Opt into extra data stream resolution (#118378)
* ESQL: Opt into extra data stream resolution This opts ESQL's data node request into extra data stream resolution. * Update docs/changelog/118378.yaml
1 parent 5406850 commit 87b0f72

File tree

4 files changed

+193
-0
lines changed

4 files changed

+193
-0
lines changed

‎docs/changelog/118378.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 118378
2+
summary: Opt into extra data stream resolution
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.util.Locale;
3535
import java.util.Map;
3636

37+
import static org.elasticsearch.test.ListMatcher.matchesList;
3738
import static org.elasticsearch.test.MapMatcher.assertMap;
3839
import static org.elasticsearch.test.MapMatcher.matchesMap;
3940
import static org.hamcrest.Matchers.containsString;
@@ -56,6 +57,11 @@ public class EsqlSecurityIT extends ESRestTestCase {
5657
.user("metadata1_read2", "x-pack-test-password", "metadata1_read2", false)
5758
.user("alias_user1", "x-pack-test-password", "alias_user1", false)
5859
.user("alias_user2", "x-pack-test-password", "alias_user2", false)
60+
.user("logs_foo_all", "x-pack-test-password", "logs_foo_all", false)
61+
.user("logs_foo_16_only", "x-pack-test-password", "logs_foo_16_only", false)
62+
.user("logs_foo_after_2021", "x-pack-test-password", "logs_foo_after_2021", false)
63+
.user("logs_foo_after_2021_pattern", "x-pack-test-password", "logs_foo_after_2021_pattern", false)
64+
.user("logs_foo_after_2021_alias", "x-pack-test-password", "logs_foo_after_2021_alias", false)
5965
.build();
6066

6167
@Override
@@ -342,6 +348,14 @@ public void testDocumentLevelSecurity() throws Exception {
342348
assertThat(respMap.get("values"), equalTo(List.of(List.of(10.0))));
343349
}
344350

351+
public void testDocumentLevelSecurityFromStar() throws Exception {
352+
Response resp = runESQLCommand("user3", "from in*x | stats sum=sum(value)");
353+
assertOK(resp);
354+
Map<String, Object> respMap = entityAsMap(resp);
355+
assertThat(respMap.get("columns"), equalTo(List.of(Map.of("name", "sum", "type", "double"))));
356+
assertThat(respMap.get("values"), equalTo(List.of(List.of(10.0))));
357+
}
358+
345359
public void testFieldLevelSecurityAllow() throws Exception {
346360
Response resp = runESQLCommand("fls_user", "FROM index* | SORT value | LIMIT 1");
347361
assertOK(resp);
@@ -545,6 +559,22 @@ private void removeEnrichPolicy() throws Exception {
545559
client().performRequest(new Request("DELETE", "_enrich/policy/songs"));
546560
}
547561

562+
public void testDataStream() throws IOException {
563+
createDataStream();
564+
MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2)));
565+
MapMatcher oneResult = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(1)));
566+
assertMap(entityAsMap(runESQLCommand("logs_foo_all", "FROM logs-foo | STATS COUNT(*)")), twoResults);
567+
assertMap(entityAsMap(runESQLCommand("logs_foo_16_only", "FROM logs-foo | STATS COUNT(*)")), oneResult);
568+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021", "FROM logs-foo | STATS COUNT(*)")), oneResult);
569+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_pattern", "FROM logs-foo | STATS COUNT(*)")), oneResult);
570+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM alias-foo | STATS COUNT(*)")), oneResult);
571+
assertMap(entityAsMap(runESQLCommand("logs_foo_all", "FROM logs-* | STATS COUNT(*)")), twoResults);
572+
assertMap(entityAsMap(runESQLCommand("logs_foo_16_only", "FROM logs-* | STATS COUNT(*)")), oneResult);
573+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021", "FROM logs-* | STATS COUNT(*)")), oneResult);
574+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_pattern", "FROM logs-* | STATS COUNT(*)")), oneResult);
575+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM alias-* | STATS COUNT(*)")), oneResult);
576+
}
577+
548578
protected Response runESQLCommand(String user, String command) throws IOException {
549579
if (command.toLowerCase(Locale.ROOT).contains("limit") == false) {
550580
// add a (high) limit to avoid warnings on default limit
@@ -592,4 +622,103 @@ static Settings randomPragmas() {
592622
}
593623
return settings.build();
594624
}
625+
626+
private void createDataStream() throws IOException {
627+
createDataStreamPolicy();
628+
createDataStreamComponentTemplate();
629+
createDataStreamIndexTemplate();
630+
createDataStreamDocuments();
631+
createDataStreamAlias();
632+
}
633+
634+
private void createDataStreamPolicy() throws IOException {
635+
Request request = new Request("PUT", "_ilm/policy/my-lifecycle-policy");
636+
request.setJsonEntity("""
637+
{
638+
"policy": {
639+
"phases": {
640+
"hot": {
641+
"actions": {
642+
"rollover": {
643+
"max_primary_shard_size": "50gb"
644+
}
645+
}
646+
},
647+
"delete": {
648+
"min_age": "735d",
649+
"actions": {
650+
"delete": {}
651+
}
652+
}
653+
}
654+
}
655+
}""");
656+
client().performRequest(request);
657+
}
658+
659+
private void createDataStreamComponentTemplate() throws IOException {
660+
Request request = new Request("PUT", "_component_template/my-template");
661+
request.setJsonEntity("""
662+
{
663+
"template": {
664+
"settings": {
665+
"index.lifecycle.name": "my-lifecycle-policy"
666+
},
667+
"mappings": {
668+
"properties": {
669+
"@timestamp": {
670+
"type": "date",
671+
"format": "date_optional_time||epoch_millis"
672+
},
673+
"data_stream": {
674+
"properties": {
675+
"namespace": {"type": "keyword"}
676+
}
677+
}
678+
}
679+
}
680+
}
681+
}""");
682+
client().performRequest(request);
683+
}
684+
685+
private void createDataStreamIndexTemplate() throws IOException {
686+
Request request = new Request("PUT", "_index_template/my-index-template");
687+
request.setJsonEntity("""
688+
{
689+
"index_patterns": ["logs-*"],
690+
"data_stream": {},
691+
"composed_of": ["my-template"],
692+
"priority": 500
693+
}""");
694+
client().performRequest(request);
695+
}
696+
697+
private void createDataStreamDocuments() throws IOException {
698+
Request request = new Request("POST", "logs-foo/_bulk");
699+
request.addParameter("refresh", "");
700+
request.setJsonEntity("""
701+
{ "create" : {} }
702+
{ "@timestamp": "2099-05-06T16:21:15.000Z", "data_stream": {"namespace": "16"} }
703+
{ "create" : {} }
704+
{ "@timestamp": "2001-05-06T16:21:15.000Z", "data_stream": {"namespace": "17"} }
705+
""");
706+
assertMap(entityAsMap(client().performRequest(request)), matchesMap().extraOk().entry("errors", false));
707+
}
708+
709+
private void createDataStreamAlias() throws IOException {
710+
Request request = new Request("PUT", "_alias");
711+
request.setJsonEntity("""
712+
{
713+
"actions": [
714+
{
715+
"add": {
716+
"index": "logs-foo",
717+
"alias": "alias-foo"
718+
}
719+
}
720+
]
721+
}""");
722+
assertMap(entityAsMap(client().performRequest(request)), matchesMap().extraOk().entry("errors", false));
723+
}
595724
}

‎x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,57 @@ fls_user:
9292
privileges: [ 'read' ]
9393
field_security:
9494
grant: [ value ]
95+
96+
logs_foo_all:
97+
cluster: []
98+
indices:
99+
- names: [ 'logs-foo' ]
100+
privileges: [ 'read' ]
101+
102+
logs_foo_16_only:
103+
cluster: []
104+
indices:
105+
- names: [ 'logs-foo' ]
106+
privileges: [ 'read' ]
107+
query: |
108+
{
109+
"term": {
110+
"data_stream.namespace": "16"
111+
}
112+
}
113+
114+
logs_foo_after_2021:
115+
cluster: []
116+
indices:
117+
- names: [ 'logs-foo' ]
118+
privileges: [ 'read' ]
119+
query: |
120+
{
121+
"range": {
122+
"@timestamp": {"gte": "2021-01-01T00:00:00"}
123+
}
124+
}
125+
126+
logs_foo_after_2021_pattern:
127+
cluster: []
128+
indices:
129+
- names: [ 'logs-*' ]
130+
privileges: [ 'read' ]
131+
query: |
132+
{
133+
"range": {
134+
"@timestamp": {"gte": "2021-01-01T00:00:00"}
135+
}
136+
}
137+
138+
logs_foo_after_2021_alias:
139+
cluster: []
140+
indices:
141+
- names: [ 'alias-foo' ]
142+
privileges: [ 'read' ]
143+
query: |
144+
{
145+
"range": {
146+
"@timestamp": {"gte": "2021-01-01T00:00:00"}
147+
}
148+
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,11 @@ public IndicesRequest indices(String... indices) {
118118
return this;
119119
}
120120

121+
@Override
122+
public boolean includeDataStreams() {
123+
return true;
124+
}
125+
121126
@Override
122127
public IndicesOptions indicesOptions() {
123128
return indicesOptions;

0 commit comments

Comments
 (0)