Skip to content

Commit 1e8b289

Browse files
vamsimanoharvamsimanohar
authored
Dynamic source selector (#4116)
Signed-off-by: Vamsi Manohar <[email protected]>
1 parent f244bd6 commit 1e8b289

File tree

4 files changed

+225
-0
lines changed

4 files changed

+225
-0
lines changed

‎ppl/src/main/antlr/OpenSearchPPLParser.g4‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,8 @@ fromClause
423423
| INDEX EQUAL tableOrSubqueryClause
424424
| SOURCE EQUAL tableFunction
425425
| INDEX EQUAL tableFunction
426+
| SOURCE EQUAL dynamicSourceClause
427+
| INDEX EQUAL dynamicSourceClause
426428
;
427429

428430
tableOrSubqueryClause
@@ -434,6 +436,27 @@ tableSourceClause
434436
: tableSource (COMMA tableSource)* (AS alias = qualifiedName)?
435437
;
436438

439+
dynamicSourceClause
440+
: LT_SQR_PRTHS sourceReferences (COMMA sourceFilterArgs)? RT_SQR_PRTHS
441+
;
442+
443+
sourceReferences
444+
: sourceReference (COMMA sourceReference)*
445+
;
446+
447+
sourceReference
448+
: (CLUSTER)? wcQualifiedName
449+
;
450+
451+
sourceFilterArgs
452+
: sourceFilterArg (COMMA sourceFilterArg)*
453+
;
454+
455+
sourceFilterArg
456+
: ident EQUAL literalValue
457+
| ident IN valueList
458+
;
459+
437460
// join
438461
joinCommand
439462
: (joinType) JOIN sideAlias joinHintList? joinCriteria? right = tableOrSubqueryClause

‎ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BinCommandContext;
1515
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DedupCommandContext;
1616
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DescribeCommandContext;
17+
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DynamicSourceClauseContext;
1718
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalCommandContext;
1819
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldsCommandContext;
1920
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.HeadCommandContext;
@@ -820,6 +821,12 @@ public UnresolvedPlan visitTableSourceClause(TableSourceClauseContext ctx) {
820821
: relation;
821822
}
822823

824+
@Override
825+
public UnresolvedPlan visitDynamicSourceClause(DynamicSourceClauseContext ctx) {
826+
throw new UnsupportedOperationException(
827+
"Dynamic source clause with metadata filters is not supported.");
828+
}
829+
823830
@Override
824831
public UnresolvedPlan visitTableFunction(TableFunctionContext ctx) {
825832
ImmutableList.Builder<UnresolvedExpression> builder = ImmutableList.builder();

‎ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,23 @@
55

66
package org.opensearch.sql.ppl.antlr;
77

8+
import static org.junit.Assert.assertEquals;
89
import static org.junit.Assert.assertNotEquals;
910
import static org.junit.Assert.assertNotNull;
1011
import static org.junit.Assert.assertThrows;
12+
import static org.junit.Assert.assertTrue;
1113

1214
import java.util.List;
15+
import org.antlr.v4.runtime.CommonTokenStream;
1316
import org.antlr.v4.runtime.tree.ParseTree;
1417
import org.hamcrest.text.StringContainsInOrder;
1518
import org.junit.Rule;
1619
import org.junit.Test;
1720
import org.junit.rules.ExpectedException;
21+
import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream;
1822
import org.opensearch.sql.common.antlr.SyntaxCheckException;
23+
import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer;
24+
import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser;
1925

2026
public class PPLSyntaxParserTest {
2127

@@ -93,6 +99,184 @@ public void testSearchFieldsCommandCrossClusterShouldPass() {
9399
assertNotEquals(null, tree);
94100
}
95101

102+
@Test
103+
public void testDynamicSourceClauseParseTreeStructure() {
104+
String query = "source=[myindex, logs, fieldIndex=\"test\", count=100]";
105+
OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query));
106+
OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer));
107+
108+
OpenSearchPPLParser.RootContext root = parser.root();
109+
assertNotNull(root);
110+
111+
// Navigate to dynamic source clause
112+
OpenSearchPPLParser.SearchFromContext searchFrom =
113+
(OpenSearchPPLParser.SearchFromContext)
114+
root.pplStatement().queryStatement().pplCommands().searchCommand();
115+
OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource =
116+
searchFrom.fromClause().dynamicSourceClause();
117+
118+
// Verify source references size and text
119+
assertEquals(
120+
"Should have 2 source references",
121+
2,
122+
dynamicSource.sourceReferences().sourceReference().size());
123+
assertEquals(
124+
"Source references text should match",
125+
"myindex,logs",
126+
dynamicSource.sourceReferences().getText());
127+
128+
// Verify filter args size and text
129+
assertEquals(
130+
"Should have 2 filter args", 2, dynamicSource.sourceFilterArgs().sourceFilterArg().size());
131+
assertEquals(
132+
"Filter args text should match",
133+
"fieldIndex=\"test\",count=100",
134+
dynamicSource.sourceFilterArgs().getText());
135+
}
136+
137+
@Test
138+
public void testDynamicSourceWithComplexFilters() {
139+
String query =
140+
"source=[vpc.flow_logs, api.gateway, region=\"us-east-1\", env=\"prod\", count=5]";
141+
OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query));
142+
OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer));
143+
144+
OpenSearchPPLParser.RootContext root = parser.root();
145+
OpenSearchPPLParser.SearchFromContext searchFrom =
146+
(OpenSearchPPLParser.SearchFromContext)
147+
root.pplStatement().queryStatement().pplCommands().searchCommand();
148+
OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource =
149+
searchFrom.fromClause().dynamicSourceClause();
150+
151+
// Verify source references
152+
assertEquals(
153+
"Should have 2 source references",
154+
2,
155+
dynamicSource.sourceReferences().sourceReference().size());
156+
assertEquals(
157+
"Source references text",
158+
"vpc.flow_logs,api.gateway",
159+
dynamicSource.sourceReferences().getText());
160+
161+
// Verify filter args
162+
assertEquals(
163+
"Should have 3 filter args", 3, dynamicSource.sourceFilterArgs().sourceFilterArg().size());
164+
assertEquals(
165+
"Filter args text",
166+
"region=\"us-east-1\",env=\"prod\",count=5",
167+
dynamicSource.sourceFilterArgs().getText());
168+
}
169+
170+
@Test
171+
public void testDynamicSourceWithSingleSource() {
172+
String query = "source=[ds:myindex, fieldIndex=\"test\"]";
173+
OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query));
174+
OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer));
175+
176+
OpenSearchPPLParser.RootContext root = parser.root();
177+
OpenSearchPPLParser.SearchFromContext searchFrom =
178+
(OpenSearchPPLParser.SearchFromContext)
179+
root.pplStatement().queryStatement().pplCommands().searchCommand();
180+
OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource =
181+
searchFrom.fromClause().dynamicSourceClause();
182+
183+
assertEquals(
184+
"Should have 1 source reference",
185+
1,
186+
dynamicSource.sourceReferences().sourceReference().size());
187+
assertEquals("Source reference text", "ds:myindex", dynamicSource.sourceReferences().getText());
188+
189+
assertEquals(
190+
"Should have 1 filter arg", 1, dynamicSource.sourceFilterArgs().sourceFilterArg().size());
191+
assertEquals(
192+
"Filter arg text", "fieldIndex=\"test\"", dynamicSource.sourceFilterArgs().getText());
193+
}
194+
195+
@Test
196+
public void testDynamicSourceRequiresAtLeastOneSource() {
197+
// The grammar requires at least one source reference before optional filter args
198+
// This test documents that behavior
199+
exceptionRule.expect(RuntimeException.class);
200+
new PPLSyntaxParser().parse("source=[fieldIndex=\"httpStatus\", region=\"us-west-2\"]");
201+
}
202+
203+
@Test
204+
public void testDynamicSourceWithDottedNames() {
205+
String query = "source=[vpc.flow_logs, api.gateway.logs, env=\"prod\"]";
206+
OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query));
207+
OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer));
208+
209+
OpenSearchPPLParser.RootContext root = parser.root();
210+
OpenSearchPPLParser.SearchFromContext searchFrom =
211+
(OpenSearchPPLParser.SearchFromContext)
212+
root.pplStatement().queryStatement().pplCommands().searchCommand();
213+
OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource =
214+
searchFrom.fromClause().dynamicSourceClause();
215+
216+
assertEquals(
217+
"Should have 2 source references",
218+
2,
219+
dynamicSource.sourceReferences().sourceReference().size());
220+
assertEquals(
221+
"Source references text",
222+
"vpc.flow_logs,api.gateway.logs",
223+
dynamicSource.sourceReferences().getText());
224+
225+
assertEquals(
226+
"Should have 1 filter arg", 1, dynamicSource.sourceFilterArgs().sourceFilterArg().size());
227+
assertEquals("Filter arg text", "env=\"prod\"", dynamicSource.sourceFilterArgs().getText());
228+
}
229+
230+
@Test
231+
public void testDynamicSourceWithSimpleFilter() {
232+
String query = "source=[logs, count=100]";
233+
OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query));
234+
OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer));
235+
236+
OpenSearchPPLParser.RootContext root = parser.root();
237+
OpenSearchPPLParser.SearchFromContext searchFrom =
238+
(OpenSearchPPLParser.SearchFromContext)
239+
root.pplStatement().queryStatement().pplCommands().searchCommand();
240+
OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource =
241+
searchFrom.fromClause().dynamicSourceClause();
242+
243+
assertEquals(
244+
"Should have 1 source reference",
245+
1,
246+
dynamicSource.sourceReferences().sourceReference().size());
247+
assertEquals("Source reference text", "logs", dynamicSource.sourceReferences().getText());
248+
249+
assertEquals(
250+
"Should have 1 filter arg", 1, dynamicSource.sourceFilterArgs().sourceFilterArg().size());
251+
assertEquals("Filter arg text", "count=100", dynamicSource.sourceFilterArgs().getText());
252+
}
253+
254+
@Test
255+
public void testDynamicSourceWithInClause() {
256+
// Note: The valueList rule expects literalValue which includes strings
257+
String query = "source=[myindex, fieldIndex IN (\"httpStatus\", \"requestId\")]";
258+
OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query));
259+
OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer));
260+
261+
OpenSearchPPLParser.RootContext root = parser.root();
262+
assertNotNull("Query should parse successfully", root);
263+
264+
// Verify IN clause is parsed
265+
OpenSearchPPLParser.SearchFromContext searchFrom =
266+
(OpenSearchPPLParser.SearchFromContext)
267+
root.pplStatement().queryStatement().pplCommands().searchCommand();
268+
OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource =
269+
searchFrom.fromClause().dynamicSourceClause();
270+
271+
assertNotNull("Dynamic source should exist", dynamicSource);
272+
assertNotNull("Filter args should exist", dynamicSource.sourceFilterArgs());
273+
274+
// The IN clause should be parsed as a sourceFilterArg
275+
assertTrue(
276+
"Should have at least one filter arg with IN clause",
277+
dynamicSource.sourceFilterArgs().sourceFilterArg().size() >= 1);
278+
}
279+
96280
@Test
97281
public void testSearchCommandWithoutSourceShouldFail() {
98282
exceptionRule.expect(RuntimeException.class);

‎ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,17 @@ public class AstBuilderTest {
8787

8888
private final Settings settings = Mockito.mock(Settings.class);
8989

90+
@Test
91+
public void testDynamicSourceClauseThrowsUnsupportedException() {
92+
String query = "source=[myindex, logs, fieldIndex=\"test\"]";
93+
94+
UnsupportedOperationException exception =
95+
assertThrows(UnsupportedOperationException.class, () -> plan(query));
96+
97+
assertEquals(
98+
"Dynamic source clause with metadata filters is not supported.", exception.getMessage());
99+
}
100+
90101
@Test
91102
public void testSearchCommand() {
92103
assertEqual(

0 commit comments

Comments
 (0)