[ES|QL] Support subqueries in AST (#241227)

## Summary

**Grammar**
```
fromCommand
    : FROM indexPatternOrSubquery (COMMA indexPatternOrSubquery)*
    ;
```

```
indexPatternOrSubquery
    : indexPattern
    | subquery
    ;
```

```
subquery
    : LP fromCommand (PIPE processingCommand)* RP
    ;
```

---------

Co-authored-by: Vadim Kibana <[email protected]>

Author: bartoval

src/platform/packages/shared/kbn-esql-ast/src/ast/is.ts

@@ -93,6 +93,14 @@ export const isColumn = (node: unknown): node is types.ESQLColumn =>
 export const isSource = (node: unknown): node is types.ESQLSource =>
   isProperNode(node) && node.type === 'source';
 
+export const isParens = (node: unknown): node is types.ESQLParens =>
+  isProperNode(node) && node.type === 'parens';
+
+export const isSubQuery = (
+  node: unknown
+): node is types.ESQLParens & { child: types.ESQLAstQueryExpression } =>
+  isParens(node) && isQuery(node.child);
+
 export const isMap = (node: unknown): node is types.ESQLMap =>
   isProperNode(node) && node.type === 'map';
 

src/platform/packages/shared/kbn-esql-ast/src/builder/builder.ts

@@ -16,6 +16,7 @@ import type {
   ESQLAstComment,
   ESQLAstCommentMultiLine,
   ESQLAstCommentSingleLine,
+  ESQLAstExpression,
   ESQLAstQueryExpression,
   ESQLColumn,
   ESQLCommand,
@@ -28,6 +29,7 @@ import type {
   ESQLLocation,
   ESQLNamedParamLiteral,
   ESQLParam,
+  ESQLParens,
   ESQLPositionalParamLiteral,
   ESQLOrderExpression,
   ESQLSource,
@@ -185,6 +187,18 @@ export namespace Builder {
       };
     }
 
+    export const parens = (
+      child: ESQLAstExpression,
+      fromParser?: Partial<AstNodeParserFields>
+    ): ESQLParens => {
+      return {
+        type: 'parens',
+        name: '',
+        child,
+        ...Builder.parserFields(fromParser),
+      };
+    };
+
     export type ColumnTemplate = Omit<AstNodeTemplate<ESQLColumn>, 'name' | 'quoted' | 'parts'>;
 
     export const column = (

src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/from.test.ts

@@ -8,6 +8,8 @@
  */
 
 import { parse } from '..';
+import type { ESQLAstQueryExpression, ESQLParens } from '../../types';
+import { isParens, isSubQuery } from '../../ast/is';
 
 describe('FROM', () => {
   describe('correctly formatted', () => {
@@ -377,4 +379,125 @@ describe('FROM', () => {
       expect(errors.length > 0).toBe(true);
     });
   });
+
+  describe('subqueries', () => {
+    it('can parse simple subquery', () => {
+      const text = 'FROM index1, (FROM index2 | WHERE a > 10)';
+      const { ast, errors } = parse(text);
+
+      expect(errors.length).toBe(0);
+      expect(ast).toMatchObject([
+        {
+          type: 'command',
+          name: 'from',
+          args: [
+            {
+              type: 'source',
+              name: 'index1',
+            },
+            {
+              type: 'parens',
+              child: {
+                type: 'query',
+                commands: [
+                  { type: 'command', name: 'from' },
+                  { type: 'command', name: 'where' },
+                ],
+              },
+            },
+          ],
+        },
+      ]);
+    });
+
+    it('can parse subquery as only source', () => {
+      const text = 'FROM (FROM index1 | WHERE x > 0)';
+      const { ast, errors } = parse(text);
+
+      expect(errors.length).toBe(0);
+      expect(ast[0].args).toHaveLength(1);
+      expect(ast[0].args[0]).toMatchObject({
+        type: 'parens',
+        child: {
+          type: 'query',
+          commands: [{ name: 'from' }, { name: 'where' }],
+        },
+      });
+    });
+
+    it('can parse subquery followed by main query pipes', () => {
+      const text = 'FROM (FROM index1 | WHERE a > 0) | WHERE b < 10 | LIMIT 5';
+      const { ast, errors } = parse(text);
+
+      expect(errors.length).toBe(0);
+      expect(ast).toHaveLength(3);
+      expect(ast[0].name).toBe('from');
+      expect(ast[1].name).toBe('where');
+      expect(ast[2].name).toBe('limit');
+    });
+
+    it('correctly captures location for deeply nested subqueries (3 levels)', () => {
+      const text = 'FROM (FROM (FROM (FROM index | WHERE a > 0) | WHERE b < 10) | LIMIT 5)';
+      const { ast } = parse(text);
+
+      // Level 1 - outermost
+      const level1Parens = ast[0].args[0] as ESQLParens;
+      expect(text.slice(level1Parens.location.min, level1Parens.location.max + 1)).toBe(
+        '(FROM (FROM (FROM index | WHERE a > 0) | WHERE b < 10) | LIMIT 5)'
+      );
+
+      // Level 2 - middle
+      const level1Query = level1Parens.child as ESQLAstQueryExpression;
+      const level2Parens = level1Query.commands[0].args[0] as ESQLParens;
+      expect(text.slice(level2Parens.location.min, level2Parens.location.max + 1)).toBe(
+        '(FROM (FROM index | WHERE a > 0) | WHERE b < 10)'
+      );
+
+      // Level 3 - innermost
+      const level2Query = level2Parens.child as ESQLAstQueryExpression;
+      const level3Parens = level2Query.commands[0].args[0] as ESQLParens;
+      expect(text.slice(level3Parens.location.min, level3Parens.location.max + 1)).toBe(
+        '(FROM index | WHERE a > 0)'
+      );
+    });
+
+    describe('error cases', () => {
+      it('errors on unclosed subquery and captures location up to end of content', () => {
+        const text = 'FROM (FROM index | WHERE a > 10';
+        const { ast, errors } = parse(text);
+
+        expect(errors.length).toBeGreaterThan(0);
+
+        const parens = ast[0].args[0] as ESQLParens;
+
+        expect(isParens(parens)).toBe(true);
+        expect(isSubQuery(parens)).toBe(true);
+        expect(parens.incomplete).toBe(true);
+
+        const query = parens.child as ESQLAstQueryExpression;
+
+        expect(query.incomplete).toBe(true);
+        expect(query.commands).toHaveLength(2);
+
+        // Verify location captures everything including missing closing paren
+        expect(text.slice(parens.location.min, parens.location.max + 1)).toBe(
+          '(FROM index | WHERE a > 10'
+        );
+      });
+
+      it('errors on empty subquery', () => {
+        const text = 'FROM index, ()';
+        const { errors } = parse(text);
+
+        expect(errors.length).toBeGreaterThan(0);
+      });
+
+      it('errors on subquery without FROM', () => {
+        const text = 'FROM (WHERE a > 10)';
+        const { errors } = parse(text);
+
+        expect(errors.length).toBeGreaterThan(0);
+      });
+    });
+  });
 });

src/platform/packages/shared/kbn-esql-ast/src/parser/cst_to_ast_converter.ts

@@ -539,9 +539,17 @@ export class CstToAstConverter {
     const indexPatternOrSubqueryCtxs = indexPatternAndMetadataCtx.indexPatternOrSubquery_list();
     const sources = indexPatternOrSubqueryCtxs
       .map((indexPatternOrSubqueryCtx) => {
-        // ToDo: handle subqueries when implemented
         const indexPatternCtx = indexPatternOrSubqueryCtx.indexPattern();
-        return indexPatternCtx ? this.toSource(indexPatternCtx) : null;
+        if (indexPatternCtx) {
+          return this.toSource(indexPatternCtx);
+        }
+
+        const subqueryCtx = indexPatternOrSubqueryCtx.subquery();
+        if (subqueryCtx) {
+          return this.fromSubquery(subqueryCtx);
+        }
+
+        return null;
       })
       .filter((source): source is ast.ESQLSource => source !== null);
 
@@ -559,6 +567,49 @@ export class CstToAstConverter {
     return command;
   }
 
+  private fromSubquery(ctx: cst.SubqueryContext): ast.ESQLParens {
+    const fromCommandCtx = ctx.fromCommand();
+    const processingCommandCtxs = ctx.processingCommand_list();
+    const commands: ast.ESQLCommand[] = [];
+
+    if (fromCommandCtx) {
+      const fromCommand = this.fromFromCommand(fromCommandCtx);
+
+      if (fromCommand) {
+        commands.push(fromCommand);
+      }
+    }
+
+    for (const procCmdCtx of processingCommandCtxs) {
+      const procCommand = this.fromProcessingCommand(procCmdCtx);
+
+      if (procCommand) {
+        commands.push(procCommand);
+      }
+    }
+
+    const openParen = ctx.LP();
+    const closeParen = ctx.RP();
+
+    // ANTLR inserts tokens with text like "<missing ')'>" when they're missing
+    const closeParenText = closeParen?.getText() ?? '';
+    const hasCloseParen = closeParen && !/<missing /.test(closeParenText);
+    const incomplete = Boolean(ctx.exception) || !hasCloseParen;
+
+    const query = Builder.expression.query(commands, {
+      ...this.getParserFields(ctx),
+      incomplete,
+    });
+
+    return Builder.expression.parens(query, {
+      incomplete: incomplete || query.incomplete,
+      location: getPosition(
+        openParen?.symbol ?? ctx.start,
+        hasCloseParen ? closeParen.symbol : ctx.stop
+      ),
+    });
+  }
+
   // ---------------------------------------------------------------------- ROW
 
   private fromRowCommand(ctx: cst.RowCommandContext): ast.ESQLCommand<'row'> {

src/platform/packages/shared/kbn-esql-ast/src/types.ts

@@ -34,6 +34,7 @@ export type ESQLSingleAstItem =
   | ESQLFunction
   | ESQLCommandOption
   | ESQLSource
+  | ESQLParens
   | ESQLColumn
   | ESQLDatePeriodLiteral
   | ESQLTimeDurationLiteral
@@ -372,6 +373,18 @@ export interface ESQLSource extends ESQLAstBaseItem {
   selector?: ESQLStringLiteral | undefined;
 }
 
+/**
+ * Represents any expression wrapped in parentheses.
+ *
+ * ```
+ * FROM ( <query> )
+ * ```
+ */
+export interface ESQLParens extends ESQLAstBaseItem {
+  type: 'parens';
+  child: ESQLAstExpression;
+}
+
 export interface ESQLColumn extends ESQLAstBaseItem {
   type: 'column';