[ES|QL] Add automated script to sync operators and add support for MATCH operators

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.where.test.ts

@@ -73,8 +73,8 @@ describe('WHERE <expression>', () => {
       const expectedComparisonWithDateSuggestions = [
         ...getDateLiterals(),
         ...getFieldNamesByType(['date']),
-        // all functions compatible with a keywordField type
-        ...getFunctionSignaturesByReturnType('where', ['date'], { scalar: true }),
+        ...getFieldNamesByType(['date_nanos']),
+        ...getFunctionSignaturesByReturnType('where', ['date', 'date_nanos'], { scalar: true }),
       ];
       await assertSuggestions(
         'from a | where dateField == /',
@@ -212,10 +212,18 @@ describe('WHERE <expression>', () => {
       ]);
       await assertSuggestions('from index | WHERE not /', [
         ...getFieldNamesByType('boolean').map((name) => attachTriggerCommand(`${name} `)),
-        ...getFunctionSignaturesByReturnType('where', 'boolean', { scalar: true }),
+        ...getFunctionSignaturesByReturnType('where', 'boolean', { scalar: true }, undefined, [
+          ':',
+        ]),
       ]);
       await assertSuggestions('FROM index | WHERE NOT ENDS_WITH(keywordField, "foo") /', [
-        ...getFunctionSignaturesByReturnType('where', 'boolean', { builtin: true }, ['boolean']),
+        ...getFunctionSignaturesByReturnType(
+          'where',
+          'boolean',
+          { builtin: true },
+          ['boolean'],
+          [':']
+        ),
         pipeCompleteItem,
       ]);
       await assertSuggestions('from index | WHERE keywordField IS NOT/', [
@@ -290,9 +298,13 @@ describe('WHERE <expression>', () => {
       const { assertSuggestions } = await setup();
 
       await assertSuggestions('FROM index | WHERE doubleField + doubleField /', [
-        ...getFunctionSignaturesByReturnType('where', 'any', { builtin: true, skipAssign: true }, [
-          'double',
-        ]),
+        ...getFunctionSignaturesByReturnType(
+          'where',
+          'any',
+          { builtin: true, skipAssign: true },
+          ['double'],
+          [':']
+        ),
       ]);
     });
 

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts

@@ -147,25 +147,37 @@ describe('autocomplete.suggest', () => {
         '| ',
         ...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true, skipAssign: true }, [
           'double',
+          'long',
         ]),
+        'IN $0',
+        'IS NOT NULL',
+        'IS NULL',
       ]);
       await assertSuggestions(
         'from a | eval a=round(doubleField, /',
         [
-          ...getFieldNamesByType('integer'),
-          ...getFunctionSignaturesByReturnType('eval', 'integer', { scalar: true }, undefined, [
-            'round',
-          ]),
+          ...getFieldNamesByType(['integer', 'long']),
+          ...getFunctionSignaturesByReturnType(
+            'eval',
+            ['integer', 'long'],
+            { scalar: true },
+            undefined,
+            ['round']
+          ),
         ],
         { triggerCharacter: '(' }
       );
       await assertSuggestions(
         'from a | eval round(doubleField, /',
         [
-          ...getFieldNamesByType('integer'),
-          ...getFunctionSignaturesByReturnType('eval', 'integer', { scalar: true }, undefined, [
-            'round',
-          ]),
+          ...getFieldNamesByType(['integer', 'long']),
+          ...getFunctionSignaturesByReturnType(
+            'eval',
+            ['integer', 'long'],
+            { scalar: true },
+            undefined,
+            ['round']
+          ),
         ],
         { triggerCharacter: '(' }
       );

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts

@@ -472,11 +472,13 @@ describe('autocomplete', () => {
       const expectedDateDiff2ndArgSuggestions = [
         TIME_PICKER_SUGGESTION,
         ...TIME_SYSTEM_PARAMS.map((t) => `${t}, `),
-        ...getFieldNamesByType('date').map((name) => `${name}, `),
-        ...getFunctionSignaturesByReturnType('eval', 'date', { scalar: true }).map((s) => ({
-          ...s,
-          text: `${s.text},`,
-        })),
+        ...getFieldNamesByType(['date', 'date_nanos']).map((name) => `${name}, `),
+        ...getFunctionSignaturesByReturnType('eval', ['date', 'date_nanos'], { scalar: true }).map(
+          (s) => ({
+            ...s,
+            text: `${s.text},`,
+          })
+        ),
       ];
       testSuggestions('FROM a | EVAL DATE_DIFF("day", /)', expectedDateDiff2ndArgSuggestions);
 

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts

@@ -81,7 +81,7 @@ import {
   getOperatorSuggestions,
   getSuggestionsAfterNot,
 } from './factories';
-import { EDITOR_MARKER, METADATA_FIELDS } from '../shared/constants';
+import { EDITOR_MARKER, FULL_TEXT_SEARCH_FUNCTIONS, METADATA_FIELDS } from '../shared/constants';
 import { getAstContext, removeMarkerArgFromArgsList } from '../shared/context';
 import {
   buildQueryUntilPreviousCommand,
@@ -1209,10 +1209,12 @@ async function getFunctionArgsSuggestions(
         }))
       );
     }
+
     // could also be in stats (bucket) but our autocomplete is not great yet
     if (
       (getTypesFromParamDefs(typesToSuggestNext).includes('date') &&
-        ['where', 'eval'].includes(command.name)) ||
+        ['where', 'eval'].includes(command.name) &&
+        !FULL_TEXT_SEARCH_FUNCTIONS.includes(fnDefinition.name)) ||
       (command.name === 'stats' &&
         typesToSuggestNext.some((t) => t && t.type === 'date' && t.constantOnly === true))
     )

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/where/index.ts

@@ -29,6 +29,7 @@ import { getOverlapRange, getSuggestionsToRightOfOperatorExpression } from '../.
 import { getPosition } from './util';
 import { pipeCompleteItem } from '../../complete_items';
 import {
+  EDITOR_MARKER,
   UNSUPPORTED_COMMANDS_BEFORE_MATCH,
   UNSUPPORTED_COMMANDS_BEFORE_QSTR,
 } from '../../../shared/constants';
@@ -169,12 +170,14 @@ export async function suggest(
       if (priorCommands.some((c) => UNSUPPORTED_COMMANDS_BEFORE_QSTR.has(c))) {
         ignored.push('qstr');
       }
-
-      const columnSuggestions = await getColumnsByType('any', [], {
-        advanceCursor: true,
-        openSuggestions: true,
-      });
-
+      const last = fullTextAst?.[fullTextAst.length - 1];
+      let columnSuggestions: SuggestionRawDefinition[] = [];
+      if (!last?.text?.endsWith(`:${EDITOR_MARKER}`)) {
+        columnSuggestions = await getColumnsByType('any', [], {
+          advanceCursor: true,
+          openSuggestions: true,
+        });
+      }
       suggestions.push(
         ...columnSuggestions,
         ...getFunctionSuggestions({ command: 'where', ignored })

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts

@@ -571,7 +571,7 @@ export async function getSuggestionsToRightOfOperatorExpression({
           operatorReturnType === 'unknown' || operatorReturnType === 'unsupported'
             ? 'any'
             : operatorReturnType,
-        ignored: ['='],
+        ignored: ['=', ':'],
       })
     );
   } else {

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/builtin.ts

@@ -8,9 +8,9 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { ESQL_NUMBER_TYPES, isNumericType } from '../shared/esql_types';
+import { isNumericType } from '../shared/esql_types';
 import type { FunctionDefinition, FunctionParameterType, FunctionReturnType } from './types';
-
+import { operatorsFunctionDefinitions } from './generated/operators';
 type MathFunctionSignature = [FunctionParameterType, FunctionParameterType, FunctionReturnType];
 
 function createMathDefinition(
@@ -384,150 +384,6 @@ export const comparisonFunctions: FunctionDefinition[] = [
   },
 ].map((op): FunctionDefinition => createComparisonDefinition(op));
 
-const likeFunctions: FunctionDefinition[] = [
-  // Skip the insensitive case equality until it gets restored back
-  // new special comparison operator for strings only
-  // {
-  //   name: '=~',
-  //   description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.equalToCaseInsensitiveDoc', {
-  //     defaultMessage: 'Case insensitive equality',
-  //   }),
-  // },
-  {
-    name: 'like',
-    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.likeDoc', {
-      defaultMessage: 'Filter data based on string patterns',
-    }),
-  },
-  { name: 'not_like', description: '' },
-  {
-    name: 'rlike',
-    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.rlikeDoc', {
-      defaultMessage: 'Filter data based on string regular expressions',
-    }),
-  },
-  { name: 'not_rlike', description: '' },
-].map(({ name, description }) => {
-  const def: FunctionDefinition = {
-    type: 'builtin' as const,
-    ignoreAsSuggestion: /not/.test(name),
-    name,
-    description,
-    supportedCommands: ['eval', 'where', 'row', 'sort'],
-    supportedOptions: ['by'],
-    signatures: [
-      {
-        params: [
-          { name: 'left', type: 'text' as const },
-          { name: 'right', type: 'text' as const },
-        ],
-        returnType: 'boolean',
-      },
-      {
-        params: [
-          { name: 'left', type: 'text' as const },
-          { name: 'right', type: 'keyword' as const },
-        ],
-        returnType: 'boolean',
-      },
-      {
-        params: [
-          { name: 'left', type: 'keyword' as const },
-          { name: 'right', type: 'text' as const },
-        ],
-        returnType: 'boolean',
-      },
-      {
-        params: [
-          { name: 'left', type: 'keyword' as const },
-          { name: 'right', type: 'keyword' as const },
-        ],
-        returnType: 'boolean',
-      },
-    ],
-  };
-
-  return def;
-});
-
-const inFunctions: FunctionDefinition[] = [
-  {
-    name: 'in',
-    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.inDoc', {
-      defaultMessage:
-        'Tests if the value an expression takes is contained in a list of other expressions',
-    }),
-  },
-  { name: 'not_in', description: '' },
-].map<FunctionDefinition>(({ name, description }) => ({
-  // set all arrays to type "any" for now
-  // this only applies to the "in" operator
-  // e.g. "foo" in ( "foo", "bar" )
-  //
-  // we did this because the "in" operator now supports
-  // mixed-type arrays like ( "1.2.3", versionVar )
-  // because of implicit casting.
-  //
-  // we need to revisit with more robust validation
-  type: 'builtin',
-  ignoreAsSuggestion: /not/.test(name),
-  name,
-  description,
-  supportedCommands: ['eval', 'where', 'row', 'sort'],
-  signatures: [
-    ...ESQL_NUMBER_TYPES.map((type) => ({
-      params: [
-        { name: 'left', type: type as FunctionParameterType },
-
-        { name: 'right', type: 'any[]' as FunctionParameterType },
-      ],
-      returnType: 'boolean' as FunctionReturnType,
-    })),
-    {
-      params: [
-        { name: 'left', type: 'keyword' },
-        { name: 'right', type: 'any[]' },
-      ],
-      returnType: 'boolean',
-    },
-    {
-      params: [
-        { name: 'left', type: 'text' },
-        { name: 'right', type: 'any[]' },
-      ],
-      returnType: 'boolean',
-    },
-    {
-      params: [
-        { name: 'left', type: 'boolean' },
-        { name: 'right', type: 'any[]' },
-      ],
-      returnType: 'boolean',
-    },
-    {
-      params: [
-        { name: 'left', type: 'date' },
-        { name: 'right', type: 'any[]' },
-      ],
-      returnType: 'boolean',
-    },
-    {
-      params: [
-        { name: 'left', type: 'version' },
-        { name: 'right', type: 'any[]' },
-      ],
-      returnType: 'boolean',
-    },
-    {
-      params: [
-        { name: 'left', type: 'ip' },
-        { name: 'right', type: 'any[]' },
-      ],
-      returnType: 'boolean',
-    },
-  ],
-}));
-
 export const logicalOperators: FunctionDefinition[] = [
   {
     name: 'and',
@@ -681,10 +537,7 @@ const otherDefinitions: FunctionDefinition[] = [
 ];
 
 export const builtinFunctions: FunctionDefinition[] = [
-  ...mathFunctions,
-  ...comparisonFunctions,
-  ...likeFunctions,
-  ...inFunctions,
+  ...operatorsFunctionDefinitions,
   ...logicalOperators,
   ...nullFunctions,
   ...otherDefinitions,