[8.18] Update lexer / parser and fix the categorize function (#209964) (#210032)

# Backport

This will backport the following commits from `main` to `8.18`:
- [Update lexer / parser and fix the categorize function
(#209964)](#209964)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Stratoula
Kalafateli","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-02-06T14:57:02Z","message":"Update
lexer / parser and fix the categorize function (#209964)\n\n##
Summary\n\n- Fixes the categorize wrong autocomplete suggestions\n-
Updates the lexer / parser as I want this to get merged in 8.18 /
9.0","sha":"ee866a745a23efd7442262232f149080fa4de609","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Feature:ES|QL","Team:ESQL","backport:version","v8.18.0","v9.1.0","v8.19.0"],"title":"Update
lexer / parser and fix the categorize
function","number":209964,"url":"https://github.com/elastic/kibana/pull/209964","mergeCommit":{"message":"Update
lexer / parser and fix the categorize function (#209964)\n\n##
Summary\n\n- Fixes the categorize wrong autocomplete suggestions\n-
Updates the lexer / parser as I want this to get merged in 8.18 /
9.0","sha":"ee866a745a23efd7442262232f149080fa4de609"}},"sourceBranch":"main","suggestedTargetBranches":["9.0","8.18","8.x"],"targetPullRequestStates":[{"branch":"9.0","label":"v9.0.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.18","label":"v8.18.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/209964","number":209964,"mergeCommit":{"message":"Update
lexer / parser and fix the categorize function (#209964)\n\n##
Summary\n\n- Fixes the categorize wrong autocomplete suggestions\n-
Updates the lexer / parser as I want this to get merged in 8.18 /
9.0","sha":"ee866a745a23efd7442262232f149080fa4de609"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Stratoula Kalafateli <[email protected]>

Author: kibanamachine

src/platform/packages/shared/kbn-esql-ast/src/antlr/esql_lexer.g4

@@ -1,5 +1,3 @@
-// DO NOT MODIFY THIS FILE BY HAND. IT IS MANAGED BY A CI JOB.
-
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
@@ -316,8 +314,8 @@ mode PROJECT_MODE;
 PROJECT_PIPE : PIPE -> type(PIPE), popMode;
 PROJECT_DOT: DOT -> type(DOT);
 PROJECT_COMMA : COMMA -> type(COMMA);
-PROJECT_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM);
-PROJECT_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
+PROJECT_PARAM : PARAM -> type(PARAM);
+PROJECT_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
 
 fragment UNQUOTED_ID_BODY_WITH_PATTERN
     : (LETTER | DIGIT | UNDERSCORE | ASTERISK)
@@ -351,8 +349,8 @@ RENAME_PIPE : PIPE -> type(PIPE), popMode;
 RENAME_ASSIGN : ASSIGN -> type(ASSIGN);
 RENAME_COMMA : COMMA -> type(COMMA);
 RENAME_DOT: DOT -> type(DOT);
-RENAME_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM);
-RENAME_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
+RENAME_PARAM : PARAM -> type(PARAM);
+RENAME_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
 
 AS : 'as';
 
@@ -424,8 +422,8 @@ ENRICH_FIELD_QUOTED_IDENTIFIER
     : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER)
     ;
 
-ENRICH_FIELD_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM);
-ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
+ENRICH_FIELD_PARAM : PARAM -> type(PARAM);
+ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
 
 ENRICH_FIELD_LINE_COMMENT
     : LINE_COMMENT -> channel(HIDDEN)
@@ -442,8 +440,8 @@ ENRICH_FIELD_WS
 mode MVEXPAND_MODE;
 MVEXPAND_PIPE : PIPE -> type(PIPE), popMode;
 MVEXPAND_DOT: DOT -> type(DOT);
-MVEXPAND_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM);
-MVEXPAND_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
+MVEXPAND_PARAM : PARAM -> type(PARAM);
+MVEXPAND_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
 
 MVEXPAND_QUOTED_IDENTIFIER
     : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER)

src/platform/packages/shared/kbn-esql-ast/src/antlr/esql_lexer.interp

@@ -195,7 +195,7 @@ identifier
 
 identifierPattern
     : ID_PATTERN
-    | {this.isDevVersion()}? parameter
+    | parameter
     ;
 
 constant
@@ -218,7 +218,7 @@ parameter
 
 identifierOrParameter
     : identifier
-    | {this.isDevVersion()}? parameter
+    | parameter
     ;
 
 limitCommand