[ES|QL] Update function scripts for 8.19 (elastic#219779)

## Summary

Part of elastic#219221

This updates the function metadata sync scripts to work against the
Elasticsearch 8.19 branch.

Review not only the code changes in this PR, but also the result of
running the CI job: elastic#219854

Author: drewdaemon

.buildkite/pipelines/esql_grammar_sync.yml

@@ -1,13 +1,13 @@
 steps:
-  - command: .buildkite/scripts/steps/esql_grammar_sync.sh
-    label: Grammar Sync
-    timeout_in_minutes: 10
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-2
-      preemptible: true
+  # - command: .buildkite/scripts/steps/esql_grammar_sync.sh
+  #   label: Grammar Sync
+  #   timeout_in_minutes: 10
+  #   agents:
+  #     image: family/kibana-ubuntu-2004
+  #     imageProject: elastic-images-prod
+  #     provider: gcp
+  #     machineType: n2-standard-2
+  #     preemptible: true
   - command: .buildkite/scripts/steps/esql_generate_function_metadata.sh
     label: Generate Function Metadata
     timeout_in_minutes: 15

.buildkite/scripts/steps/esql_generate_function_metadata.sh

@@ -16,6 +16,13 @@ main () {
 
   rm -rf elasticsearch
   git clone https://github.com/elastic/elasticsearch --depth 1
+  cd "$PARENT_DIR/elasticsearch"
+
+  echo "FETCHING 8.19 branch"
+  git fetch origin 8.19
+
+  echo "CHECKING OUT 8.19 branch"
+  git checkout FETCH_HEAD -b 8.19
 
   report_main_step "Bootstrapping Kibana"
 
@@ -56,18 +63,18 @@ main () {
   git config --global user.name "$KIBANA_MACHINE_USERNAME"
   git config --global user.email '[email protected]'
 
-  PR_TITLE='[ES|QL] Update function metadata'
+  PR_TITLE='[ES|QL] Update function metadata for 8.19'
   PR_BODY='This PR updates the function definitions and inline docs based on the latest metadata from Elasticsearch.'
 
-  # Check if a PR already exists
-  pr_search_result=$(gh pr list --search "$PR_TITLE" --state open --author "$KIBANA_MACHINE_USERNAME"  --limit 1 --json title -q ".[].title")
+  # # Check if a PR already exists
+  # pr_search_result=$(gh pr list --search "$PR_TITLE" --state open --author "$KIBANA_MACHINE_USERNAME"  --limit 1 --json title -q ".[].title")
 
-  if [ "$pr_search_result" == "$PR_TITLE" ]; then
-    echo "PR already exists. Exiting."
-    exit
-  fi
+  # if [ "$pr_search_result" == "$PR_TITLE" ]; then
+  #   echo "PR already exists. Exiting."
+  #   exit
+  # fi
 
-  echo "No existing PR found. Committing changes."
+  # echo "No existing PR found. Committing changes."
 
   # Make a commit
   BRANCH_NAME="esql_generate_function_metadata_$(date +%s)"
@@ -82,7 +89,7 @@ main () {
   git push origin "$BRANCH_NAME"
 
   # Create a PR
-  gh pr create --title "$PR_TITLE" --body "$PR_BODY" --base main --head "${BRANCH_NAME}" --label 'release_note:skip' --label 'Team:ESQL' 
+  gh pr create --title "$PR_TITLE" --body "$PR_BODY" --base 8.19 --head "${BRANCH_NAME}" --label 'release_note:skip' --label 'Team:ESQL' --label 'backport:skip'
 }
 
 main

src/platform/packages/private/kbn-language-documentation/scripts/generate_esql_docs.ts

@@ -22,17 +22,26 @@ interface DocsSectionContent {
   const pathToElasticsearch = process.argv[2];
   const { scalarFunctions, aggregationFunctions, groupingFunctions } =
     loadFunctionDocs(pathToElasticsearch);
+
+  const allFunctionNames = Array.from(scalarFunctions.keys()).concat(
+    Array.from(aggregationFunctions.keys()),
+    Array.from(groupingFunctions.keys())
+  );
+
   writeFunctionDocs(
     scalarFunctions,
-    path.join(__dirname, '../src/sections/generated/scalar_functions.tsx')
+    path.join(__dirname, '../src/sections/generated/scalar_functions.tsx'),
+    allFunctionNames
   );
   writeFunctionDocs(
     aggregationFunctions,
-    path.join(__dirname, '../src/sections/generated/aggregation_functions.tsx')
+    path.join(__dirname, '../src/sections/generated/aggregation_functions.tsx'),
+    allFunctionNames
   );
   writeFunctionDocs(
     groupingFunctions,
-    path.join(__dirname, '../src/sections/generated/grouping_functions.tsx')
+    path.join(__dirname, '../src/sections/generated/grouping_functions.tsx'),
+    allFunctionNames
   );
 })();
 
@@ -74,20 +83,18 @@ function loadFunctionDocs(pathToElasticsearch: string) {
       const functionName = path.basename(file, '.md');
 
       // Add the function name and content to the map
-      if (functionDefinition.type === 'scalar') {
-        scalarFunctions.set(functionName, {
+      if (['bucket', 'categorize'].includes(functionDefinition.name)) {
+        groupingFunctions.set(functionName, {
           description: content,
           preview: functionDefinition.preview,
         });
-      }
-      if (functionDefinition.type === 'agg') {
+      } else if (functionDefinition.type === 'agg') {
         aggregationFunctions.set(functionName, {
           description: content,
           preview: functionDefinition.preview,
         });
-      }
-      if (functionDefinition.type === 'grouping') {
-        groupingFunctions.set(functionName, {
+      } else if (functionDefinition.type === 'eval') {
+        scalarFunctions.set(functionName, {
           description: content,
           preview: functionDefinition.preview,
         });
@@ -98,13 +105,25 @@ function loadFunctionDocs(pathToElasticsearch: string) {
   return { scalarFunctions, aggregationFunctions, groupingFunctions };
 }
 
-function writeFunctionDocs(functionDocs: Map<string, DocsSectionContent>, pathToDocsFile: string) {
+function replaceCodeBlocksForESQLFormatting(text: string) {
+  return text.replace(/```(\n[\s\S]*?\n)```/g, '```esql$1```');
+}
+
+function writeFunctionDocs(
+  functionDocs: Map<string, DocsSectionContent>,
+  pathToDocsFile: string,
+  allFunctionNames: string[]
+) {
   const codeStrings = Array.from(functionDocs.entries()).map(([name, doc]) => {
     const docWithoutLinks = removeAsciiDocInternalCrossReferences(
       doc.description,
-      Array.from(functionDocs.keys())
+      allFunctionNames
+    );
+    const defaultMessage = replaceCodeBlocksForESQLFormatting(docWithoutLinks).replaceAll(
+      '`',
+      '\\`'
     );
-    const defaultMessage = docWithoutLinks.replaceAll('`', '\\`');
+
     return `
   const foo = 
   // Do not edit manually... automatically generated by scripts/generate_esql_docs.ts

src/platform/packages/shared/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts

@@ -311,7 +311,7 @@ const convertDateTime = (s: string) => (s === 'datetime' ? 'date' : s);
  */
 function getFunctionDefinition(ESFunctionDefinition: Record<string, any>): FunctionDefinition {
   let locationsAvailable =
-    ESFunctionDefinition.type === FunctionDefinitionTypes.SCALAR
+    ESFunctionDefinition.type === 'eval'
       ? defaultScalarFunctionLocations
       : defaultAggFunctionLocations;
 
@@ -320,7 +320,11 @@ function getFunctionDefinition(ESFunctionDefinition: Record<string, any>): Funct
     locationsAvailable = [Location.WHERE];
   }
   const ret = {
-    type: ESFunctionDefinition.type,
+    type: ['bucket', 'categorize'].includes(ESFunctionDefinition.name)
+      ? 'grouping'
+      : ESFunctionDefinition.type === 'eval'
+      ? 'scalar'
+      : ESFunctionDefinition.type,
     name: ESFunctionDefinition.name,
     operator: ESFunctionDefinition.operator,
     locationsAvailable,
@@ -688,13 +692,13 @@ const enrichGrouping = (
       ];
       return {
         ...op,
-        locationsAvailable: [...op.locationsAvailable, Location.STATS_BY],
+        locationsAvailable: [Location.STATS, Location.STATS_BY],
         signatures,
       };
     }
     return {
       ...op,
-      locationsAvailable: [...op.locationsAvailable, Location.STATS_BY],
+      locationsAvailable: [Location.STATS, Location.STATS_BY],
     };
   });
 };
@@ -951,7 +955,6 @@ ${
 
     const functionDefinition = getFunctionDefinition(ESDefinition);
     const isLikeOperator = functionDefinition.name.toLowerCase().includes('like');
-    const arePredicates = functionDefinition.name.toLowerCase().includes('predicates');
 
     if (functionDefinition.name.toLowerCase() === 'match') {
       scalarFunctionDefinitions.push({
@@ -961,30 +964,6 @@ ${
       continue;
     }
 
-    if (arePredicates) {
-      const nullFunctions: FunctionDefinition[] = [
-        {
-          name: 'is null',
-          description: 'Predicate for NULL comparison: returns true if the value is NULL',
-          operator: 'is null',
-        },
-        {
-          name: 'is not null',
-          description: 'Predicate for NULL comparison: returns true if the value is not NULL',
-          operator: 'is not null',
-        },
-      ].map<FunctionDefinition>(({ name, description, operator }) => {
-        return {
-          ...functionDefinition,
-          name,
-          operator,
-          description,
-        };
-      });
-      operatorDefinitions.push(...nullFunctions);
-      continue;
-    }
-
     if (functionDefinition.type === FunctionDefinitionTypes.OPERATOR || isLikeOperator) {
       operatorDefinitions.push(functionDefinition);
     }