[ES|QL] Removes the expressions dependency (#237403)

## Summary

This PR is fixing a tech debt.

- The expressions dependency on the ES|QL editor doesnt make any sense
and is a bit of reduntant, we can retrieve the columns without it
- The same for the alerts, they are just getting the table (fun fact
they dont even need our extra formatiing that we do in the expressions
so this will make it a bit faster)
- For Lens it makes sense to get the columns with expressions, we are
also including there some formatting and config that it is important for
the Lens ES|QL charts. I moved the util to them
- The editor package is private. Importing this helper from there is
wrong considering the recent changes we did on the packages visibility.

### Checklist

- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Marco Liberati <[email protected]>

Author: 3 people

src/platform/packages/private/kbn-esql-editor/index.ts

@@ -8,7 +8,6 @@
  */
 
 export type { DataErrorsControl } from './src/types';
-export { fetchFieldsFromESQL } from './src/fetch_fields_from_esql';
 export type { ESQLEditorProps } from './src/esql_editor';
 import { ESQLEditor } from './src/esql_editor';
 export type { ESQLEditorRestorableState } from './src/restorable_state';

src/platform/packages/private/kbn-esql-editor/src/esql_editor.tsx

@@ -20,6 +20,7 @@ import {
   type EuiButtonColor,
 } from '@elastic/eui';
 import { Global, css } from '@emotion/react';
+import { getESQLQueryColumns } from '@kbn/esql-utils';
 import type { CodeEditorProps } from '@kbn/code-editor';
 import { CodeEditor } from '@kbn/code-editor';
 import type { CoreStart } from '@kbn/core/public';
@@ -34,7 +35,6 @@ import {
   type IndicesAutocompleteResult,
 } from '@kbn/esql-types';
 import { fixESQLQueryWithVariables, getRemoteClustersFromESQLQuery } from '@kbn/esql-utils';
-import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
 import { KBN_FIELD_TYPES } from '@kbn/field-types';
 import { i18n } from '@kbn/i18n';
 import type { SerializedEnrichPolicy } from '@kbn/index-management-shared-types';
@@ -58,7 +58,6 @@ import {
   esqlEditorStyles,
 } from './esql_editor.styles';
 import { ESQLEditorTelemetryService } from './telemetry/telemetry_service';
-import { fetchFieldsFromESQL } from './fetch_fields_from_esql';
 import {
   clearCacheWhenOld,
   filterDataErrors,
@@ -138,7 +137,7 @@ const ESQLEditorInternal = function ESQLEditor({
   const datePickerOpenStatusRef = useRef<boolean>(false);
   const theme = useEuiTheme();
   const kibana = useKibana<ESQLEditorDeps>();
-  const { dataViews, expressions, application, core, fieldsMetadata, uiSettings, uiActions, data } =
+  const { dataViews, application, core, fieldsMetadata, uiSettings, uiActions, data } =
     kibana.services;
 
   const activeSolutionId = useObservable(core.chrome.getActiveSolutionNavId$());
@@ -440,18 +439,20 @@ const ESQLEditorInternal = function ESQLEditor({
     const fn = memoize(
       (
         ...args: [
-          { esql: string },
-          ExpressionsStart,
-          TimeRange,
-          AbortController?,
-          string?,
-          ESQLControlVariable[]?
+          {
+            esqlQuery: string;
+            search: any;
+            timeRange: TimeRange;
+            signal?: AbortSignal;
+            dropNullColumns?: boolean;
+            variables?: ESQLControlVariable[];
+          }
         ]
       ) => ({
         timestamp: Date.now(),
-        result: fetchFieldsFromESQL(...args),
+        result: getESQLQueryColumns(...args),
       }),
-      ({ esql }) => esql
+      ({ esqlQuery }) => esqlQuery
     );
 
     return { cache: fn.cache, memoizedFieldsFromESQL: fn };
@@ -511,24 +512,20 @@ const ESQLEditorInternal = function ESQLEditor({
       },
       getColumnsFor: async ({ query: queryToExecute }: { query?: string } | undefined = {}) => {
         if (queryToExecute) {
-          // ES|QL with limit 0 returns only the columns and is more performant
-          const esqlQuery = {
-            esql: `${queryToExecute} | limit 0`,
-          };
           // Check if there's a stale entry and clear it
-          clearCacheWhenOld(esqlFieldsCache, esqlQuery.esql);
+          clearCacheWhenOld(esqlFieldsCache, `${queryToExecute} | limit 0`);
           const timeRange = data.query.timefilter.timefilter.getTime();
           try {
-            const table = await memoizedFieldsFromESQL(
-              esqlQuery,
-              expressions,
+            const columns = await memoizedFieldsFromESQL({
+              esqlQuery: queryToExecute,
+              search: data.search.search,
               timeRange,
-              abortController,
-              undefined,
-              variablesService?.esqlVariables
-            ).result;
-            const columns: ESQLFieldWithMetadata[] =
-              table?.columns.map((c) => {
+              signal: abortController.signal,
+              variables: variablesService?.esqlVariables,
+              dropNullColumns: true,
+            }).result;
+            const columnsWithMetadata: ESQLFieldWithMetadata[] =
+              columns.map((c) => {
                 return {
                   name: c.name,
                   type: c.meta.esType as FieldType,
@@ -537,7 +534,7 @@ const ESQLEditorInternal = function ESQLEditor({
                 };
               }) || [];
 
-            return columns;
+            return columnsWithMetadata;
           } catch (e) {
             // no action yet
           }
@@ -613,8 +610,8 @@ const ESQLEditorInternal = function ESQLEditor({
     core,
     esqlFieldsCache,
     data.query.timefilter.timefilter,
+    data.search.search,
     memoizedFieldsFromESQL,
-    expressions,
     abortController,
     variablesService?.esqlVariables,
     variablesService?.areSuggestionsEnabled,

src/platform/packages/private/kbn-esql-editor/src/types.ts

@@ -10,7 +10,6 @@
 import type { CoreStart } from '@kbn/core/public';
 import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
 import type { AggregateQuery } from '@kbn/es-query';
-import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
 import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 import type { ILicense } from '@kbn/licensing-types';
 import type { UsageCollectionStart } from '@kbn/usage-collection-plugin/public';
@@ -129,7 +128,6 @@ export interface ESQLEditorDeps {
   core: CoreStart;
   dataViews: DataViewsPublicPluginStart;
   data: DataPublicPluginStart;
-  expressions: ExpressionsStart;
   storage: Storage;
   uiActions: UiActionsStart;
   fieldsMetadata?: FieldsMetadataPublicStart;

src/platform/packages/private/kbn-esql-editor/tsconfig.json

@@ -21,7 +21,6 @@
     "@kbn/language-documentation",
     "@kbn/test-jest-helpers",
     "@kbn/data-plugin",
-    "@kbn/expressions-plugin",
     "@kbn/data-views-plugin",
     "@kbn/index-management-shared-types",
     "@kbn/code-editor",

src/platform/packages/shared/kbn-esql-utils/src/utils/run_query.ts

@@ -70,12 +70,16 @@ export async function getESQLQueryColumnsRaw({
   esqlQuery,
   search,
   signal,
+  filter,
+  dropNullColumns,
   timeRange,
   variables,
 }: {
   esqlQuery: string;
   search: ISearchGeneric;
   signal?: AbortSignal;
+  dropNullColumns?: boolean;
+  filter?: unknown;
   timeRange?: TimeRange;
   variables?: ESQLControlVariable[];
 }): Promise<ESQLColumn[]> {
@@ -85,7 +89,9 @@ export async function getESQLQueryColumnsRaw({
       search(
         {
           params: {
+            ...(filter ? { filter } : {}),
             query: `${esqlQuery} | limit 0`,
+            ...(dropNullColumns ? { dropNullColumns: true } : {}),
             ...(namedParams.length ? { params: namedParams } : {}),
           },
         },
@@ -96,7 +102,20 @@ export async function getESQLQueryColumnsRaw({
       )
     );
 
-    return (response.rawResponse as unknown as ESQLSearchResponse).columns ?? [];
+    const table = response.rawResponse as unknown as ESQLSearchResponse;
+    const hasEmptyColumns = table.all_columns && table.all_columns?.length > table.columns.length;
+    const lookup = new Set(hasEmptyColumns ? table.columns?.map(({ name }) => name) || [] : []);
+
+    const allColumns =
+      (table.all_columns ?? table.columns)?.map(({ name, type }) => {
+        return {
+          name,
+          type,
+          isNull: hasEmptyColumns ? !lookup.has(name) : false,
+        };
+      }) ?? [];
+
+    return allColumns ?? [];
   } catch (error) {
     throw new Error(
       i18n.translate('esqlUtils.columnsErrorMsg', {
@@ -113,19 +132,25 @@ export async function getESQLQueryColumns({
   esqlQuery,
   search,
   signal,
+  filter,
+  dropNullColumns,
   timeRange,
   variables,
 }: {
   esqlQuery: string;
   search: ISearchGeneric;
   signal?: AbortSignal;
+  filter?: unknown;
+  dropNullColumns?: boolean;
   timeRange?: TimeRange;
   variables?: ESQLControlVariable[];
 }): Promise<DatatableColumn[]> {
   try {
     const rawColumns = await getESQLQueryColumnsRaw({
       esqlQuery,
       search,
+      filter,
+      dropNullColumns,
       signal,
       timeRange,
       variables,

src/platform/plugins/shared/esql/public/kibana_services.ts

@@ -10,7 +10,6 @@
 import { BehaviorSubject } from 'rxjs';
 import type { CoreStart } from '@kbn/core/public';
 import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
-import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
 import type { Storage } from '@kbn/kibana-utils-plugin/public';
 import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
@@ -24,7 +23,6 @@ interface ServiceDeps {
   core: CoreStart;
   dataViews: DataViewsPublicPluginStart;
   data: DataPublicPluginStart;
-  expressions: ExpressionsStart;
   storage: Storage;
   uiActions: UiActionsStart;
   fieldsMetadata?: FieldsMetadataPublicStart;
@@ -50,7 +48,6 @@ export const setKibanaServices = (
   kibanaCore: CoreStart,
   dataViews: DataViewsPublicPluginStart,
   data: DataPublicPluginStart,
-  expressions: ExpressionsStart,
   storage: Storage,
   uiActions: UiActionsStart,
   fieldsMetadata?: FieldsMetadataPublicStart,
@@ -60,7 +57,6 @@ export const setKibanaServices = (
   servicesReady$.next({
     core,
     dataViews,
-    expressions,
     data,
     storage,
     uiActions,

src/platform/plugins/shared/esql/public/plugin.ts

@@ -9,7 +9,6 @@
 
 import type { CoreSetup, CoreStart, Plugin } from '@kbn/core/public';
 import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
-import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
 import type { LicensingPluginStart } from '@kbn/licensing-plugin/public';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { UiActionsSetup, UiActionsStart } from '@kbn/ui-actions-plugin/public';
@@ -45,7 +44,6 @@ interface EsqlPluginSetupDependencies {
 
 interface EsqlPluginStartDependencies {
   dataViews: DataViewsPublicPluginStart;
-  expressions: ExpressionsStart;
   uiActions: UiActionsStart;
   fieldsMetadata: FieldsMetadataPublicStart;
   licensing?: LicensingPluginStart;
@@ -81,7 +79,6 @@ export class EsqlPlugin implements Plugin<{}, EsqlPluginStart> {
     core: CoreStart,
     {
       dataViews,
-      expressions,
       data,
       uiActions,
       fieldsMetadata,
@@ -202,7 +199,6 @@ export class EsqlPlugin implements Plugin<{}, EsqlPluginStart> {
       core,
       dataViews,
       data,
-      expressions,
       storage,
       uiActions,
       fieldsMetadata,

src/platform/plugins/shared/esql/tsconfig.json

@@ -12,7 +12,6 @@
     "@kbn/esql-editor",
     "@kbn/kibana-react-plugin",
     "@kbn/core",
-    "@kbn/expressions-plugin",
     "@kbn/data-views-plugin",
     "@kbn/i18n",
     "@kbn/config-schema",

x-pack/platform/plugins/shared/lens/public/datasources/form_based/esql_layer/components/dimension_editor.test.tsx

@@ -9,12 +9,12 @@ import { render, screen, waitFor } from '@testing-library/react';
 import type { TextBasedDimensionEditorProps } from './dimension_editor';
 import { TextBasedDimensionEditor } from './dimension_editor';
 
-// Mock fetchFieldsFromESQL
-jest.mock('@kbn/esql-editor', () => ({
-  fetchFieldsFromESQL: jest.fn(),
+// Mock fetchFieldsFromESQLExpression
+jest.mock('./fetch_fields_from_esql_expression', () => ({
+  fetchFieldsFromESQLExpression: jest.fn(),
 }));
 
-const { fetchFieldsFromESQL } = jest.requireMock('@kbn/esql-editor');
+const { fetchFieldsFromESQLExpression } = jest.requireMock('./fetch_fields_from_esql_expression');
 
 describe('TextBasedDimensionEditor', () => {
   const defaultProps = {
@@ -51,7 +51,7 @@ describe('TextBasedDimensionEditor', () => {
   beforeEach(() => {
     jest.clearAllMocks();
 
-    fetchFieldsFromESQL.mockResolvedValue({
+    fetchFieldsFromESQLExpression.mockResolvedValue({
       columns: [
         { id: 'field1', name: 'Field One', meta: { type: 'string' } },
         { id: 'field2', name: 'Field Two', meta: { type: 'number' } },
@@ -64,8 +64,8 @@ describe('TextBasedDimensionEditor', () => {
 
     // Check if fetchFieldsFromESQL was called
     await waitFor(() => {
-      expect(fetchFieldsFromESQL).toHaveBeenCalledTimes(1);
-      expect(fetchFieldsFromESQL).toHaveBeenCalledWith(
+      expect(fetchFieldsFromESQLExpression).toHaveBeenCalledTimes(1);
+      expect(fetchFieldsFromESQLExpression).toHaveBeenCalledWith(
         { esql: 'FROM my_data | limit 0' },
         {},
         { from: defaultProps.dateRange.fromDate, to: defaultProps.dateRange.toDate },

x-pack/platform/plugins/shared/lens/public/datasources/form_based/esql_layer/components/dimension_editor.tsx

@@ -9,7 +9,6 @@ import React, { useEffect, useState, useMemo, useCallback } from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiFormRow, useEuiTheme, EuiText } from '@elastic/eui';
 import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
-import { fetchFieldsFromESQL } from '@kbn/esql-editor';
 import { NameInput } from '@kbn/visualization-ui-components';
 import { css } from '@emotion/react';
 import { mergeLayer, updateColumnFormat, updateColumnLabel } from '../utils';
@@ -20,6 +19,7 @@ import { FieldSelect, type FieldOptionCompatible } from './field_select';
 import type { TextBasedPrivateState } from '../types';
 import { isNotNumeric, isNumeric } from '../utils';
 import type { TextBasedLayer } from '../types';
+import { fetchFieldsFromESQLExpression } from './fetch_fields_from_esql_expression';
 
 export type TextBasedDimensionEditorProps =
   DatasourceDimensionEditorProps<TextBasedPrivateState> & {
@@ -46,7 +46,7 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
     // in case the columns are not in the cache, I refetch them
     async function fetchColumns() {
       if (query) {
-        const table = await fetchFieldsFromESQL(
+        const table = await fetchFieldsFromESQLExpression(
           { esql: `${query.esql} | limit 0` },
           expressions,
           { from: dateRange.fromDate, to: dateRange.toDate },
@@ -56,6 +56,7 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
             : undefined,
           esqlVariables
         );
+
         if (table) {
           const hasNumberTypeColumns = table.columns?.some(isNumeric);
           const columns = table.columns.map((col) => {
@@ -82,11 +83,10 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
   }, [
     dateRange.fromDate,
     dateRange.toDate,
+    esqlVariables,
     expressions,
     indexPatterns,
     props,
-    props.expressions,
-    esqlVariables,
     query,
   ]);