[8.19] [ES|QL] Fixes the columns selection in Lens when there are ES|QL variables (#221083) (#221150)

# Backport

This will backport the following commits from `main` to `8.19`:
- [[ES|QL] Fixes the columns selection in Lens when there are ES|QL
variables (#221083)](#221083)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Stratoula
Kalafateli","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-05-21T16:11:37Z","message":"[ES|QL]
Fixes the columns selection in Lens when there are ES|QL variables
(#221083)\n\n## Summary\n\nIt fixes the problem described in the PR.
Specifically when you have a\nvariable in your esql query and you try to
select extra columns. Because\nwe were not passing the esql Variables,
the query failed and the columns\narray was empty.\n\nThis PR is passing
the dashboard variables in the component, and now the\nquery runs
successfully\n\n<img width=\"616\"
alt=\"image\"\nsrc=\"https://github.com/user-attachments/assets/cd009af9-9465-48cd-bb3b-918c3a872cff\"\n/>\n
\n\n### Checklist\n- [ ] [Unit or
functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere
updated or added to match the most common
scenarios","sha":"6fa3152f2773890e4bf721b9149b875ece83f45c","branchLabelMapping":{"^v9.1.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Visualizations","release_note:skip","Feature:Lens","Feature:ES|QL","backport:version","v9.1.0","v8.19.0","v8.18.2"],"title":"[ES|QL]
Fixes the columns selection in Lens when there are ES|QL
variables","number":221083,"url":"https://github.com/elastic/kibana/pull/221083","mergeCommit":{"message":"[ES|QL]
Fixes the columns selection in Lens when there are ES|QL variables
(#221083)\n\n## Summary\n\nIt fixes the problem described in the PR.
Specifically when you have a\nvariable in your esql query and you try to
select extra columns. Because\nwe were not passing the esql Variables,
the query failed and the columns\narray was empty.\n\nThis PR is passing
the dashboard variables in the component, and now the\nquery runs
successfully\n\n<img width=\"616\"
alt=\"image\"\nsrc=\"https://github.com/user-attachments/assets/cd009af9-9465-48cd-bb3b-918c3a872cff\"\n/>\n
\n\n### Checklist\n- [ ] [Unit or
functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere
updated or added to match the most common
scenarios","sha":"6fa3152f2773890e4bf721b9149b875ece83f45c"}},"sourceBranch":"main","suggestedTargetBranches":["8.19","8.18"],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/221083","number":221083,"mergeCommit":{"message":"[ES|QL]
Fixes the columns selection in Lens when there are ES|QL variables
(#221083)\n\n## Summary\n\nIt fixes the problem described in the PR.
Specifically when you have a\nvariable in your esql query and you try to
select extra columns. Because\nwe were not passing the esql Variables,
the query failed and the columns\narray was empty.\n\nThis PR is passing
the dashboard variables in the component, and now the\nquery runs
successfully\n\n<img width=\"616\"
alt=\"image\"\nsrc=\"https://github.com/user-attachments/assets/cd009af9-9465-48cd-bb3b-918c3a872cff\"\n/>\n
\n\n### Checklist\n- [ ] [Unit or
functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere
updated or added to match the most common
scenarios","sha":"6fa3152f2773890e4bf721b9149b875ece83f45c"}},{"branch":"8.19","label":"v8.19.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.18","label":"v8.18.2","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Stratoula Kalafateli <[email protected]>

Author: kibanamachine

x-pack/platform/plugins/shared/lens/public/datasources/form_based/esql_layer/components/dimension_editor.test.tsx

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import React from 'react';
+import { render, screen, waitFor } from '@testing-library/react';
+import { TextBasedDimensionEditor, TextBasedDimensionEditorProps } from './dimension_editor';
+
+// Mock fetchFieldsFromESQL
+jest.mock('@kbn/esql-editor', () => ({
+  fetchFieldsFromESQL: jest.fn(),
+}));
+
+const { fetchFieldsFromESQL } = jest.requireMock('@kbn/esql-editor');
+
+describe('TextBasedDimensionEditor', () => {
+  const defaultProps = {
+    isFullscreen: false,
+    columnId: 'dim1',
+    layerId: 'layer1',
+    state: {
+      layers: {
+        layer1: {
+          query: { esql: 'FROM my_data' },
+          columns: [],
+          indexPatternRefs: [],
+        },
+      },
+      indexPatternRefs: [],
+    },
+    setState: jest.fn(),
+    indexPatterns: {},
+    dateRange: { fromDate: '2023-01-01', toDate: '2023-01-31' },
+    expressions: {},
+    esqlVariables: [
+      {
+        key: 'agent_keyword',
+        value: 'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1',
+        type: 'values',
+      },
+    ],
+    isMetricDimension: false,
+    filterOperations: jest.fn(() => true),
+    core: { docLinks: {} },
+    groupId: 'rows',
+  } as unknown as TextBasedDimensionEditorProps;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+
+    fetchFieldsFromESQL.mockResolvedValue({
+      columns: [
+        { id: 'field1', name: 'Field One', meta: { type: 'string' } },
+        { id: 'field2', name: 'Field Two', meta: { type: 'number' } },
+      ],
+    });
+  });
+
+  it('renders correctly and fetches columns on mount', async () => {
+    render(<TextBasedDimensionEditor {...defaultProps} />);
+
+    // Check if fetchFieldsFromESQL was called
+    await waitFor(() => {
+      expect(fetchFieldsFromESQL).toHaveBeenCalledTimes(1);
+      expect(fetchFieldsFromESQL).toHaveBeenCalledWith(
+        { esql: 'FROM my_data | limit 0' },
+        {},
+        { from: defaultProps.dateRange.fromDate, to: defaultProps.dateRange.toDate },
+        undefined,
+        undefined, // No index patterns
+        defaultProps.esqlVariables
+      );
+    });
+
+    expect(screen.getByTestId('text-based-dimension-field')).toBeInTheDocument();
+  });
+});

x-pack/platform/plugins/shared/lens/public/datasources/form_based/esql_layer/components/dimension_editor.tsx

@@ -39,6 +39,7 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
     indexPatterns,
     dateRange,
     expressions,
+    esqlVariables,
   } = props;
 
   useEffect(() => {
@@ -52,7 +53,8 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
           undefined,
           Object.values(indexPatterns).length
             ? Object.values(indexPatterns)[0].timeFieldName
-            : undefined
+            : undefined,
+          esqlVariables
         );
         if (table) {
           const hasNumberTypeColumns = table.columns?.some(isNumeric);
@@ -84,6 +86,7 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
     indexPatterns,
     props,
     props.expressions,
+    esqlVariables,
     query,
   ]);
 
@@ -122,6 +125,7 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
         className="lnsIndexPatternDimensionEditor--padded"
       >
         <FieldSelect
+          data-test-subj="text-based-dimension-field"
           existingFields={allColumns ?? []}
           selectedField={selectedField}
           onChoose={(choice) => {

x-pack/platform/plugins/shared/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx

@@ -16,11 +16,13 @@ import {
   EuiIconTip,
   useEuiTheme,
 } from '@elastic/eui';
+import { BehaviorSubject } from 'rxjs';
 import { i18n } from '@kbn/i18n';
 import { css } from '@emotion/react';
 import { euiThemeVars } from '@kbn/ui-theme';
 import { DragDropIdentifier, ReorderProvider, DropType } from '@kbn/dom-drag-drop';
 import { DimensionButton } from '@kbn/visualization-ui-components';
+import { useStateFromPublishingSubject } from '@kbn/presentation-publishing';
 import { isOfAggregateQueryType } from '@kbn/es-query';
 import { LayerActions } from './layer_actions';
 import { isOperation, LayerAction, VisualizationDimensionGroupConfig } from '../../../types';
@@ -40,6 +42,7 @@ import { getSharedActions } from './layer_actions/layer_actions';
 import { FlyoutContainer } from '../../../shared_components/flyout_container';
 import { FakeDimensionButton } from './buttons/fake_dimension_button';
 import { getLongMessage } from '../../../user_messages_utils';
+import { isApiESQLVariablesCompatible } from '../../../react_embeddable/types';
 import { ESQLEditor } from './esql_editor';
 
 export function LayerPanel(props: LayerPanelProps) {
@@ -78,6 +81,13 @@ export function LayerPanel(props: LayerPanelProps) {
     ...editorProps
   } = props;
 
+  const { parentApi } = editorProps;
+  const esqlVariables = useStateFromPublishingSubject(
+    isApiESQLVariablesCompatible(parentApi)
+      ? parentApi?.esqlVariables$
+      : new BehaviorSubject(undefined)
+  );
+
   const isInlineEditing = Boolean(props?.setIsInlineFlyoutVisible);
 
   const isSaveable = useLensSelector((state) => state.lens.isSaveable);
@@ -814,6 +824,7 @@ export function LayerPanel(props: LayerPanelProps) {
                 layerType: activeVisualization.getLayerType(layerId, visualizationState),
                 indexPatterns: dataViews.indexPatterns,
                 activeData: layerVisualizationConfigProps.activeData,
+                esqlVariables,
                 dataSectionExtra: !isFullscreen &&
                   openDimension.isComplete &&
                   activeVisualization.DimensionEditorDataExtraComponent && (

x-pack/platform/plugins/shared/lens/public/types.ts

@@ -9,6 +9,7 @@ import type { IconType } from '@elastic/eui/src/components/icon/icon';
 import type { CoreStart, SavedObjectReference, ResolvedSimpleSavedObject } from '@kbn/core/public';
 import type { ColorMapping, PaletteOutput } from '@kbn/coloring';
 import type { TopNavMenuData } from '@kbn/navigation-plugin/public';
+import type { ESQLControlVariable } from '@kbn/esql-types';
 import type { MutableRefObject, ReactElement } from 'react';
 import type { Query, AggregateQuery, Filter, TimeRange } from '@kbn/es-query';
 import type {
@@ -673,6 +674,7 @@ export type DatasourceDimensionEditorProps<T = unknown> = DatasourceDimensionPro
     | 'docLinks'
   >;
   dateRange: DateRange;
+  esqlVariables?: ESQLControlVariable[] | undefined;
   dimensionGroups: VisualizationDimensionGroupConfig[];
   toggleFullscreen: () => void;
   isFullscreen: boolean;