[Explore Vis] fix field formatter not applied to explore query result (#10574)

Currently, time fields in the result set may appear as plain strings (e.g., "2025-09-24 14:50:00") without timezone information.
As a result, visualizations always display them in UTC, and the times do not adjust according to the system timezone settings (e.g., UTC+8).

Root Cause
In Explore, the PPL query results were fetched without passing the language-level field formatter to searchSource.fetch.
Without this formatter, date values such as "2025-09-24 13:50:46.476" remain as raw UTC strings and are not normalized to ISO8601 (YYYY-MM-DDTHH:mm:ss.SSSZ), preventing timezone conversion.
Additionally, the logic for determining whether a field should be treated as a date field was unreliable — it inferred types based on the field name (e.g., checking if the name included "date" or "timestamp"), which caused incorrect or inconsistent behavior.

Additional Note
Vega visualizations internally use d3-time and d3-time-format to handle time scales and formatting.
These libraries only support local time and UTC modes — arbitrary timezones are not supported.
Therefore, visualization time display is now set to local time by default (the browser’s timezone).

---------

Signed-off-by: Luna Liu <[email protected]>
Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>
Co-authored-by: Yulong Ruan <[email protected]>
Co-authored-by: Adam Tackett <[email protected]>

Author: 4 people

changelogs/fragments/10574.yml

@@ -0,0 +1,2 @@
+fix:
+- Fix field formatter not applied to explore query result ([#10574](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/10574))

src/plugins/data/common/data_frames/utils.test.ts

@@ -8,6 +8,7 @@ import {
   convertResult,
   DATA_FRAME_TYPES,
   formatTimePickerDate,
+  getFieldType,
   IDataFrameErrorResponse,
   IDataFrameResponse,
 } from '.';
@@ -36,6 +37,52 @@ describe('formatTimePickerDate', () => {
   });
 });
 
+describe('getFieldType', () => {
+  it('should return object for struct type', () => {
+    const field = { type: 'struct' };
+    const result = getFieldType(field);
+    expect(result).toBe('object');
+  });
+
+  it('should return date for timestamp type', () => {
+    const field = { type: 'timestamp' };
+    const result = getFieldType(field);
+    expect(result).toBe('date');
+  });
+
+  it('should return date for field name containing date', () => {
+    const field = { name: 'created_date' };
+    const result = getFieldType(field);
+    expect(result).toBe('date');
+  });
+
+  it('should return date for field name containing timestamp', () => {
+    const field = { name: 'event_timestamp' };
+    const result = getFieldType(field);
+    expect(result).toBe('date');
+  });
+
+  it('should return date for field with Date values', () => {
+    const field = { values: [new Date()] };
+    const result = getFieldType(field);
+    expect(result).toBe('date');
+  });
+
+  it('should return date for field with datemath parseable values', () => {
+    jest.spyOn(datemath, 'isDateTime').mockReturnValue(true);
+    const field = { values: ['2025-02-13T00:51:50Z'] };
+    const result = getFieldType(field);
+    expect(result).toBe('date');
+    expect(datemath.isDateTime).toHaveBeenCalledWith('2025-02-13T00:51:50Z');
+  });
+
+  it('should return original type if no special conditions match', () => {
+    const field = { type: 'keyword' };
+    const result = getFieldType(field);
+    expect(result).toBe('keyword');
+  });
+});
+
 describe('convertResult', () => {
   const mockDateString = '2025-02-13 00:51:50';
   const expectedFormattedDate = moment.utc(mockDateString).format('YYYY-MM-DDTHH:mm:ssZ');

src/plugins/data/common/data_frames/utils.ts

@@ -174,16 +174,20 @@ export const convertResult = ({
  * @returns field type
  */
 export const getFieldType = (field: IFieldType | Partial<IFieldType>): string | undefined => {
-  const fieldName = field.name?.toLowerCase();
-  if (fieldName?.includes('date') || fieldName?.includes('timestamp')) {
+  const rawType = field.type?.toString().toLowerCase();
+  if (rawType) {
+    if (rawType === 'struct') return 'object';
+    if (rawType === 'timestamp') return 'date';
+    return rawType;
+  }
+
+  const fieldName = field.name?.toLowerCase() ?? '';
+  if (fieldName.includes('date') || fieldName.includes('timestamp')) {
     return 'date';
   }
   if (field.values?.some((value) => value instanceof Date || datemath.isDateTime(value))) {
     return 'date';
   }
-  if (field.type === 'struct') {
-    return 'object';
-  }
 
   return field.type;
 };

src/plugins/explore/public/application/utils/state_management/actions/query_actions.test.ts

@@ -25,6 +25,22 @@ jest.mock('moment-timezone', () => {
   return moment;
 });
 
+jest.mock('./utils', () => ({
+  buildPPLHistogramQuery: jest.fn((queryString) => queryString),
+  processRawResultsForHistogram: jest.fn((_queryString, rawResults) => rawResults),
+  createHistogramConfigWithInterval: jest.fn(() => ({
+    toDsl: jest.fn().mockReturnValue({}),
+    aggs: [
+      {},
+      {
+        buckets: {
+          getInterval: jest.fn(() => ({ interval: '1h', scale: 1 })),
+        },
+      },
+    ],
+  })),
+}));
+
 import { configureStore } from '@reduxjs/toolkit';
 import {
   abortAllActiveQueries,
@@ -208,6 +224,9 @@ describe('Query Actions - Comprehensive Test Suite', () => {
         query: {
           queryString: {
             getQuery: jest.fn().mockReturnValue({ query: '', language: 'PPL' }),
+            getLanguageService: jest.fn().mockReturnValue({
+              getLanguage: jest.fn().mockReturnValue({}),
+            }),
           },
           filterManager: {
             getFilters: jest.fn().mockReturnValue([]),
@@ -640,7 +659,6 @@ describe('Query Actions - Comprehensive Test Suite', () => {
       mockBuildPointSeriesData.mockReturnValue([{ x: 1609459200000, y: 5 }] as any);
       mockTabifyAggResponse.mockReturnValue({ rows: [], columns: [] } as any);
     });
-
     it('should process histogram data when timeFieldName exists', () => {
       const rawResults = {
         hits: {
@@ -1158,7 +1176,34 @@ describe('Query Actions - Comprehensive Test Suite', () => {
             },
           },
         ]),
+        aggs: [
+          {} as any,
+          {
+            buckets: {
+              getInterval: jest.fn(() => ({ interval: '1h', scale: 1 })),
+            },
+          } as any,
+        ],
       } as any);
+      mockSearchSource.fetch.mockReset();
+      mockSearchSource.fetch.mockResolvedValue({
+        hits: {
+          hits: [
+            { _id: '1', _source: { field1: 'value1' } },
+            { _id: '2', _source: { field2: 'value2' } },
+          ],
+          total: 2,
+        },
+        took: 5,
+        aggregations: {
+          histogram: {
+            buckets: [
+              { key: 1609459200000, doc_count: 5 },
+              { key: 1609462800000, doc_count: 3 },
+            ],
+          },
+        },
+      });
     });
 
     it('should execute histogram query with custom interval', async () => {
@@ -1174,6 +1219,92 @@ describe('Query Actions - Comprehensive Test Suite', () => {
 
       expect(mockServices.data.dataViews.get).toHaveBeenCalled();
       expect(mockSearchSource.fetch).toHaveBeenCalled();
+      expect(mockDispatch).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'queryEditor/setIndividualQueryStatus',
+          payload: expect.objectContaining({
+            cacheKey: 'test-cache-key',
+            status: expect.objectContaining({
+              status: QueryExecutionStatus.READY,
+            }),
+          }),
+        })
+      );
+    });
+
+    it('should execute histogram query with formatter in language config', async () => {
+      const mockFormatter = jest.fn();
+      (mockServices.data.query.queryString.getLanguageService as jest.Mock).mockReturnValue({
+        getLanguage: jest.fn().mockReturnValue({
+          fields: {
+            formatter: mockFormatter,
+          },
+        }),
+      });
+
+      const params = {
+        services: mockServices,
+        cacheKey: 'test-cache-key',
+        queryString: 'source=logs',
+        interval: '5m',
+      };
+
+      const thunk = executeHistogramQuery(params);
+      await thunk(mockDispatch, mockGetState, undefined);
+
+      expect(mockServices.data.dataViews.get).toHaveBeenCalled();
+      expect(mockSearchSource.fetch).toHaveBeenCalledWith(
+        expect.objectContaining({
+          abortSignal: expect.any(Object),
+          formatter: mockFormatter,
+        })
+      );
+      expect(mockDispatch).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'queryEditor/setIndividualQueryStatus',
+          payload: expect.objectContaining({
+            cacheKey: 'test-cache-key',
+            status: expect.objectContaining({
+              status: QueryExecutionStatus.READY,
+            }),
+          }),
+        })
+      );
+    });
+
+    it('should execute histogram query without formatter in language config', async () => {
+      (mockServices.data.query.queryString.getLanguageService as jest.Mock).mockReturnValue({
+        getLanguage: jest.fn().mockReturnValue({}),
+      });
+
+      const params = {
+        services: mockServices,
+        cacheKey: 'test-cache-key',
+        queryString: 'source=logs',
+        interval: '5m',
+      };
+
+      const thunk = executeHistogramQuery(params);
+      await thunk(mockDispatch, mockGetState, undefined);
+
+      expect(mockServices.data.dataViews.get).toHaveBeenCalled();
+      expect(mockSearchSource.fetch).toHaveBeenCalledWith(
+        expect.objectContaining({
+          abortSignal: expect.any(Object),
+          withLongNumeralsSupport: false,
+        })
+      );
+      expect(mockDispatch).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'queryEditor/setIndividualQueryStatus',
+          payload: expect.objectContaining({
+            cacheKey: 'test-cache-key',
+            status: expect.objectContaining({
+              status: QueryExecutionStatus.READY,
+            }),
+          }),
+        })
+      );
     });
 
     it('should handle missing interval gracefully', async () => {
@@ -1187,6 +1318,18 @@ describe('Query Actions - Comprehensive Test Suite', () => {
       await thunk(mockDispatch, mockGetState, undefined);
 
       expect(mockServices.data.dataViews.get).toHaveBeenCalled();
+      expect(mockSearchSource.fetch).toHaveBeenCalled();
+      expect(mockDispatch).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'queryEditor/setIndividualQueryStatus',
+          payload: expect.objectContaining({
+            cacheKey: 'test-cache-key',
+            status: expect.objectContaining({
+              status: QueryExecutionStatus.READY,
+            }),
+          }),
+        })
+      );
     });
 
     it('should handle dataView without timeFieldName', async () => {
@@ -1202,16 +1345,32 @@ describe('Query Actions - Comprehensive Test Suite', () => {
 
       const thunk = executeHistogramQuery(params);
       await thunk(mockDispatch, mockGetState, undefined);
-
+      expect(mockServices.data.dataViews.get).toHaveBeenCalled();
       expect(mockSearchSource.fetch).toHaveBeenCalled();
+      expect(mockDispatch).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'queryEditor/setIndividualQueryStatus',
+          payload: expect.objectContaining({
+            cacheKey: 'test-cache-key',
+            status: expect.objectContaining({
+              status: QueryExecutionStatus.READY,
+            }),
+          }),
+        })
+      );
     });
 
     it('should handle search errors gracefully', async () => {
       const error = {
         body: {
           error: 'Search failed',
-          message:
-            '{"error":{"details":"Query syntax error","reason":"Invalid query","type":"parsing_exception"}}',
+          message: JSON.stringify({
+            error: {
+              details: 'Query syntax error',
+              reason: 'Invalid query',
+              type: 'parsing_exception',
+            },
+          }),
           statusCode: 400,
         },
       };
@@ -1235,11 +1394,20 @@ describe('Query Actions - Comprehensive Test Suite', () => {
       // Verify error status is set in Redux
       expect(mockDispatch).toHaveBeenCalledWith(
         expect.objectContaining({
-          type: expect.stringContaining('setIndividualQueryStatus'),
+          type: 'queryEditor/setIndividualQueryStatus',
           payload: expect.objectContaining({
             cacheKey: 'test-cache-key',
             status: expect.objectContaining({
               status: QueryExecutionStatus.ERROR,
+              error: expect.objectContaining({
+                error: 'Search failed',
+                message: {
+                  details: 'Query syntax error',
+                  reason: 'Invalid query',
+                  type: 'parsing_exception',
+                },
+                statusCode: 400,
+              }),
             }),
           }),
         })
@@ -1384,6 +1552,17 @@ describe('Query Actions - Comprehensive Test Suite', () => {
 
       const mockIndexPatterns = dataPublicModule.indexPatterns as any;
       mockIndexPatterns.isDefault.mockReturnValue(true);
+      mockSearchSource.fetch.mockReset();
+      mockSearchSource.fetch.mockResolvedValue({
+        hits: {
+          hits: [
+            { _id: '1', _source: { field1: 'value1' } },
+            { _id: '2', _source: { field2: 'value2' } },
+          ],
+          total: 2,
+        },
+        took: 5,
+      });
     });
 
     it('should execute tab query successfully', async () => {
@@ -1399,6 +1578,17 @@ describe('Query Actions - Comprehensive Test Suite', () => {
       expect(mockServices.data.dataViews.get).toHaveBeenCalled();
       expect(mockSearchSource.fetch).toHaveBeenCalled();
       expect(setResults).toHaveBeenCalled();
+      expect(mockDispatch).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'queryEditor/setIndividualQueryStatus',
+          payload: expect.objectContaining({
+            cacheKey: 'test-cache-key',
+            status: expect.objectContaining({
+              status: QueryExecutionStatus.READY,
+            }),
+          }),
+        })
+      );
     });
 
     it('should handle missing services gracefully', async () => {

src/plugins/explore/public/application/utils/state_management/actions/query_actions.ts

@@ -468,10 +468,15 @@ const executeQueryBase = async (
       });
     }
 
+    const languageConfig = services.data.query.queryString
+      .getLanguageService()
+      .getLanguage(query.language);
+
     // Execute query
     const rawResults = await searchSource.fetch({
       abortSignal: abortController.signal,
       withLongNumeralsSupport: await services.uiSettings.get('data:withLongNumerals'),
+      ...(languageConfig?.fields?.formatter ? { formatter: languageConfig.fields.formatter } : {}),
     });
 
     // Add response stats to inspector