refactor(vis): implement field normalization for Vega-Lite compatibility

changelogs/fragments/10917.yml

@@ -0,0 +1,2 @@
+refactor:
+- Implement field normalization for Vega-Lite compatibility ([#10917](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/10917))

src/plugins/explore/public/components/visualizations/table/table_vis.tsx

@@ -45,15 +45,15 @@ export const TableVis = React.memo(
 
         // First, add columns in user-specified order
         userOrder.forEach((columnName) => {
-          const foundColumn = baseColumns.find((col) => col.name === columnName);
+          const foundColumn = baseColumns.find((col) => col.column === columnName);
           if (foundColumn) {
             orderedColumns.push(foundColumn);
           }
         });
 
         // Then add any new columns that weren't in the saved order
         baseColumns.forEach((col) => {
-          if (!userOrder.includes(col.name)) {
+          if (!userOrder.includes(col.column)) {
             orderedColumns.push(col);
           }
         });
@@ -73,7 +73,7 @@ export const TableVis = React.memo(
       // Apply hiddenColumns from saved configuration if available
       const hiddenColumns = styleOptions?.hiddenColumns || [];
       const visibleColumnsFromConfig = sortedColumns.filter(
-        (col) => !hiddenColumns.includes(col.name)
+        (col) => !hiddenColumns.includes(col.column)
       );
 
       setVisibleColumns(visibleColumnsFromConfig.map((col) => col.column));
@@ -103,20 +103,10 @@ export const TableVis = React.memo(
 
             finalUserOrder.push(...updatedVisibleColumns);
 
-            onStyleChange({
-              visibleColumns: finalUserOrder
-                .map((id) => sortedColumns.find((col) => col.column === id)?.name ?? '')
-                .filter(Boolean),
-            });
+            onStyleChange({ visibleColumns: finalUserOrder });
           } else {
             // This is a visibility change - save hidden columns and update order if needed
-            const updates: Partial<TableChartStyle> = {
-              hiddenColumns: newHiddenColumns
-                .map((id) => sortedColumns.find((col) => col.column === id)?.name ?? '')
-                .filter(Boolean),
-            };
-
-            onStyleChange(updates);
+            onStyleChange({ hiddenColumns: newHiddenColumns });
           }
         }
       },

src/plugins/explore/public/components/visualizations/utils/field.test.ts

@@ -0,0 +1,162 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { normalizeField } from './field';
+
+describe('normalizeField', () => {
+  describe('basic functionality', () => {
+    it('should return the same string for simple field names without special characters', () => {
+      expect(normalizeField('fieldName')).toBe('fieldName');
+      expect(normalizeField('field_name')).toBe('field_name');
+      expect(normalizeField('field123')).toBe('field123');
+      expect(normalizeField('@timestamp')).toBe('@timestamp');
+    });
+
+    it('should handle empty string', () => {
+      expect(normalizeField('')).toBe('');
+    });
+
+    it('should trim whitespace', () => {
+      expect(normalizeField(' fieldName ')).toBe('fieldName');
+      expect(normalizeField('\tfieldName\n')).toBe('fieldName');
+    });
+  });
+
+  describe('dot notation handling', () => {
+    it('should replace single dot with underscore', () => {
+      expect(normalizeField('machine.os')).toBe('machine_os');
+    });
+
+    it('should replace multiple dots with underscores', () => {
+      expect(normalizeField('data.metrics.cpu')).toBe('data_metrics_cpu');
+    });
+
+    it('should handle dots at the beginning', () => {
+      expect(normalizeField('.hidden')).toBe('_hidden');
+    });
+
+    it('should handle dots at the end', () => {
+      expect(normalizeField('field.')).toBe('field_');
+    });
+
+    it('should handle consecutive dots', () => {
+      expect(normalizeField('field..name')).toBe('field__name');
+    });
+  });
+
+  describe('square bracket notation handling', () => {
+    it('should replace opening square bracket with opening parenthesis', () => {
+      expect(normalizeField('user[name')).toBe('user(name');
+    });
+
+    it('should replace closing square bracket with closing parenthesis', () => {
+      expect(normalizeField('user]name')).toBe('user)name');
+    });
+
+    it('should replace both square brackets with parentheses', () => {
+      expect(normalizeField('user[name]')).toBe('user(name)');
+    });
+
+    it('should handle array-like notation', () => {
+      expect(normalizeField('items[0]')).toBe('items(0)');
+      expect(normalizeField('data[items][0]')).toBe('data(items)(0)');
+    });
+
+    it('should handle nested bracket notation', () => {
+      expect(normalizeField('data[user[name]]')).toBe('data(user(name))');
+    });
+  });
+
+  describe('combined special characters', () => {
+    it('should handle combination of dots and brackets', () => {
+      expect(normalizeField('data.items[0]')).toBe('data_items(0)');
+      expect(normalizeField('machine.os[version]')).toBe('machine_os(version)');
+    });
+
+    it('should handle complex nested field names', () => {
+      expect(normalizeField('response.data.users[0].profile.name')).toBe(
+        'response_data_users(0)_profile_name'
+      );
+    });
+
+    it('should handle mixed notation patterns', () => {
+      expect(normalizeField('obj.arr[0].nested[key].value')).toBe('obj_arr(0)_nested(key)_value');
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should handle field names with only special characters', () => {
+      expect(normalizeField('...')).toBe('___');
+      expect(normalizeField('[]')).toBe('()');
+      expect(normalizeField('.[][')).toBe('_()(');
+    });
+
+    it('should handle very long field names', () => {
+      const longField =
+        'very.long.nested.field.name.with.many.dots.and[brackets].everywhere[0][1][2]';
+      const expected =
+        'very_long_nested_field_name_with_many_dots_and(brackets)_everywhere(0)(1)(2)';
+      expect(normalizeField(longField)).toBe(expected);
+    });
+
+    it('should handle field names with numbers and special characters', () => {
+      expect(normalizeField('field123.data[0]')).toBe('field123_data(0)');
+      expect(normalizeField('data[123].field')).toBe('data(123)_field');
+    });
+
+    it('should handle mixed brackets and dots with whitespace', () => {
+      expect(normalizeField(' data.items[0].value ')).toBe('data_items(0)_value');
+    });
+  });
+
+  describe('real-world examples', () => {
+    it('should handle common OpenSearch field patterns', () => {
+      expect(normalizeField('@timestamp')).toBe('@timestamp');
+      expect(normalizeField('host.name')).toBe('host_name');
+      expect(normalizeField('system.cpu.total.pct')).toBe('system_cpu_total_pct');
+      expect(normalizeField('kubernetes.pod.name')).toBe('kubernetes_pod_name');
+    });
+
+    it('should handle log field patterns', () => {
+      expect(normalizeField('log.level')).toBe('log_level');
+      expect(normalizeField('error.message')).toBe('error_message');
+      expect(normalizeField('http.request.method')).toBe('http_request_method');
+    });
+
+    it('should handle metric field patterns', () => {
+      expect(normalizeField('metrics.cpu.usage')).toBe('metrics_cpu_usage');
+      expect(normalizeField('system.memory.used.bytes')).toBe('system_memory_used_bytes');
+    });
+
+    it('should handle array-like field patterns', () => {
+      expect(normalizeField('tags[0]')).toBe('tags(0)');
+      expect(normalizeField('users[admin]')).toBe('users(admin)');
+      expect(normalizeField('config[database][host]')).toBe('config(database)(host)');
+    });
+
+    it('should handle complex nested structures', () => {
+      expect(normalizeField('response.data.items[0].attributes.name')).toBe(
+        'response_data_items(0)_attributes_name'
+      );
+      expect(normalizeField('logs[2023-01-01].events[error].count')).toBe(
+        'logs(2023-01-01)_events(error)_count'
+      );
+    });
+  });
+
+  describe('consistency and predictability', () => {
+    it('should be idempotent for already normalized fields', () => {
+      const alreadyNormalized = 'field_name';
+      expect(normalizeField(alreadyNormalized)).toBe(alreadyNormalized);
+    });
+
+    it('should handle repeated normalization consistently', () => {
+      const original = 'data.items[0].value';
+      const firstNormalization = normalizeField(original);
+      const secondNormalization = normalizeField(firstNormalization);
+      expect(firstNormalization).toBe(secondNormalization);
+    });
+  });
+});

src/plugins/explore/public/components/visualizations/utils/field.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Normalizes field names for use in Vega-Lite visualizations by replacing special characters
+ * with underscores to ensure compatibility with Vega-Lite's field referencing system.
+ *
+ * This function is essential for handling nested object field names (e.g., "machine.os",
+ * "user[name]") that contain characters with special meaning in Vega-Lite. By normalizing
+ * these field names, we ensure that Vega-Lite can properly reference and process the fields
+ * in visualization specifications.
+ *
+ * See: https://vega.github.io/vega-lite/docs/field.html
+ *
+ * @example
+ * normalizeField("machine.os") // returns "machine_os"
+ * normalizeField("user[name]") // returns "user(name)"
+ * normalizeField("data.metrics[0]") // returns "data_metrics(0)"
+ */
+export const normalizeField = (field: string) => {
+  return field.replace(/\./g, '_').replace(/\[/g, '(').replace(/\]/g, ')').trim();
+};

src/plugins/explore/public/components/visualizations/utils/normalize_result_rows.ts

@@ -6,6 +6,7 @@
 import { OpenSearchSearchHit } from '../../../types/doc_views_types';
 import { FIELD_TYPE_MAP } from '../constants';
 import { VisColumn, VisFieldType } from '../types';
+import { normalizeField } from './field';
 
 export const normalizeResultRows = <T = unknown>(
   rows: Array<OpenSearchSearchHit<T>>,
@@ -16,7 +17,7 @@ export const normalizeResultRows = <T = unknown>(
       id: index,
       schema: FIELD_TYPE_MAP[field.type || ''] || VisFieldType.Unknown,
       name: field.name || '',
-      column: `field-${index}`,
+      column: field.name ? normalizeField(field.name) : '',
       validValuesCount: 0,
       uniqueValuesCount: 0,
     };

src/plugins/explore/public/components/visualizations/visualization_builder.test.ts

@@ -466,7 +466,7 @@ describe('VisualizationBuilder', () => {
     expect(builder.data$.value).toEqual({
       categoricalColumns: [
         {
-          column: 'field-1',
+          column: 'name',
           id: 1,
           name: 'name',
           schema: 'categorical',
@@ -477,7 +477,7 @@ describe('VisualizationBuilder', () => {
       dateColumns: [],
       numericalColumns: [
         {
-          column: 'field-0',
+          column: 'age',
           id: 0,
           name: 'age',
           schema: 'numerical',
@@ -487,8 +487,8 @@ describe('VisualizationBuilder', () => {
       ],
       transformedData: [
         {
-          'field-0': 10,
-          'field-1': 'name',
+          age: 10,
+          name: 'name',
         },
       ],
     });