diff --git a/changelogs/fragments/10917.yml b/changelogs/fragments/10917.yml new file mode 100644 index 000000000000..c32c77203494 --- /dev/null +++ b/changelogs/fragments/10917.yml @@ -0,0 +1,2 @@ +refactor: +- Implement field normalization for Vega-Lite compatibility ([#10917](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/10917)) \ No newline at end of file diff --git a/src/plugins/explore/public/components/visualizations/table/table_vis.tsx b/src/plugins/explore/public/components/visualizations/table/table_vis.tsx index 0ff400049d66..3ae63ec8d162 100644 --- a/src/plugins/explore/public/components/visualizations/table/table_vis.tsx +++ b/src/plugins/explore/public/components/visualizations/table/table_vis.tsx @@ -45,7 +45,7 @@ export const TableVis = React.memo( // First, add columns in user-specified order userOrder.forEach((columnName) => { - const foundColumn = baseColumns.find((col) => col.name === columnName); + const foundColumn = baseColumns.find((col) => col.column === columnName); if (foundColumn) { orderedColumns.push(foundColumn); } @@ -53,7 +53,7 @@ export const TableVis = React.memo( // Then add any new columns that weren't in the saved order baseColumns.forEach((col) => { - if (!userOrder.includes(col.name)) { + if (!userOrder.includes(col.column)) { orderedColumns.push(col); } }); @@ -73,7 +73,7 @@ export const TableVis = React.memo( // Apply hiddenColumns from saved configuration if available const hiddenColumns = styleOptions?.hiddenColumns || []; const visibleColumnsFromConfig = sortedColumns.filter( - (col) => !hiddenColumns.includes(col.name) + (col) => !hiddenColumns.includes(col.column) ); setVisibleColumns(visibleColumnsFromConfig.map((col) => col.column)); @@ -103,20 +103,10 @@ export const TableVis = React.memo( finalUserOrder.push(...updatedVisibleColumns); - onStyleChange({ - visibleColumns: finalUserOrder - .map((id) => sortedColumns.find((col) => col.column === id)?.name ?? '') - .filter(Boolean), - }); + onStyleChange({ visibleColumns: finalUserOrder }); } else { // This is a visibility change - save hidden columns and update order if needed - const updates: Partial = { - hiddenColumns: newHiddenColumns - .map((id) => sortedColumns.find((col) => col.column === id)?.name ?? '') - .filter(Boolean), - }; - - onStyleChange(updates); + onStyleChange({ hiddenColumns: newHiddenColumns }); } } }, diff --git a/src/plugins/explore/public/components/visualizations/utils/field.test.ts b/src/plugins/explore/public/components/visualizations/utils/field.test.ts new file mode 100644 index 000000000000..255c6fe44829 --- /dev/null +++ b/src/plugins/explore/public/components/visualizations/utils/field.test.ts @@ -0,0 +1,162 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { normalizeField } from './field'; + +describe('normalizeField', () => { + describe('basic functionality', () => { + it('should return the same string for simple field names without special characters', () => { + expect(normalizeField('fieldName')).toBe('fieldName'); + expect(normalizeField('field_name')).toBe('field_name'); + expect(normalizeField('field123')).toBe('field123'); + expect(normalizeField('@timestamp')).toBe('@timestamp'); + }); + + it('should handle empty string', () => { + expect(normalizeField('')).toBe(''); + }); + + it('should trim whitespace', () => { + expect(normalizeField(' fieldName ')).toBe('fieldName'); + expect(normalizeField('\tfieldName\n')).toBe('fieldName'); + }); + }); + + describe('dot notation handling', () => { + it('should replace single dot with underscore', () => { + expect(normalizeField('machine.os')).toBe('machine_os'); + }); + + it('should replace multiple dots with underscores', () => { + expect(normalizeField('data.metrics.cpu')).toBe('data_metrics_cpu'); + }); + + it('should handle dots at the beginning', () => { + expect(normalizeField('.hidden')).toBe('_hidden'); + }); + + it('should handle dots at the end', () => { + expect(normalizeField('field.')).toBe('field_'); + }); + + it('should handle consecutive dots', () => { + expect(normalizeField('field..name')).toBe('field__name'); + }); + }); + + describe('square bracket notation handling', () => { + it('should replace opening square bracket with opening parenthesis', () => { + expect(normalizeField('user[name')).toBe('user(name'); + }); + + it('should replace closing square bracket with closing parenthesis', () => { + expect(normalizeField('user]name')).toBe('user)name'); + }); + + it('should replace both square brackets with parentheses', () => { + expect(normalizeField('user[name]')).toBe('user(name)'); + }); + + it('should handle array-like notation', () => { + expect(normalizeField('items[0]')).toBe('items(0)'); + expect(normalizeField('data[items][0]')).toBe('data(items)(0)'); + }); + + it('should handle nested bracket notation', () => { + expect(normalizeField('data[user[name]]')).toBe('data(user(name))'); + }); + }); + + describe('combined special characters', () => { + it('should handle combination of dots and brackets', () => { + expect(normalizeField('data.items[0]')).toBe('data_items(0)'); + expect(normalizeField('machine.os[version]')).toBe('machine_os(version)'); + }); + + it('should handle complex nested field names', () => { + expect(normalizeField('response.data.users[0].profile.name')).toBe( + 'response_data_users(0)_profile_name' + ); + }); + + it('should handle mixed notation patterns', () => { + expect(normalizeField('obj.arr[0].nested[key].value')).toBe('obj_arr(0)_nested(key)_value'); + }); + }); + + describe('edge cases', () => { + it('should handle field names with only special characters', () => { + expect(normalizeField('...')).toBe('___'); + expect(normalizeField('[]')).toBe('()'); + expect(normalizeField('.[][')).toBe('_()('); + }); + + it('should handle very long field names', () => { + const longField = + 'very.long.nested.field.name.with.many.dots.and[brackets].everywhere[0][1][2]'; + const expected = + 'very_long_nested_field_name_with_many_dots_and(brackets)_everywhere(0)(1)(2)'; + expect(normalizeField(longField)).toBe(expected); + }); + + it('should handle field names with numbers and special characters', () => { + expect(normalizeField('field123.data[0]')).toBe('field123_data(0)'); + expect(normalizeField('data[123].field')).toBe('data(123)_field'); + }); + + it('should handle mixed brackets and dots with whitespace', () => { + expect(normalizeField(' data.items[0].value ')).toBe('data_items(0)_value'); + }); + }); + + describe('real-world examples', () => { + it('should handle common OpenSearch field patterns', () => { + expect(normalizeField('@timestamp')).toBe('@timestamp'); + expect(normalizeField('host.name')).toBe('host_name'); + expect(normalizeField('system.cpu.total.pct')).toBe('system_cpu_total_pct'); + expect(normalizeField('kubernetes.pod.name')).toBe('kubernetes_pod_name'); + }); + + it('should handle log field patterns', () => { + expect(normalizeField('log.level')).toBe('log_level'); + expect(normalizeField('error.message')).toBe('error_message'); + expect(normalizeField('http.request.method')).toBe('http_request_method'); + }); + + it('should handle metric field patterns', () => { + expect(normalizeField('metrics.cpu.usage')).toBe('metrics_cpu_usage'); + expect(normalizeField('system.memory.used.bytes')).toBe('system_memory_used_bytes'); + }); + + it('should handle array-like field patterns', () => { + expect(normalizeField('tags[0]')).toBe('tags(0)'); + expect(normalizeField('users[admin]')).toBe('users(admin)'); + expect(normalizeField('config[database][host]')).toBe('config(database)(host)'); + }); + + it('should handle complex nested structures', () => { + expect(normalizeField('response.data.items[0].attributes.name')).toBe( + 'response_data_items(0)_attributes_name' + ); + expect(normalizeField('logs[2023-01-01].events[error].count')).toBe( + 'logs(2023-01-01)_events(error)_count' + ); + }); + }); + + describe('consistency and predictability', () => { + it('should be idempotent for already normalized fields', () => { + const alreadyNormalized = 'field_name'; + expect(normalizeField(alreadyNormalized)).toBe(alreadyNormalized); + }); + + it('should handle repeated normalization consistently', () => { + const original = 'data.items[0].value'; + const firstNormalization = normalizeField(original); + const secondNormalization = normalizeField(firstNormalization); + expect(firstNormalization).toBe(secondNormalization); + }); + }); +}); diff --git a/src/plugins/explore/public/components/visualizations/utils/field.ts b/src/plugins/explore/public/components/visualizations/utils/field.ts new file mode 100644 index 000000000000..ad0b5bebc5ca --- /dev/null +++ b/src/plugins/explore/public/components/visualizations/utils/field.ts @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Normalizes field names for use in Vega-Lite visualizations by replacing special characters + * with underscores to ensure compatibility with Vega-Lite's field referencing system. + * + * This function is essential for handling nested object field names (e.g., "machine.os", + * "user[name]") that contain characters with special meaning in Vega-Lite. By normalizing + * these field names, we ensure that Vega-Lite can properly reference and process the fields + * in visualization specifications. + * + * See: https://vega.github.io/vega-lite/docs/field.html + * + * @example + * normalizeField("machine.os") // returns "machine_os" + * normalizeField("user[name]") // returns "user(name)" + * normalizeField("data.metrics[0]") // returns "data_metrics(0)" + */ +export const normalizeField = (field: string) => { + return field.replace(/\./g, '_').replace(/\[/g, '(').replace(/\]/g, ')').trim(); +}; diff --git a/src/plugins/explore/public/components/visualizations/utils/normalize_result_rows.ts b/src/plugins/explore/public/components/visualizations/utils/normalize_result_rows.ts index 6db1d24fab3f..c759b4524930 100644 --- a/src/plugins/explore/public/components/visualizations/utils/normalize_result_rows.ts +++ b/src/plugins/explore/public/components/visualizations/utils/normalize_result_rows.ts @@ -6,6 +6,7 @@ import { OpenSearchSearchHit } from '../../../types/doc_views_types'; import { FIELD_TYPE_MAP } from '../constants'; import { VisColumn, VisFieldType } from '../types'; +import { normalizeField } from './field'; export const normalizeResultRows = ( rows: Array>, @@ -16,7 +17,7 @@ export const normalizeResultRows = ( id: index, schema: FIELD_TYPE_MAP[field.type || ''] || VisFieldType.Unknown, name: field.name || '', - column: `field-${index}`, + column: field.name ? normalizeField(field.name) : '', validValuesCount: 0, uniqueValuesCount: 0, }; diff --git a/src/plugins/explore/public/components/visualizations/visualization_builder.test.ts b/src/plugins/explore/public/components/visualizations/visualization_builder.test.ts index 5aa0567b75c8..883f83cdb980 100644 --- a/src/plugins/explore/public/components/visualizations/visualization_builder.test.ts +++ b/src/plugins/explore/public/components/visualizations/visualization_builder.test.ts @@ -466,7 +466,7 @@ describe('VisualizationBuilder', () => { expect(builder.data$.value).toEqual({ categoricalColumns: [ { - column: 'field-1', + column: 'name', id: 1, name: 'name', schema: 'categorical', @@ -477,7 +477,7 @@ describe('VisualizationBuilder', () => { dateColumns: [], numericalColumns: [ { - column: 'field-0', + column: 'age', id: 0, name: 'age', schema: 'numerical', @@ -487,8 +487,8 @@ describe('VisualizationBuilder', () => { ], transformedData: [ { - 'field-0': 10, - 'field-1': 'name', + age: 10, + name: 'name', }, ], });