fix(ui/column-stats): fix unopenable side panel for nested column stats (#14874)

Author: acrylJonny

datahub-web-react/src/app/entityV2/dataset/profile/schema/utils/utils.ts

@@ -58,10 +58,17 @@ export function downgradeV2FieldPath(fieldPath?: string | null) {
 
     const cleanedFieldPath = fieldPath.replace(KEY_SCHEMA_PREFIX, '').replace(VERSION_PREFIX, '');
 
-    // strip out all annotation segments
+    // Remove all bracket annotations (e.g., [0], [*], [key]) from the field path
     return cleanedFieldPath
         .split('.')
-        .map((segment) => (segment.startsWith('[') ? null : segment))
+        .map((segment) => {
+            // Remove segments that are entirely brackets (e.g., "[0]", "[*]")
+            if (segment.startsWith('[') && segment.endsWith(']')) {
+                return null;
+            }
+            // Remove bracket suffixes from segments (e.g., "addresses[0]" -> "addresses")
+            return segment.replace(/\[[^\]]*\]/g, '');
+        })
         .filter(Boolean)
         .join('.');
 }

datahub-web-react/src/app/entityV2/shared/tabs/Dataset/Stats/StatsTabV2/columnStats/ColumnStatsTable.tsx

@@ -2,14 +2,36 @@ import { Table, Text } from '@components';
 import React, { useEffect, useMemo, useRef, useState } from 'react';
 import styled from 'styled-components';
 
+import { ExtendedSchemaFields } from '@app/entityV2/dataset/profile/schema/utils/types';
 import SchemaFieldDrawer from '@app/entityV2/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/SchemaFieldDrawer';
 import { useGetEntityWithSchema } from '@app/entityV2/shared/tabs/Dataset/Schema/useGetEntitySchema';
 import useKeyboardControls from '@app/entityV2/shared/tabs/Dataset/Schema/useKeyboardControls';
 import { decimalToPercentStr } from '@app/entityV2/shared/tabs/Dataset/Schema/utils/statsUtil';
+import {
+    createStatsOnlyField,
+    filterColumnStatsByQuery,
+    flattenFields,
+    handleRowScrollIntoView,
+    mapToSchemaFields,
+} from '@app/entityV2/shared/tabs/Dataset/Stats/StatsTabV2/columnStats/ColumnStatsTable.utils';
 import { useGetColumnStatsColumns } from '@app/entityV2/shared/tabs/Dataset/Stats/StatsTabV2/columnStats/useGetColumnStatsColumns';
 import { isPresent } from '@app/entityV2/shared/tabs/Dataset/Stats/StatsTabV2/utils';
 import { downgradeV2FieldPath, groupByFieldPath } from '@src/app/entityV2/dataset/profile/schema/utils/utils';
-import { DatasetFieldProfile } from '@src/types.generated';
+
+// Local type definitions since generated types aren't available
+interface DatasetFieldProfile {
+    fieldPath: string;
+    nullCount?: number | null;
+    nullProportion?: number | null;
+    uniqueCount?: number | null;
+    min?: string | null;
+    max?: string | null;
+}
+
+// Extended type that includes the fieldPath property we know exists
+interface ExtendedSchemaFieldsWithFieldPath extends ExtendedSchemaFields {
+    fieldPath: string;
+}
 
 const EmptyContainer = styled.div`
     display: flex;
@@ -22,38 +44,49 @@ const EmptyContainer = styled.div`
 `;
 
 interface Props {
-    columnStats: Array<DatasetFieldProfile>;
+    columnStats: DatasetFieldProfile[];
     searchQuery: string;
 }
 
-const ColumnStatsTable = ({ columnStats, searchQuery }: Props) => {
+function ColumnStatsTable({ columnStats, searchQuery }: Props) {
     const { entityWithSchema } = useGetEntityWithSchema();
-    const schemaMetadata: any = entityWithSchema?.schemaMetadata || undefined;
-    const editableSchemaMetadata: any = entityWithSchema?.editableSchemaMetadata || undefined;
-    const fields = schemaMetadata?.fields;
+    const rawFields = entityWithSchema?.schemaMetadata?.fields;
+
+    const fields = useMemo(() => {
+        return rawFields ? mapToSchemaFields(rawFields) : [];
+    }, [rawFields]);
 
     const columnStatsTableData = useMemo(
         () =>
-            columnStats.map((doc) => ({
-                column: downgradeV2FieldPath(doc.fieldPath),
-                type: fields?.find((field) => field.fieldPath === doc.fieldPath)?.type,
-                nullPercentage: isPresent(doc.nullProportion) && decimalToPercentStr(doc.nullProportion, 2),
-                uniqueValues: isPresent(doc.uniqueCount) && doc.uniqueCount.toString(),
-                min: doc.min,
-                max: doc.max,
+            columnStats.map((stat) => ({
+                column: downgradeV2FieldPath(stat.fieldPath),
+                originalFieldPath: stat.fieldPath,
+                type: fields.find((field) => field.fieldPath === stat.fieldPath)?.type,
+                nullPercentage: isPresent(stat.nullProportion) && decimalToPercentStr(stat.nullProportion, 2),
+                uniqueValues: isPresent(stat.uniqueCount) && stat.uniqueCount.toString(),
+                min: stat.min,
+                max: stat.max,
             })) || [],
         [columnStats, fields],
     );
 
     const [expandedDrawerFieldPath, setExpandedDrawerFieldPath] = useState<string | null>(null);
 
     const rows = useMemo(() => {
-        return groupByFieldPath(fields);
-    }, [fields]);
+        const schemaFields = fields;
 
-    const filteredData = columnStatsTableData.filter((columnStat) =>
-        columnStat.column?.toLowerCase().includes(searchQuery.toLowerCase()),
-    );
+        // Add fields from column stats that don't exist in schema
+        const statsOnlyFields = columnStats
+            .filter((stat) => !schemaFields.find((field) => field.fieldPath === stat.fieldPath))
+            .map(createStatsOnlyField);
+
+        const combinedFields = [...schemaFields, ...statsOnlyFields];
+        const groupedFields = groupByFieldPath(combinedFields as any);
+
+        return flattenFields(groupedFields);
+    }, [fields, columnStats]);
+
+    const filteredData = filterColumnStatsByQuery(columnStatsTableData, searchQuery);
 
     const columnStatsColumns = useGetColumnStatsColumns({
         tableData: columnStatsTableData,
@@ -72,7 +105,9 @@ const ColumnStatsTable = ({ columnStats, searchQuery }: Props) => {
 
     useEffect(() => {
         if (expandedDrawerFieldPath) {
-            const selectedIndex = rows.findIndex((row) => row.fieldPath === expandedDrawerFieldPath);
+            const selectedIndex = rows.findIndex(
+                (row) => (row as ExtendedSchemaFieldsWithFieldPath).fieldPath === expandedDrawerFieldPath,
+            );
             const row = rowRefs.current[selectedIndex];
             const header = headerRef.current;
 
@@ -83,20 +118,9 @@ const ColumnStatsTable = ({ columnStats, searchQuery }: Props) => {
                     block: 'nearest',
                 });
             }
-            // To bring the row hidden behind the fixed header into view fully
+            // Adjust scroll position to account for fixed header
             setTimeout(() => {
-                if (row && header) {
-                    const rowRect = row.getBoundingClientRect();
-                    const headerRect = header.getBoundingClientRect();
-                    const rowTop = rowRect.top;
-                    const headerBottom = headerRect.bottom;
-                    const scrollContainer = row.closest('table')?.parentElement;
-
-                    if (scrollContainer && rowTop < headerBottom) {
-                        const scrollAmount = headerBottom - rowTop;
-                        scrollContainer.scrollTop -= scrollAmount;
-                    }
-                }
+                handleRowScrollIntoView(row, header);
             }, 100);
         }
     }, [expandedDrawerFieldPath, rows]);
@@ -112,11 +136,13 @@ const ColumnStatsTable = ({ columnStats, searchQuery }: Props) => {
     }
 
     const getRowClassName = (record) => {
-        return expandedDrawerFieldPath === record.column ? 'selected-row' : '';
+        return expandedDrawerFieldPath === record.originalFieldPath ? 'selected-row' : '';
     };
 
     const onRowClick = (record) => {
-        setExpandedDrawerFieldPath(expandedDrawerFieldPath === record.column ? null : record.column);
+        setExpandedDrawerFieldPath(
+            expandedDrawerFieldPath === record.originalFieldPath ? null : record.originalFieldPath,
+        );
     };
 
     return (
@@ -132,11 +158,11 @@ const ColumnStatsTable = ({ columnStats, searchQuery }: Props) => {
                 rowRefs={rowRefs}
                 headerRef={headerRef}
             />
-            {!!fields && (
+            {fields.length > 0 && (
                 <SchemaFieldDrawer
-                    schemaFields={fields}
+                    schemaFields={fields as any}
                     expandedDrawerFieldPath={expandedDrawerFieldPath}
-                    editableSchemaMetadata={editableSchemaMetadata}
+                    editableSchemaMetadata={entityWithSchema?.editableSchemaMetadata as any}
                     setExpandedDrawerFieldPath={setExpandedDrawerFieldPath}
                     displayedRows={rows}
                     defaultSelectedTabName="Statistics"
@@ -146,6 +172,6 @@ const ColumnStatsTable = ({ columnStats, searchQuery }: Props) => {
             )}
         </>
     );
-};
+}
 
 export default ColumnStatsTable;

datahub-web-react/src/app/entityV2/shared/tabs/Dataset/Stats/StatsTabV2/columnStats/ColumnStatsTable.utils.ts

@@ -0,0 +1,129 @@
+import { ExtendedSchemaFields } from '@app/entityV2/dataset/profile/schema/utils/types';
+
+// Type definitions for safe type mapping
+interface SchemaField {
+    fieldPath: string;
+    type?: any;
+    nativeDataType?: string | null;
+    schemaFieldEntity?: any;
+    nullable: boolean;
+    recursive?: boolean;
+    description?: string | null;
+    children?: SchemaField[];
+}
+
+// For DatasetFieldProfile, we'll define it locally since the generated types aren't available
+interface DatasetFieldProfile {
+    fieldPath: string;
+    nullCount?: number | null;
+    nullProportion?: number | null;
+    uniqueCount?: number | null;
+    min?: string | null;
+    max?: string | null;
+}
+
+/**
+ * Infers if a field is nullable based on column statistics.
+ * Uses null count or proportion data when available, defaults to nullable.
+ */
+export function inferIsFieldNullable(stat: DatasetFieldProfile): boolean {
+    if (stat.nullCount != null) {
+        return stat.nullCount > 0;
+    }
+
+    if (stat.nullProportion != null) {
+        return stat.nullProportion > 0;
+    }
+
+    return true; // Default to nullable when data unavailable
+}
+
+/**
+ * Safely maps unknown field data to SchemaField type.
+ */
+export function mapToSchemaField(field: unknown): SchemaField {
+    if (!field || typeof field !== 'object') {
+        throw new Error('Invalid field data provided to mapToSchemaField');
+    }
+
+    const fieldObj = field as Record<string, any>;
+
+    return {
+        fieldPath: fieldObj.fieldPath || '',
+        type: fieldObj.type || null,
+        nativeDataType: fieldObj.nativeDataType || null,
+        schemaFieldEntity: fieldObj.schemaFieldEntity || null,
+        nullable: fieldObj.nullable ?? false,
+        recursive: fieldObj.recursive || false,
+        description: fieldObj.description || null,
+        children: fieldObj.children || undefined,
+    };
+}
+
+/**
+ * Safely maps an array of unknown field data to SchemaField array.
+ */
+export function mapToSchemaFields(fields: unknown[]): SchemaField[] {
+    if (!Array.isArray(fields)) {
+        return [];
+    }
+
+    return fields.map(mapToSchemaField);
+}
+
+/**
+ * Creates a stats-only field object for fields that exist in column stats but not in schema.
+ */
+export function createStatsOnlyField(stat: DatasetFieldProfile): SchemaField {
+    return {
+        fieldPath: stat.fieldPath,
+        type: null,
+        nativeDataType: null,
+        schemaFieldEntity: null,
+        nullable: inferIsFieldNullable(stat),
+        recursive: false,
+        description: null,
+    };
+}
+
+/**
+ * Flattens nested field hierarchies to enable drawer field path matching.
+ */
+export function flattenFields(fieldList: ExtendedSchemaFields[]): ExtendedSchemaFields[] {
+    const result: ExtendedSchemaFields[] = [];
+    fieldList.forEach((field) => {
+        result.push(field);
+        if (field.children) {
+            result.push(...flattenFields(field.children));
+        }
+    });
+    return result;
+}
+
+/**
+ * Handles scroll adjustment when a row is selected to ensure it's visible.
+ */
+export function handleRowScrollIntoView(row: HTMLTableRowElement | undefined, header: HTMLTableSectionElement | null) {
+    if (!row || !header) return;
+
+    const rowRect = row.getBoundingClientRect();
+    const headerRect = header.getBoundingClientRect();
+    const rowTop = rowRect.top;
+    const headerBottom = headerRect.bottom;
+    const scrollContainer = row.closest('table')?.parentElement;
+
+    if (scrollContainer && rowTop < headerBottom) {
+        const scrollAmount = headerBottom - rowTop;
+        scrollContainer.scrollTop -= scrollAmount;
+    }
+}
+
+/**
+ * Filters column stats data based on search query.
+ */
+export function filterColumnStatsByQuery(data: any[], query: string) {
+    if (!query.trim()) return data;
+
+    const lowercaseQuery = query.toLowerCase();
+    return data.filter((columnStat) => columnStat.column?.toLowerCase().includes(lowercaseQuery));
+}