support to handle large nominal fields on x/y axes

Chenglong-MS · Chenglong-MS · commit 400de05904e3 · 2025-02-26T14:29:00.000-08:00
diff --git a/src/app/utils.tsx b/src/app/utils.tsx
@@ -240,7 +240,13 @@ export function baseTableToExtTable(table: any[], derivedFields: FieldItem[], al
 }
 
 
-export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key in Channel]: EncodingItem; }, allFields: FieldItem[], workingTable: any[]) => {
+export const assembleVegaChart = (
+    chartType: string, 
+    encodingMap: { [key in Channel]: EncodingItem; }, 
+    conceptShelfItems: FieldItem[], 
+    workingTable: any[],
+    maxNominalValues: number = 68
+) => {
 
     if (chartType == "Table") {
         return ["Table", undefined];
@@ -250,7 +256,6 @@ export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key i
     //console.log(chartTemplate);
 
     let vgObj = JSON.parse(JSON.stringify(chartTemplate.template));
-    const baseTableSchemaObj: any = {};
 
     for (const [channel, encoding] of Object.entries(encodingMap)) {
 
@@ -260,30 +265,8 @@ export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key i
             encodingObj["scale"] = {"type": "sqrt", "zero": true};
         }
 
-        const field = encoding.fieldID ? _.find(allFields, (f) => f.id === encoding.fieldID) : undefined;
+        const field = encoding.fieldID ? _.find(conceptShelfItems, (f) => f.id === encoding.fieldID) : undefined;
         if (field) {
-            //console.log(field)
-            // the synthesizer only need to see base table schema
-            let baseFields = (field.source == "derived" ? 
-                    (field.transform as ConceptTransformation).parentIDs.map((parentID) => allFields.find((f) => f.id == parentID) as FieldItem) 
-                  : [field]);
-
-            for (let baseField of baseFields) {
-                if (Object.keys(baseTableSchemaObj).includes(baseField.name)) {
-                    continue;
-                }
-                baseTableSchemaObj[baseField.name] = {
-                    channel, 
-                    dtype: getDType(baseField.type, workingTable.map(r => r[baseField.name])),
-                    name: baseField.name,
-                    original: baseField.source == "original",
-                    // domain: {
-                    //     values: [...new Set(baseField.domain.values)],
-                    //     is_complete: baseField.domain.isComplete
-                    // },
-                };
-            }
-
             // create the encoding
             encodingObj["field"] = field.name;
             encodingObj["type"] = getDType(field.type, workingTable.map(r => r[field.name]));
@@ -426,16 +409,8 @@ export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key i
         vgObj = chartTemplate.postProcessor(vgObj, workingTable);
     }
 
-    // console.log(JSON.stringify(vgObj))
-
-    return [vgObj, baseTableSchemaObj];
-}
-
-export const assembleChart = (chart: Chart, conceptShelfItems: FieldItem[], dataValues: any[]) => {
-
-    let vgSpec: any = instantiateVegaTemplate(chart.chartType, chart.encodingMap, conceptShelfItems, dataValues)[0];
-
-    let values = JSON.parse(JSON.stringify(dataValues));
+    // this is the data that will be assembled into the vega chart
+    let values = JSON.parse(JSON.stringify(workingTable));
     values = values.map((r: any) => { 
         let keys = Object.keys(r);
         let temporalKeys = keys.filter((k: string) => conceptShelfItems.some(concept => concept.name == k && (concept.type == "date" || concept.semanticType == "Year")));
@@ -444,9 +419,75 @@ export const assembleChart = (chart: Chart, conceptShelfItems: FieldItem[], data
         }
         return r;
     })
-    return {...vgSpec, data: {values: values}}
+
+    // Handle nominal axes with many entries
+    for (const channel of ['x', 'y']) {
+        const encoding = vgObj.encoding?.[channel];
+        if (encoding?.type === 'nominal') {
+            const fieldName = encoding.field;
+            const uniqueValues = [...new Set(workingTable.map(r => r[fieldName]))];
+            
+            if (uniqueValues.length > maxNominalValues) {
+                const oppositeChannel = channel === 'x' ? 'y' : 'x';
+                const oppositeEncoding = vgObj.encoding?.[oppositeChannel];
+                
+                let valuesToKeep: any[];
+                if (oppositeEncoding?.type === 'quantitative') {
+                    // Sort by the quantitative field and take top maxNominalValues
+                    const quantField = oppositeEncoding.field;
+                    valuesToKeep = uniqueValues
+                        .map(val => ({
+                            value: val,
+                            sum: workingTable
+                                .filter(r => r[fieldName] === val)
+                                .reduce((sum, r) => sum + (r[quantField] || 0), 0)
+                        }))
+                        .sort((a, b) => b.sum - a.sum)
+                        .slice(0, maxNominalValues)
+                        .map(v => v.value);
+                } else {
+                    // If no quantitative axis, just take first maxNominalValues
+                    valuesToKeep = uniqueValues.slice(0, maxNominalValues);
+                }
+
+                // Filter the working table
+                const omittedCount = uniqueValues.length - maxNominalValues;
+                const placeholder = `...${omittedCount} items omitted`;
+                values = values.filter((row: any) => valuesToKeep.includes(row[fieldName]));
+
+                // Add text formatting configuration
+                if (!encoding.axis) {
+                    encoding.axis = {};
+                }
+                encoding.axis.labelColor = {
+                    condition: {
+                        test: `datum.label == '${placeholder}'`,
+                        value: "#999999"
+                    },
+                    value: "#000000" // default color for other labels
+                };
+                encoding.axis.labelFont = {
+                    condition: {
+                        test: `datum.label == '${placeholder}'`,
+                        value: "italic"
+                    },
+                    value: "normal" // default font style for other labels
+                };
+
+                // Add placeholder to domain
+                if (!encoding.scale) {
+                    encoding.scale = {};
+                }
+                encoding.scale.domain = [...valuesToKeep, placeholder]
+            }
+        }
+    }
+
+    return {...vgObj, data: {values: values}}
 }
 
+
+
 export const adaptChart = (chart: Chart, targetTemplate: ChartTemplate) => {
 
     let discardedChannels = Object.entries(chart.encodingMap).filter(([ch, enc]) => {
diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx
@@ -24,7 +24,7 @@ import { VegaLite } from 'react-vega'
 import '../scss/VisualizationView.scss';
 import { useDispatch, useSelector } from 'react-redux';
 import { DataFormulatorState, dfActions } from '../app/dfSlice';
-import { assembleChart, baseTableToExtTable, getTriggers } from '../app/utils';
+import { assembleVegaChart, baseTableToExtTable, getTriggers } from '../app/utils';
 import { Chart, DictTable, EncodingItem, Trigger } from "../components/ComponentType";
 
 import DeleteIcon from '@mui/icons-material/Delete';
@@ -367,7 +367,7 @@ export const DataThread: FC<{}> = function ({ }) {
         }
 
         // prepare the chart to be rendered
-        let assembledChart = assembleChart(chart, conceptShelfItems, extTable);
+        let assembledChart = assembleVegaChart(chart.chartType, chart.encodingMap, conceptShelfItems, extTable, 20);
         assembledChart["background"] = "transparent";
 
         // Temporary fix, down sample the dataset
diff --git a/src/views/DerivedDataDialog.tsx b/src/views/DerivedDataDialog.tsx
@@ -20,7 +20,7 @@ import {
 
 import React from 'react';
 
-import { baseTableToExtTable, instantiateVegaTemplate } from '../app/utils';
+import { baseTableToExtTable, assembleVegaChart } from '../app/utils';
 import { Chart } from '../components/ComponentType';
 import { findBaseFields } from './ViewUtils';
 import { useSelector } from 'react-redux';
@@ -74,10 +74,7 @@ export const DerivedDataDialog: FC<DerivedDataDialogProps> = function DerivedDat
                 let toDeriveFields = derivedFields.filter(f => f.name != "").filter(f => findBaseFields(f, conceptShelfItems).every(f2 => table.names.includes(f2.name)))
                 let extTable = baseTableToExtTable(JSON.parse(JSON.stringify(table.rows)), toDeriveFields, conceptShelfItems);
             
-                let vgSpec: any = instantiateVegaTemplate(chart.chartType, chart.encodingMap, conceptShelfItems, extTable)[0];
-            
-            
-                let assembledChart = {...vgSpec, data: {values: JSON.parse(JSON.stringify(extTable))}};
+                let assembledChart: any = assembleVegaChart(chart.chartType, chart.encodingMap, conceptShelfItems, extTable)[0];
                 assembledChart["background"] = "transparent";
                 // chart["autosize"] = {
                 //     "type": "fit",
diff --git a/src/views/EncodingShelfCard.tsx b/src/views/EncodingShelfCard.tsx
@@ -89,9 +89,6 @@ let selectBaseTables = (activeFields: FieldItem[], conceptShelfItems: FieldItem[
         baseTables.push(...tablesToAdd.filter(t => !baseTables.map(t2 => t2.id).includes(t.id)));
     }
 
-    console.log("selectBaseTables baseTables");
-    console.log(baseTables);
-
     return baseTables;
 }
 
@@ -430,19 +427,12 @@ export const EncodingShelfCard: FC<EncodingShelfCardProps> = function ({ chartId
         if (mode == "formulate" && currentTable.derive?.dialog) {
                 let sourceTableIds = currentTable.derive?.source;
 
-                console.log("sourceTableIds ---- and ---- baseTableIds");
-                console.log(sourceTableIds);
-                console.log(actionTableIds);
-
                 // Compare if source and base table IDs are different
                 if (!sourceTableIds.every(id => actionTableIds.includes(id)) || 
                     !actionTableIds.every(id => sourceTableIds.includes(id))) {
                     
                     let additionalMessages = currentTable.derive.dialog;
 
-                    console.log("in here");
-                    console.log(additionalMessages);
-
                     // in this case, because table ids has changed, we need to use the additional messages and reformulate
                     messageBody = JSON.stringify({
                         token: token,
@@ -490,8 +480,7 @@ export const EncodingShelfCard: FC<EncodingShelfCardProps> = function ({ chartId
             .then((data) => {
                 
                 dispatch(dfActions.changeChartRunningStatus({chartId, status: false}))
-                console.log(data);
-                console.log(token);
+
                 if (data.results.length > 0) {
                     if (data["token"] == token) {
                         let candidates = data["results"].filter((item: any) => {
diff --git a/src/views/EncodingShelfThread.tsx b/src/views/EncodingShelfThread.tsx
@@ -27,7 +27,7 @@ import '../scss/EncodingShelf.scss';
 import { createDictTable, DictTable } from "../components/ComponentType";
 import embed from 'vega-embed';
 
-import { getTriggers, getUrls, assembleChart, resolveChartFields } from '../app/utils';
+import { getTriggers, getUrls, assembleVegaChart, resolveChartFields } from '../app/utils';
 
 import { getChartTemplate } from '../components/ChartTemplates';
 import { chartAvailabilityCheck, generateChartSkeleton } from './VisualizationView';
@@ -74,7 +74,7 @@ export let ChartElementFC: FC<{chart: Chart, tableRows: any[], boxWidth?: number
     // }
 
     // prepare the chart to be rendered
-    let assembledChart = assembleChart(chart, conceptShelfItems, tableRows);
+    let assembledChart = assembleVegaChart(chart.chartType, chart.encodingMap, conceptShelfItems, tableRows);
     assembledChart["background"] = "transparent";
     // chart["autosize"] = {
     //     "type": "fit",
diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx
@@ -36,7 +36,7 @@ import AnimateOnChange from 'react-animate-on-change'
 import '../scss/VisualizationView.scss';
 import { useDispatch, useSelector } from 'react-redux';
 import { DataFormulatorState, dfActions } from '../app/dfSlice';
-import { assembleChart, baseTableToExtTable  } from '../app/utils';
+import { assembleVegaChart, baseTableToExtTable  } from '../app/utils';
 import { Chart, EncodingItem, EncodingMap, FieldItem } from '../components/ComponentType';
 import { DictTable } from "../components/ComponentType";
 
@@ -311,7 +311,7 @@ export const ChartEditorFC: FC<{  cachedCandidates: DictTable[],
 
         element = <Box id={id} key={`focused-chart`} ></Box>    
 
-        let assembledChart = assembleChart(chart, conceptShelfItems, extTable);
+        let assembledChart = assembleVegaChart(chart.chartType, chart.encodingMap, conceptShelfItems, extTable);
         assembledChart['resize'] = true;
 
         embed('#' + id, { ...assembledChart }, { actions: true, renderer: "svg" }).then(function (result) {
@@ -796,7 +796,7 @@ export const VisualizationViewFC: FC<VisPanelProps> = function VisualizationView
                             key="skeleton" onClick={setIndexFunc}>{generateChartSkeleton(chartTemplate?.icon)}</Box>;
             }
 
-            let assembledChart = assembleChart(chart, conceptShelfItems, extTable);
+            let assembledChart = assembleVegaChart(chart.chartType, chart.encodingMap, conceptShelfItems, extTable);
 
             const id = `chart-element-${index}`;
 
