Skip to content

Commit a00c777

Browse files
Chenglong-MSChenglong-MS
committed
fix csv data load issue, and support excel table loading
1 parent b161bfc commit a00c777

File tree

4 files changed

+152
-32
lines changed

4 files changed

+152
-32
lines changed

‎package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@
4545
"vega": "^5.26.0",
4646
"vega-embed": "^6.21.0",
4747
"vega-lite": "^5.5.0",
48-
"vm-browserify": "^1.1.2"
48+
"vm-browserify": "^1.1.2",
49+
"xlsx": "^0.18.5"
4950
},
5051
"scripts": {
5152
"lint": "eslint -c eslint.config.js src/**/*.{ts,tsx} --fix",

‎src/data/utils.ts

Lines changed: 40 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@
33

44
import * as d3 from 'd3';
55
import Column from './column';
6+
import * as XLSX from 'xlsx';
67

78
import { DictTable } from '../components/ComponentType';
89
import { CoerceType, TestType, Type } from './types';
910
import { ColumnTable } from './table';
1011

11-
export const loadDataWrapper = (title: string, text: string, fileType: string): DictTable | undefined => {
12+
export const loadTextDataWrapper = (title: string, text: string, fileType: string): DictTable | undefined => {
1213

1314
let tableName = title;
1415
//let tableName = title.replace(/\.[^/.]+$/ , "");
@@ -18,8 +19,7 @@ export const loadDataWrapper = (title: string, text: string, fileType: string):
1819
table = createTableFromText(tableName, text);
1920
} else if (fileType == "application/json") {
2021
table = createTableFromFromObjectArray(tableName, JSON.parse(text));
21-
}
22-
22+
}
2323
return table;
2424
};
2525

@@ -43,7 +43,14 @@ export const createTableFromText = (title: string, text: string): DictTable | un
4343
// Should check the data file as well for the ending
4444
const isTabSeparated = tabNum / lineNum >= 1;
4545

46-
const values = isTabSeparated ? d3.tsvParse(text) : d3.csvParse(text);
46+
// Use d3.dsvFormat to create a custom parser that properly handles quoted fields
47+
// This ensures commas inside quoted fields won't be treated as delimiters
48+
const values = isTabSeparated
49+
? d3.tsvParse(text)
50+
: d3.dsvFormat(',').parse(text, row => {
51+
// Process each row to ensure proper type handling
52+
return row;
53+
});
4754

4855
return createTableFromFromObjectArray(title, values);
4956
};
@@ -145,20 +152,32 @@ export function tupleEqual(a: any[], b: any[]) {
145152
return true;
146153
}
147154

148-
// export function arrayEqual(_arr1: any[], _arr2: any[]) {
149-
// if (Array.isArray(_arr1) || !Array.isArray(_arr2) || _arr1.length !== _arr2.length) {
150-
// return false;
151-
// }
152-
153-
// // .concat() to not mutate arguments
154-
// const arr1 = _arr1.concat().sort();
155-
// const arr2 = _arr2.concat().sort();
156-
157-
// for (let i = 0; i < arr1.length; i++) {
158-
// if (arr1[i] !== arr2[i]) {
159-
// return false;
160-
// }
161-
// }
162-
163-
// return true;
164-
// }
155+
export const loadBinaryDataWrapper = (title: string, arrayBuffer: ArrayBuffer): DictTable[] => {
156+
try {
157+
// Read the Excel file
158+
const workbook = XLSX.read(arrayBuffer, { type: 'array' });
159+
160+
// Get all sheet names
161+
const sheetNames = workbook.SheetNames;
162+
163+
// Create tables for each sheet
164+
const tables: DictTable[] = [];
165+
166+
for (const sheetName of sheetNames) {
167+
// Get the worksheet
168+
const worksheet = workbook.Sheets[sheetName];
169+
170+
// Convert the worksheet to JSON
171+
const jsonData = XLSX.utils.sheet_to_json(worksheet);
172+
173+
// Create a table from the JSON data with sheet name included in the title
174+
const sheetTable = createTableFromFromObjectArray(`${title}-${sheetName}`, jsonData);
175+
tables.push(sheetTable);
176+
}
177+
178+
return tables;
179+
} catch (error) {
180+
console.error('Error processing Excel file:', error);
181+
return [];
182+
}
183+
};

‎src/views/TableSelectionView.tsx

Lines changed: 52 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import { DictTable } from "../components/ComponentType";
1717

1818
import DeleteIcon from '@mui/icons-material/Delete';
1919
import { getUrls } from '../app/utils';
20-
import { createTableFromFromObjectArray, createTableFromText, loadDataWrapper } from '../data/utils';
20+
import { createTableFromFromObjectArray, createTableFromText, loadTextDataWrapper, loadBinaryDataWrapper } from '../data/utils';
2121

2222
import CloseIcon from '@mui/icons-material/Close';
2323
import KeyboardReturnIcon from '@mui/icons-material/KeyboardReturn';
@@ -275,14 +275,56 @@ export const TableUploadDialog: React.FC<TableUploadDialogProps> = ({ buttonElem
275275

276276
if (files) {
277277
for (let file of files) {
278-
file.text().then((text) => {
279-
const uniqueName = getUniqueTableName(file.name, existingNames);
280-
let table = loadDataWrapper(uniqueName, text, file.type);
281-
if (table) {
282-
dispatch(dfActions.loadTable(table));
283-
dispatch(fetchFieldSemanticType(table));
284-
}
285-
});
278+
const uniqueName = getUniqueTableName(file.name, existingNames);
279+
280+
// Check if file is a text type (csv, tsv, json)
281+
if (file.type === 'text/csv' ||
282+
file.type === 'text/tab-separated-values' ||
283+
file.type === 'application/json' ||
284+
file.name.endsWith('.csv') ||
285+
file.name.endsWith('.tsv') ||
286+
file.name.endsWith('.json')) {
287+
288+
// Handle text files
289+
file.text().then((text) => {
290+
let table = loadTextDataWrapper(uniqueName, text, file.type);
291+
if (table) {
292+
dispatch(dfActions.loadTable(table));
293+
dispatch(fetchFieldSemanticType(table));
294+
}
295+
});
296+
} else if (file.type === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ||
297+
file.type === 'application/vnd.ms-excel' ||
298+
file.name.endsWith('.xlsx') ||
299+
file.name.endsWith('.xls')) {
300+
// Handle Excel files
301+
const reader = new FileReader();
302+
reader.onload = (e) => {
303+
const arrayBuffer = e.target?.result as ArrayBuffer;
304+
if (arrayBuffer) {
305+
let tables = loadBinaryDataWrapper(uniqueName, arrayBuffer);
306+
for (let table of tables) {
307+
dispatch(dfActions.loadTable(table));
308+
dispatch(fetchFieldSemanticType(table));
309+
}
310+
if (tables.length == 0) {
311+
dispatch(dfActions.addMessages({
312+
"timestamp": Date.now(),
313+
"type": "error",
314+
"value": `Failed to parse Excel file ${file.name}. Please check the file format.`
315+
}));
316+
}
317+
}
318+
};
319+
reader.readAsArrayBuffer(file);
320+
} else {
321+
// Unsupported file type
322+
dispatch(dfActions.addMessages({
323+
"timestamp": Date.now(),
324+
"type": "error",
325+
"value": `Unsupported file format: ${file.name}. Please use CSV, TSV, JSON, or Excel files.`
326+
}));
327+
}
286328
}
287329
}
288330
if (inputRef.current) {
@@ -294,7 +336,7 @@ export const TableUploadDialog: React.FC<TableUploadDialogProps> = ({ buttonElem
294336
<>
295337
<Input
296338
inputProps={{
297-
accept: '.csv,.tsv,.json',
339+
accept: '.csv,.tsv,.json,.xlsx,.xls',
298340
multiple: true,
299341
}}
300342
id="upload-data-file"

‎yarn.lock

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1313,6 +1313,11 @@ acorn@^8.14.0:
13131313
resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.14.0.tgz#063e2c70cac5fb4f6467f0b11152e04c682795b0"
13141314
integrity sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==
13151315

1316+
adler-32@~1.3.0:
1317+
version "1.3.1"
1318+
resolved "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2"
1319+
integrity sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==
1320+
13161321
[email protected]:
13171322
version "10.0.2"
13181323
resolved "https://registry.yarnpkg.com/ag-charts-community/-/ag-charts-community-10.0.2.tgz#ff10bc760a6f5f8b65d9ed8e23c46ac47ae36de9"
@@ -1592,6 +1597,14 @@ callsites@^3.0.0:
15921597
resolved "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
15931598
integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
15941599

1600+
cfb@~1.2.1:
1601+
version "1.2.2"
1602+
resolved "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz#94e687628c700e5155436dac05f74e08df23bc44"
1603+
integrity sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==
1604+
dependencies:
1605+
adler-32 "~1.3.0"
1606+
crc-32 "~1.2.0"
1607+
15951608
chalk@^2.0.0, chalk@^2.4.2:
15961609
version "2.4.2"
15971610
resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
@@ -1658,6 +1671,11 @@ clsx@^2.0.0:
16581671
resolved "https://registry.npmjs.org/clsx/-/clsx-2.1.0.tgz#e851283bcb5c80ee7608db18487433f7b23f77cb"
16591672
integrity sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==
16601673

1674+
codepage@~1.15.0:
1675+
version "1.15.0"
1676+
resolved "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz#2e00519024b39424ec66eeb3ec07227e692618ab"
1677+
integrity sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==
1678+
16611679
color-convert@^1.9.0:
16621680
version "1.9.3"
16631681
resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
@@ -1724,6 +1742,11 @@ cosmiconfig@^7.0.0:
17241742
path-type "^4.0.0"
17251743
yaml "^1.10.0"
17261744

1745+
crc-32@~1.2.0, crc-32@~1.2.1:
1746+
version "1.2.2"
1747+
resolved "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff"
1748+
integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==
1749+
17271750
create-emotion@^10.0.14, create-emotion@^10.0.27:
17281751
version "10.0.27"
17291752
resolved "https://registry.npmjs.org/create-emotion/-/create-emotion-10.0.27.tgz#cb4fa2db750f6ca6f9a001a33fbf1f6c46789503"
@@ -2562,6 +2585,11 @@ for-each@^0.3.3:
25622585
dependencies:
25632586
is-callable "^1.1.3"
25642587

2588+
frac@~1.1.2:
2589+
version "1.1.2"
2590+
resolved "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz#3d74f7f6478c88a1b5020306d747dc6313c74d0b"
2591+
integrity sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==
2592+
25652593
fsevents@~2.3.2, fsevents@~2.3.3:
25662594
version "2.3.3"
25672595
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
@@ -3788,6 +3816,13 @@ source-map@^0.5.7:
37883816
resolved "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz"
37893817
integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
37903818

3819+
ssf@~0.11.2:
3820+
version "0.11.2"
3821+
resolved "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz#0b99698b237548d088fc43cdf2b70c1a7512c06c"
3822+
integrity sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==
3823+
dependencies:
3824+
frac "~1.1.2"
3825+
37913826
string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
37923827
version "4.2.3"
37933828
resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
@@ -4557,11 +4592,21 @@ which@^2.0.1:
45574592
dependencies:
45584593
isexe "^2.0.0"
45594594

4595+
wmf@~1.0.1:
4596+
version "1.0.2"
4597+
resolved "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz#7d19d621071a08c2bdc6b7e688a9c435298cc2da"
4598+
integrity sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==
4599+
45604600
word-wrap@^1.2.5:
45614601
version "1.2.5"
45624602
resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34"
45634603
integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==
45644604

4605+
word@~0.3.0:
4606+
version "0.3.0"
4607+
resolved "https://registry.npmjs.org/word/-/word-0.3.0.tgz#8542157e4f8e849f4a363a288992d47612db9961"
4608+
integrity sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==
4609+
45654610
wrap-ansi@^7.0.0:
45664611
version "7.0.0"
45674612
resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
@@ -4571,6 +4616,19 @@ wrap-ansi@^7.0.0:
45714616
string-width "^4.1.0"
45724617
strip-ansi "^6.0.0"
45734618

4619+
xlsx@^0.18.5:
4620+
version "0.18.5"
4621+
resolved "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz#16711b9113c848076b8a177022799ad356eba7d0"
4622+
integrity sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==
4623+
dependencies:
4624+
adler-32 "~1.3.0"
4625+
cfb "~1.2.1"
4626+
codepage "~1.15.0"
4627+
crc-32 "~1.2.1"
4628+
ssf "~0.11.2"
4629+
wmf "~1.0.1"
4630+
word "~0.3.0"
4631+
45744632
y18n@^5.0.5:
45754633
version "5.0.8"
45764634
resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz"

0 commit comments

Comments
 (0)