Skip to content

Commit 20aab7f

Browse files
Chenglong-MSChenglong-MS
committed
still messy, but able to work with remote data
1 parent 3a087d5 commit 20aab7f

File tree

5 files changed

+240
-153
lines changed

5 files changed

+240
-153
lines changed

‎py-src/data_formulator/app.py

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -614,22 +614,53 @@ def sample_table():
614614
data = request.get_json()
615615
table_id = data.get('table')
616616
sample_size = data.get('size', 1000)
617+
projection_fields = data.get('projection_fields', []) # if empty, we want to include all fields
617618
method = data.get('method', 'random') # one of 'random', 'head', 'bottom'
618-
619+
order_by_fields = data.get('order_by_fields', [])
620+
621+
# Validate field names against table columns to prevent SQL injection
619622
with db_manager.connection(session['session_id']) as db:
623+
# Get valid column names
624+
columns = [col[0] for col in db.execute(f"DESCRIBE {table_id}").fetchall()]
625+
626+
# Filter order_by_fields to only include valid column names
627+
valid_order_by_fields = [field for field in order_by_fields if field in columns]
628+
valid_projection_fields = [field for field in projection_fields if field in columns]
629+
630+
if len(valid_projection_fields) == 0:
631+
projection_fields_str = "*"
632+
else:
633+
projection_fields_str = ", ".join(valid_projection_fields)
634+
620635
if method == 'random':
621-
result = db.execute(f"SELECT * FROM {table_id} ORDER BY RAND() LIMIT {sample_size}").fetchall()
636+
result = db.execute(f"SELECT {projection_fields_str} FROM {table_id} ORDER BY RANDOM() LIMIT {sample_size}").fetchall()
622637
elif method == 'head':
623-
result = db.execute(f"SELECT * FROM {table_id} LIMIT {sample_size}").fetchall()
638+
if valid_order_by_fields:
639+
# Build ORDER BY clause with validated fields
640+
order_by_clause = ", ".join([f'"{field}"' for field in valid_order_by_fields])
641+
result = db.execute(f"SELECT {projection_fields_str} FROM {table_id} ORDER BY {order_by_clause} LIMIT {sample_size}").fetchall()
642+
else:
643+
result = db.execute(f"SELECT {projection_fields_str} FROM {table_id} LIMIT {sample_size}").fetchall()
624644
elif method == 'bottom':
625-
# Get the bottom rows by ordering in descending order and limiting
626-
result = db.execute(f"SELECT * FROM {table_id} ORDER BY ROWID DESC LIMIT {sample_size}").fetchall()
645+
if valid_order_by_fields:
646+
# Build ORDER BY clause with validated fields in descending order
647+
order_by_clause = ", ".join([f'"{field}" DESC' for field in valid_order_by_fields])
648+
result = db.execute(f"SELECT {projection_fields_str} FROM {table_id} ORDER BY {order_by_clause} LIMIT {sample_size}").fetchall()
649+
else:
650+
result = db.execute(f"SELECT {projection_fields_str} FROM {table_id} ORDER BY ROWID DESC LIMIT {sample_size}").fetchall()
651+
652+
# When using projection_fields, we need to use those as our column names
653+
if len(valid_projection_fields) > 0:
654+
column_names = valid_projection_fields
655+
else:
656+
column_names = columns
627657

628658
return jsonify({
629659
"status": "success",
630-
"rows": result
660+
"rows": [dict(zip(column_names, row)) for row in result]
631661
})
632662
except Exception as e:
663+
print(e)
633664
return jsonify({
634665
"status": "error",
635666
"message": str(e)

‎src/views/DataThread.tsx

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ import 'prismjs/components/prism-python' // Language
4747
import 'prismjs/components/prism-typescript' // Language
4848
import 'prismjs/themes/prism.css'; //Example style, you can use another
4949

50-
import { chartAvailabilityCheck, generateChartSkeleton, getDataTable } from './VisualizationView';
50+
import { checkChartAvailability, generateChartSkeleton, getDataTable } from './VisualizationView';
5151
import { TriggerCard } from './EncodingShelfCard';
5252

5353
import ChevronLeftIcon from '@mui/icons-material/ChevronLeft';
@@ -634,7 +634,7 @@ export const DataThread: FC<{}> = function ({ }) {
634634
return { chartId: chart.id, tableId: table.id, element }
635635
}
636636

637-
let [available, unfilledFields] = chartAvailabilityCheck(chart.encodingMap, conceptShelfItems, extTable);
637+
let available = checkChartAvailability(chart, conceptShelfItems, extTable);
638638

639639
if (!available || chart.chartType == "Table") {
640640

‎src/views/EncodingShelfThread.tsx

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import embed from 'vega-embed';
3030
import { getTriggers, getUrls, assembleVegaChart, resolveChartFields } from '../app/utils';
3131

3232
import { getChartTemplate } from '../components/ChartTemplates';
33-
import { chartAvailabilityCheck, generateChartSkeleton } from './VisualizationView';
33+
import { checkChartAvailability, generateChartSkeleton } from './VisualizationView';
3434
import TableRowsIcon from '@mui/icons-material/TableRowsOutlined';
3535
import InsightsIcon from '@mui/icons-material/Insights';
3636
import AnchorIcon from '@mui/icons-material/Anchor';
@@ -56,7 +56,7 @@ export let ChartElementFC: FC<{chart: Chart, tableRows: any[], boxWidth?: number
5656

5757
let chartTemplate = getChartTemplate(chart.chartType);
5858

59-
let [available, unfilledFields] = chartAvailabilityCheck(chart.encodingMap, conceptShelfItems, tableRows);
59+
let available = checkChartAvailability(chart, conceptShelfItems, tableRows);
6060

6161
if (chart.chartType == "Auto") {
6262
return <Box sx={{ position: "relative", display: "flex", flexDirection: "column", margin: 'auto', color: 'darkgray' }}>
@@ -420,10 +420,6 @@ export const EncodingShelfThread: FC<EncodingShelfThreadProps> = function ({ cha
420420
let leafTable = tables.find(t => t.id == activeTableThread[activeTableThread.length - 1]) as DictTable;
421421
let triggers = getTriggers(leafTable, tables)
422422

423-
424-
console.log('from local data thread')
425-
console.log(triggers[triggers.length - 1]);
426-
427423
let instructionCards = triggers.map((trigger, i) => {
428424
let extractActiveFields = (t: Trigger) => {
429425
let encodingMap = (charts.find(c => c.id == t.chartRef) as Chart).encodingMap

‎src/views/SelectableDataGrid.tsx

Lines changed: 94 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import TableRow from '@mui/material/TableRow';
1212
import { Box } from '@mui/system';
1313

1414
import { useTheme } from '@mui/material/styles';
15-
import { alpha, Collapse, Divider, Paper, ToggleButton, Tooltip } from "@mui/material";
15+
import { alpha, Collapse, Divider, Paper, ToggleButton, Tooltip, CircularProgress } from "@mui/material";
1616

1717
import { TSelectableItemProps, createSelectable } from 'react-selectable-fast';
1818
import { SelectableGroup } from 'react-selectable-fast';
@@ -163,6 +163,8 @@ export const SelectableDataGrid: React.FC<SelectableDataGridProps> = ({ tableId,
163163

164164
const [rowsToDisplay, setRowsToDisplay] = React.useState<any[]>(rows);
165165

166+
const [isLoading, setIsLoading] = React.useState<boolean>(false);
167+
166168
React.useEffect(() => {
167169
// use this to handle cases when the table add new columns/remove new columns etc
168170
$tableRef.current?.clearSelection();
@@ -214,7 +216,6 @@ export const SelectableDataGrid: React.FC<SelectableDataGridProps> = ({ tableId,
214216
debouncedSearchHandler(searchText);
215217
}, [searchText, debouncedSearchHandler]);
216218

217-
218219
const handleSelectionFinish = (selected: any[]) => {
219220
let newSelectedCells = _.uniq(selected.map(x => x.props.indices));
220221
setSelectedCells(newSelectedCells);
@@ -288,28 +289,70 @@ export const SelectableDataGrid: React.FC<SelectableDataGridProps> = ({ tableId,
288289
};
289290
}, []);
290291

291-
// At the component level, add state for the menu
292-
const [menuAnchorEl, setMenuAnchorEl] = React.useState<null | HTMLElement>(null);
293-
const open = Boolean(menuAnchorEl);
294-
295-
const handleMenuClick = (event: React.MouseEvent<HTMLElement>) => {
296-
setMenuAnchorEl(event.currentTarget);
297-
};
298-
299-
const handleMenuClose = () => {
300-
setMenuAnchorEl(null);
292+
const fetchSortedVirtualData = (columnIds: string[], sortOrder: 'asc' | 'desc') => {
293+
// Set loading to true when starting the fetch
294+
setIsLoading(true);
295+
296+
// Use the SAMPLE_TABLE endpoint with appropriate ordering
297+
fetch(getUrls().SAMPLE_TABLE, {
298+
method: 'POST',
299+
headers: {
300+
'Content-Type': 'application/json',
301+
},
302+
body: JSON.stringify({
303+
table: tableId,
304+
size: 1000,
305+
method: sortOrder === 'asc' ? 'head' : 'bottom',
306+
order_by_fields: columnIds
307+
}),
308+
})
309+
.then(response => response.json())
310+
.then(data => {
311+
if (data.status === 'success') {
312+
setRowsToDisplay(data.rows);
313+
}
314+
// Set loading to false when done
315+
setIsLoading(false);
316+
})
317+
.catch(error => {
318+
console.error('Error fetching sorted table data:', error);
319+
// Ensure loading is set to false even on error
320+
setIsLoading(false);
321+
});
301322
};
302323

303324
return (
304325
<Box className="table-container table-container-small"
305326
sx={{
306327
width: '100%',
307328
height: '100%',
329+
position: 'relative',
308330
"& .MuiTableCell-root": {
309331
fontSize: 12, maxWidth: "120px", padding: "2px 6px", cursor: "default",
310332
overflow: "clip", textOverflow: "ellipsis", whiteSpace: "nowrap"
311333
}
312334
}}>
335+
{/* Loading Overlay */}
336+
{isLoading && (
337+
<Box sx={{
338+
position: 'absolute',
339+
top: 0,
340+
left: 0,
341+
right: 0,
342+
zIndex: 10,
343+
display: 'flex',
344+
alignItems: 'center',
345+
justifyContent: 'center',
346+
backgroundColor: 'rgba(255, 255, 255, 0.7)',
347+
padding: '8px',
348+
height: '100%',
349+
borderTopLeftRadius: '4px',
350+
borderTopRightRadius: '4px'
351+
}}>
352+
<CircularProgress size={24} sx={{ mr: 1 }} />
353+
<Typography variant="body2" color="darkgray">Fetching data...</Typography>
354+
</Box>
355+
)}
313356
{/* @ts-expect-error */}
314357
<SelectableGroup
315358
ref={$tableRef}
@@ -373,9 +416,23 @@ export const SelectableDataGrid: React.FC<SelectableDataGridProps> = ({ tableId,
373416
active={orderBy === columnDef.id}
374417
direction={orderBy === columnDef.id ? order : 'asc'}
375418
onClick={() => {
376-
const newOrder = (orderBy === columnDef.id && order === 'asc') ? 'desc' : 'asc';
419+
let newOrder: 'asc' | 'desc' = 'asc';
420+
let newOrderBy : string | undefined = columnDef.id;
421+
if (orderBy === columnDef.id && order === 'asc') {
422+
newOrder = 'desc';
423+
} else if (orderBy === columnDef.id && order === 'desc') {
424+
newOrder = 'asc';
425+
newOrderBy = undefined;
426+
} else {
427+
newOrder = 'asc';
428+
}
429+
377430
setOrder(newOrder);
378-
setOrderBy(columnDef.id);
431+
setOrderBy(newOrderBy);
432+
433+
if (virtual) {
434+
fetchSortedVirtualData(newOrderBy ? [newOrderBy] : [], newOrder);
435+
}
379436
}}
380437
>
381438
<span role="img" style={{ fontSize: "inherit", padding: "2px", display: "inline-flex", alignItems: "center" }}>
@@ -447,12 +504,33 @@ export const SelectableDataGrid: React.FC<SelectableDataGridProps> = ({ tableId,
447504
</Typography>
448505
{virtual && (
449506
<>
450-
<Tooltip title="Sample data from this table">
507+
<Tooltip title="view 1000 random rows from this table">
451508
<IconButton
452509
size="small"
453510
color="primary"
454511
sx={{marginRight: 1}}
455-
onClick={handleMenuClick}
512+
onClick={() => {
513+
fetch(getUrls().SAMPLE_TABLE, {
514+
method: 'POST',
515+
headers: {
516+
'Content-Type': 'application/json',
517+
},
518+
body: JSON.stringify({
519+
table: tableId,
520+
size: 1000,
521+
method: 'random'
522+
}),
523+
})
524+
.then(response => response.json())
525+
.then(data => {
526+
if (data.status === 'success') {
527+
setRowsToDisplay(data.rows);
528+
}
529+
})
530+
.catch(error => {
531+
console.error('Error sampling table:', error);
532+
});
533+
}}
456534
>
457535
<CasinoIcon sx={{
458536
fontSize: 18,
@@ -463,78 +541,6 @@ export const SelectableDataGrid: React.FC<SelectableDataGridProps> = ({ tableId,
463541
}} />
464542
</IconButton>
465543
</Tooltip>
466-
<Menu
467-
anchorEl={menuAnchorEl}
468-
open={open}
469-
onClose={handleMenuClose}
470-
slotProps={{
471-
paper: {
472-
elevation: 3,
473-
sx: { minWidth: 180 }
474-
}
475-
}}
476-
>
477-
<Typography variant="subtitle2" sx={{ px: 2, py: 1, fontSize: "12px", color: 'darkgray' }}>
478-
Sample Method
479-
</Typography>
480-
{[
481-
{ label: "Top 1000 Rows", method: "head", icon: <ArrowUpwardIcon fontSize="small" /> },
482-
{ label: "Random 1000 Rows", method: "random", icon: <CasinoIcon fontSize="small" /> },
483-
{ label: "Bottom 1000 Rows", method: "bottom", icon: <ArrowDownwardIcon fontSize="small" /> }
484-
].map((option) => (
485-
<MenuItem
486-
key={option.method}
487-
sx={{
488-
'& .MuiListItemText-primary': { fontSize: "12px" },
489-
'& .MuiListItemIcon-root': { minWidth: '24px' },
490-
'& .MuiSvgIcon-root': { fontSize: '16px' }
491-
}}
492-
onClick={() => {
493-
handleMenuClose();
494-
495-
// Use fetch to get the sample data
496-
fetch(getUrls().SAMPLE_TABLE, {
497-
method: 'POST',
498-
headers: {
499-
'Content-Type': 'application/json',
500-
},
501-
body: JSON.stringify({
502-
table: tableId,
503-
size: 1000,
504-
method: option.method
505-
}),
506-
})
507-
.then(response => response.json())
508-
.then(data => {
509-
if (data.status === 'success') {
510-
// Update rows state with the new sample
511-
console.log("sampled rows", data.rows);
512-
513-
// Convert array rows to dictionary rows
514-
// This assumes the order of elements in each row matches the order of columnDefs
515-
const dictRows = data.rows.map((row: any) => {
516-
const dictRow: any = {};
517-
columnDefs.forEach((col, index) => {
518-
dictRow[col.id] = row[index];
519-
});
520-
return dictRow;
521-
});
522-
523-
setRowsToDisplay(dictRows);
524-
}
525-
})
526-
.catch(error => {
527-
console.error('Error sampling table:', error);
528-
});
529-
}}
530-
>
531-
<ListItemIcon>
532-
{option.icon}
533-
</ListItemIcon>
534-
<ListItemText>{option.label}</ListItemText>
535-
</MenuItem>
536-
))}
537-
</Menu>
538544
</>
539545
)}
540546
{!virtual && <Tooltip title={`Download ${tableName} as CSV`}>

0 commit comments

Comments
 (0)