Skip to content

Commit 8df59e0

Browse files
Chenglong-MSChenglong-MS
committed
a bit messy, but I'll clean up later
1 parent 4b2d5f6 commit 8df59e0

File tree

14 files changed

+579
-531
lines changed

14 files changed

+579
-531
lines changed

‎py-src/data_formulator/app.py

Lines changed: 332 additions & 25 deletions
Large diffs are not rendered by default.

‎requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ docker
55
namedlist
66
matplotlib
77
flask
8-
flask-cors
98
openai
109
azure-identity
1110
azure-keyvault-secrets
1211
python-dotenv
1312
vega_datasets
1413
litellm
14+
duckdb
1515
-e . #also need to install data formulator itself

‎src/app/App.tsx

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
dfActions,
1111
fetchAvailableModels,
1212
fetchFieldSemanticType,
13-
fetchSessionId
13+
getSessionId,
1414
} from './dfSlice'
1515

1616
import blue from '@mui/material/colors/blue';
@@ -68,6 +68,7 @@ import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
6868
import ContentPasteIcon from '@mui/icons-material/ContentPaste';
6969
import UploadFileIcon from '@mui/icons-material/UploadFile';
7070
import DownloadIcon from '@mui/icons-material/Download';
71+
import { DBTableManager } from '../views/DBTableManager';
7172

7273
const AppBar = styled(MuiAppBar)(({ theme }) => ({
7374
color: 'black',
@@ -472,7 +473,7 @@ export const AppFC: FC<AppFCProps> = function AppFC(appProps) {
472473
useEffect(() => {
473474
document.title = toolName;
474475
dispatch(fetchAvailableModels());
475-
dispatch(fetchSessionId());
476+
dispatch(getSessionId());
476477
}, []);
477478

478479
let theme = createTheme({
@@ -579,6 +580,9 @@ export const AppFC: FC<AppFCProps> = function AppFC(appProps) {
579580
{
580581
path: "/about",
581582
element: <About />,
583+
}, {
584+
path: "/test",
585+
element: <DBTableManager />,
582586
}, {
583587
path: "*",
584588
element: <DataFormulatorFC />,

‎src/app/dfSlice.tsx

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -228,10 +228,10 @@ export const fetchAvailableModels = createAsyncThunk(
228228
}
229229
);
230230

231-
export const fetchSessionId = createAsyncThunk(
232-
"dataFormulatorSlice/fetchSessionId",
231+
export const getSessionId = createAsyncThunk(
232+
"dataFormulatorSlice/getSessionId",
233233
async () => {
234-
const response = await fetch(getUrls().SESSION_ID, {
234+
const response = await fetch(`${getUrls().GET_SESSION_ID}`, {
235235
method: 'GET',
236236
headers: {
237237
'Content-Type': 'application/json',
@@ -240,7 +240,7 @@ export const fetchSessionId = createAsyncThunk(
240240
return response.json();
241241
}
242242
);
243-
243+
244244
export const dataFormulatorSlice = createSlice({
245245
name: 'dataFormulatorSlice',
246246
initialState: initialState,
@@ -250,6 +250,7 @@ export const dataFormulatorSlice = createSlice({
250250

251251
// avoid resetting inputted models
252252
// state.oaiModels = state.oaiModels.filter((m: any) => m.endpoint != 'default');
253+
253254
state.selectedModelId = state.models.length > 0 ? state.models[0].id : undefined;
254255
state.testedModels = [];
255256

@@ -702,7 +703,8 @@ export const dataFormulatorSlice = createSlice({
702703
console.log("fetched codeExpl");
703704
console.log(action.payload);
704705
})
705-
.addCase(fetchSessionId.fulfilled, (state, action) => {
706+
.addCase(getSessionId.fulfilled, (state, action) => {
707+
console.log("got sessionId ", action.payload.session_id);
706708
state.sessionId = action.payload.session_id;
707709
})
708710
},

‎src/app/utils.tsx

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import { DictTable } from "../components/ComponentType";
1010
import { getDType } from "../data/types";
1111

1212
export interface AppConfig {
13-
serverUrl: string;
1413
popupConfig?: PopupConfig;
1514
}
1615

@@ -20,7 +19,6 @@ export interface PopupConfig {
2019
}
2120

2221
export const appConfig: AppConfig = {
23-
serverUrl: process.env.NODE_ENV == "production" ? "./" : "/api",
2422
};
2523

2624
export function assignAppConfig(config: AppConfig) {
@@ -31,29 +29,36 @@ export function assignAppConfig(config: AppConfig) {
3129

3230
export function getUrls() {
3331
return {
34-
CHECK_AVAILABLE_MODELS: `${appConfig.serverUrl}/check-available-models`,
35-
TEST_MODEL: `${appConfig.serverUrl}/test-model`,
32+
CHECK_AVAILABLE_MODELS: `/api/check-available-models`,
33+
TEST_MODEL: `/api/test-model`,
3634

3735
// these functions involves openai models
38-
DERIVE_CONCEPT_URL: `${appConfig.serverUrl}/derive-concept-request`,
39-
SORT_DATA_URL: `${appConfig.serverUrl}/codex-sort-request`,
40-
CLEAN_DATA_URL: `${appConfig.serverUrl}/clean-data`,
41-
SERVER_DERIVE_DATA_URL: `${appConfig.serverUrl}/derive-data`,
42-
SERVER_REFINE_DATA_URL: `${appConfig.serverUrl}/refine-data`,
43-
CODE_EXPL_URL: `${appConfig.serverUrl}/code-expl`,
44-
SERVER_PROCESS_DATA_ON_LOAD: `${appConfig.serverUrl}/process-data-on-load`,
45-
46-
DATASET_INFO_URL: `${appConfig.serverUrl}/datasets-info`,
47-
DATASET_REQUEST_PREFIX: `${appConfig.serverUrl}/datasets/`,
48-
49-
VEGA_DATASET_LIST: `${appConfig.serverUrl}/vega-datasets`,
50-
VEGA_DATASET_REQUEST_PREFIX: `${appConfig.serverUrl}/vega-dataset/`,
51-
52-
APP_CONFIG: `${appConfig.serverUrl}/app-config`,
53-
54-
AUTH_INFO_PREFIX: `${appConfig.serverUrl}/.auth/`,
55-
56-
SESSION_ID: `${appConfig.serverUrl}/get-session-id`,
36+
DERIVE_CONCEPT_URL: `/api/derive-concept-request`,
37+
SORT_DATA_URL: `/api/codex-sort-request`,
38+
CLEAN_DATA_URL: `/api/clean-data`,
39+
SERVER_DERIVE_DATA_URL: `/api/derive-data`,
40+
SERVER_REFINE_DATA_URL: `/api/refine-data`,
41+
CODE_EXPL_URL: `/api/code-expl`,
42+
SERVER_PROCESS_DATA_ON_LOAD: `/api/process-data-on-load`,
43+
44+
DATASET_INFO_URL: `/api/datasets-info`,
45+
DATASET_REQUEST_PREFIX: `/api/datasets/`,
46+
47+
VEGA_DATASET_LIST: `/api/vega-datasets`,
48+
VEGA_DATASET_REQUEST_PREFIX: `/api/vega-dataset/`,
49+
50+
APP_CONFIG: `/api/app-config`,
51+
52+
AUTH_INFO_PREFIX: `/api/.auth/`,
53+
54+
GET_SESSION_ID: `/api/get-session-id`,
55+
LIST_TABLES: `/api/tables`,
56+
TABLE_DATA: `/api/tables/get-table`,
57+
CREATE_TABLE: `/api/tables/create-table`,
58+
DELETE_TABLE: `/api/tables/delete-table`,
59+
ANALYZE_TABLE: `/api/tables/analyze`,
60+
QUERY_TABLE: `/api/tables/query`,
61+
SAMPLE_TABLE: `/api/tables/sample-table`,
5762
};
5863
}
5964

‎src/components/ComponentType.tsx

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,10 @@ export interface DictTable {
7272
// in fact, right now dict tables are all triggered from charts
7373
trigger: Trigger,
7474
};
75+
virtual?: {
76+
rowCount: number; // total number of rows in the full table
77+
available: boolean; // whether the table is available to use
78+
};
7579
anchored: boolean; // whether this table is anchored as a persistent table used to derive other tables
7680
}
7781

‎src/views/DataFormulator.tsx

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import { DataThread } from './DataThread';
4141
import dfLogo from '../assets/df-logo.png';
4242
import exampleImageTable from "../assets/example-image-table.png";
4343
import { ModelSelectionButton } from './ModelSelectionDialog';
44+
import { DBTableSelectionDialog } from './DBTableManager';
4445

4546
//type AppProps = ConnectedProps<typeof connector>;
4647

@@ -137,7 +138,7 @@ Totals (7 entries) 5 5 5 15
137138

138139
<Typography variant="h4">
139140
Load data from
140-
<TableSelectionDialog buttonElement={"Examples"} />, <TableUploadDialog buttonElement={"file"} disabled={false} />, or <TableCopyDialogV2 buttonElement={"clipboard"} disabled={false} />
141+
<TableSelectionDialog buttonElement={"Examples"} />, <TableUploadDialog buttonElement={"file"} disabled={false} />, <TableCopyDialogV2 buttonElement={"clipboard"} disabled={false} /> or <DBTableSelectionDialog buttonElement={"Database"} />
141142
</Typography>
142143
<Typography sx={{ width: 960, margin: "auto" }} variant="body1">
143144
Besides formatted data (csv, tsv or json), you can copy-paste&nbsp;

‎src/views/DataThread.tsx

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ import { TriggerCard } from './EncodingShelfCard';
5252

5353
import ChevronLeftIcon from '@mui/icons-material/ChevronLeft';
5454
import ChevronRightIcon from '@mui/icons-material/ChevronRight';
55+
import CloudQueueIcon from '@mui/icons-material/CloudQueue';
5556

5657
let buildChartCard = (chartElement: { tableId: string, chartId: string, element: any },
5758
focusedChartId?: string) => {
@@ -353,6 +354,14 @@ let SingleThreadView: FC<{
353354
// only charts without dependency can be deleted
354355
let tableDeleteEnabled = !tables.some(t => t.derive?.trigger.tableId == tableId);
355356

357+
let tableCardIcon = table?.virtual? <CloudQueueIcon sx={{ fontSize: 18 }} /> : ( table?.anchored ?
358+
<AnchorIcon sx={{
359+
fontSize: tableId === focusedTableId ? 20 : 16,
360+
color: tableId === focusedTableId ? theme.palette.primary.main : 'rgba(0,0,0,0.5)',
361+
fontWeight: tableId === focusedTableId ? 'bold' : 'normal',
362+
}} /> :
363+
<TableRowsIcon sx={{ fontSize: 16 }} />)
364+
356365
let regularTableBox = <Box ref={relevantCharts.some(c => c.chartId == focusedChartId) ? scrollRef : null}
357366
sx={{ padding: '0px' }}>
358367
<Card className={`data-thread-card ${selectedClassName}`} variant="outlined"
@@ -411,14 +420,7 @@ let SingleThreadView: FC<{
411420
event.stopPropagation();
412421
dispatch(dfActions.updateTableAnchored({tableId: tableId, anchored: !table?.anchored}));
413422
}}>
414-
{table?.anchored ?
415-
<AnchorIcon sx={{
416-
fontSize: tableId === focusedTableId ? 20 : 16,
417-
color: tableId === focusedTableId ? theme.palette.primary.main : 'rgba(0,0,0,0.5)',
418-
fontWeight: tableId === focusedTableId ? 'bold' : 'normal',
419-
}} /> :
420-
<TableRowsIcon sx={{ fontSize: 16 }} />
421-
}
423+
{tableCardIcon}
422424
</IconButton>
423425
</span>
424426
</Tooltip>

‎src/views/DataView.tsx

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,10 @@ export const FreeDataViewFC: FC<FreeDataViewProps> = function DataView({ $table
6262

6363
// given a table render the table
6464
let renderTableBody = (targetTable: DictTable | undefined) => {
65-
const rowData = targetTable ? targetTable.rows.map((r: any, i: number) => ({ ...r, "#rowId": i })) : [];
65+
66+
const rowData = targetTable ?
67+
targetTable.virtual ? targetTable.rows : targetTable.rows.map((r: any, i: number) => ({ ...r, "#rowId": i }))
68+
: [];
6669

6770
// Randomly sample up to 29 rows for column width calculation
6871
const sampleSize = Math.min(29, rowData.length);
@@ -106,7 +109,7 @@ export const FreeDataViewFC: FC<FreeDataViewProps> = function DataView({ $table
106109
};
107110
}) : [];
108111

109-
if (colDefs) {
112+
if (colDefs && !targetTable?.virtual) {
110113
colDefs = [{
111114
id: "#rowId", label: "#", minWidth: 10, align: undefined, width: 40,
112115
format: (value: any) => <Typography fontSize="inherit" color="rgba(0,0,0,0.65)">{value}</Typography>,
@@ -115,8 +118,16 @@ export const FreeDataViewFC: FC<FreeDataViewProps> = function DataView({ $table
115118
}, ...colDefs]
116119
}
117120

118-
return <SelectableDataGrid $tableRef={$tableRef} tableName={targetTable?.id || "table"} rows={rowData}
119-
columnDefs={colDefs} onSelectionFinished={onRangeSelectionChanged} />
121+
return <SelectableDataGrid
122+
tableId={targetTable?.id || ""}
123+
$tableRef={$tableRef}
124+
tableName={targetTable?.displayId || targetTable?.id || "table"}
125+
rows={rowData}
126+
columnDefs={colDefs}
127+
onSelectionFinished={onRangeSelectionChanged}
128+
rowCount={targetTable?.virtual?.rowCount || rowData.length}
129+
virtual={targetTable?.virtual ? true : false}
130+
/>
120131
}
121132

122133
// handle when selection changes

‎src/views/ReactTable.tsx

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import TableHead from '@mui/material/TableHead';
1010
import TablePagination from '@mui/material/TablePagination';
1111
import TableRow from '@mui/material/TableRow';
1212
import { alpha, Box, useTheme } from '@mui/system';
13+
import Typography from '@mui/material/Typography';
1314

1415

1516
export interface ColumnDef {
@@ -27,9 +28,10 @@ interface CustomReactTableProps {
2728
rowsPerPageNum: number;
2829
compact: boolean;
2930
maxCellWidth? : number;
31+
isIncompleteTable?: boolean;
3032
}
3133

32-
export const CustomReactTable: React.FC<CustomReactTableProps> = ({ rows, columnDefs, rowsPerPageNum, compact, maxCellWidth }) => {
34+
export const CustomReactTable: React.FC<CustomReactTableProps> = ({ rows, columnDefs, rowsPerPageNum, compact, maxCellWidth, isIncompleteTable }) => {
3335

3436
let theme = useTheme();
3537

@@ -108,6 +110,15 @@ export const CustomReactTable: React.FC<CustomReactTableProps> = ({ rows, column
108110
</TableRow>
109111
);
110112
})}
113+
{isIncompleteTable && (
114+
<TableRow>
115+
{columnDefs.map((column, i) => (
116+
<TableCell key={i} sx={{padding: 0}} align="left">
117+
......
118+
</TableCell>
119+
))}
120+
</TableRow>
121+
)}
111122
</TableBody>
112123
</Table>
113124
</TableContainer>

0 commit comments

Comments
 (0)