lots of tweaks and fixes to get synthetic data generation working for a v0

rubenlangius · rubenlangius · commit 54a3bce7bf62 · 2024-11-05T13:25:23.000+01:00
diff --git a/package-lock.json b/package-lock.json
diff --git a/public/Bar-Pass-Prediction.csv b/public/Bar-Pass-Prediction.csv
diff --git a/src/assets/synthetic-data.tsx b/src/assets/synthetic-data.tsx
@@ -17,13 +17,9 @@ start = time.time()
 
 from js import data
 from js import setResult
-from js import setMostBiasedCluster
-from js import setOtherClusters
-from js import iter
-from js import clusters
-from js import targetColumn
 from js import dataType
-from js import higherIsBetter
+from js import isDemo
+
 
 class GaussianCopulaSynthesizer:
     def __init__(self):
@@ -124,7 +120,17 @@ def run():
     admissions_sub = admissions_df[['sex', 'race', 'gpa']]
     real_data = admissions_sub.dropna()
 
-    print(real_data.head())
+    if isDemo:
+        setResult(json.dumps(
+            {'type': 'heading', 'data': '''Demo'''}
+        ))
+        setResult(json.dumps(
+            {'type': 'text', 'data': '''A demo dataset is loaded below. We will now generate synthetic data on the columns: 'sex', 'gpa', 'race'. We will be using the Gaussian Copula method and evaluate the distribution and correlation differences between the real and synthetic data.'''}
+        ))
+
+    setResult(json.dumps(
+        {'type': 'data-set-preview', 'data': ''}
+    ))
 
     # Initialize synthesizer and fit it to the data
     synthesizer = GaussianCopulaSynthesizer()
@@ -139,16 +145,26 @@ def run():
 
     results = run_diagnostic(real_data, synthetic_data, target_column='gpa')  
     print('Results:', results)
-
     setResult(json.dumps(
-        {'type': 'heading', 'data': 'Parameters selected'}
+        {'type': 'heading', 'data': 'Diagnostic Results:'}
     ))
+    setResult(json.dumps({'type': 'table', 'data': json.dumps([
+        {
+            'attribute': key,
+            'ks_stat': values['ks_stat'],
+            'p_value': values['p_value']
+        }
+        for key, values in results['distribution_results'].items()
+    ])}))
 
-    print('table output', json.dumps({'type': 'table', 'data': json.loads(synthetic_data.to_json(orient="records"))}))
+    setResult(json.dumps(
+        {'type': 'heading', 'data': 'Correlation difference: ' + str(results['correlation_diff']) }
+    ))
 
     setResult(json.dumps(
-        {'type': 'table', 'data': json.dumps({'type': 'table', 'data': json.loads(synthetic_data.to_json(orient="records"))})}
+        {'type': 'heading', 'data': 'Output file:'}
     ))
+    setResult(json.dumps({'type': 'table', 'data': synthetic_data.to_json(orient="records")}))
     return 
     
 
diff --git a/src/components/SimpleTable.tsx b/src/components/SimpleTable.tsx
@@ -15,6 +15,9 @@ export default function SimpleTable({
     title?: string;
     data: Record<string, string | number>[];
 }) {
+    // limit data to the first 100 rows.
+    data = data.slice(0, 100);
+
     return (
         <div className={`bg-white border border-gray-200 ${title && 'mb-4'}`}>
             <Table className={`text-xs ${title && 'mb-4'}`}>
diff --git a/src/components/SyntheticDataSettings.tsx b/src/components/SyntheticDataSettings.tsx
@@ -1,24 +1,14 @@
-import { Label } from '@/components/ui/label';
-import {
-    Select,
-    SelectContent,
-    SelectItem,
-    SelectTrigger,
-    SelectValue,
-} from '@/components/ui/select';
-import { Slider } from '@/components/ui/slider';
-import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group';
-
 import CSVReader, { csvReader } from './CSVReader';
 import { useEffect, useState } from 'react';
 import { Button } from './ui/button';
 import { ArrowDown, ArrowRight } from 'lucide-react';
 import { z } from 'zod';
 import { useForm } from 'react-hook-form';
 import { zodResolver } from '@hookform/resolvers/zod';
-import { Form, FormControl, FormField, FormItem, FormLabel } from './ui/form';
+import { Form, FormField } from './ui/form';
 import { Card, CardDescription, CardHeader, CardTitle } from './ui/card';
 import Papa from 'papaparse';
+import { SyntheticDataParameters } from './synthetic-data-interfaces/BiasDetectionParameters';
 
 const FormSchema = z.object({
     file: z.string({
@@ -33,13 +23,7 @@ export default function BiasSettings({
     isErrorDuringAnalysis,
     isInitialised,
 }: {
-    onRun: (
-        clusterSize: number,
-        iterations: number,
-        targetColumn: string,
-        dataType: string,
-        higherIsBetter: boolean
-    ) => void;
+    onRun: (params: SyntheticDataParameters) => void;
     onDataLoad: csvReader['onChange'];
     isLoading: boolean;
     isErrorDuringAnalysis: boolean;
@@ -48,10 +32,6 @@ export default function BiasSettings({
     const form = useForm<z.infer<typeof FormSchema>>({
         resolver: zodResolver(FormSchema),
     });
-    const [iter, setIter] = useState([10]);
-    const [clusters, setClusters] = useState([25]);
-
-    const [dataKey, setDataKey] = useState<string>(new Date().toISOString());
     const [data, setData] = useState<{
         data: Record<string, string>[];
         stringified: string;
@@ -69,18 +49,14 @@ export default function BiasSettings({
             form.setValue('file', stringified);
         }
         setData({ data, stringified, fileName });
-
-        const dataLength = (data?.length || 1000) / 10;
-        setClusters([Math.round(dataLength / 4)]);
-        setDataKey(new Date().toISOString());
     };
 
     useEffect(() => {
         onDataLoad(data.data, data.stringified, data.fileName);
     }, [data]);
 
     const onDemoRun = async () => {
-        const file = await fetch('/FP-test-set.csv')
+        const file = await fetch('/Bar-Pass-Prediction.csv')
             .then(response => response.text())
             .then(data => Papa.parse(data, { header: true }));
         onDataLoad(
@@ -91,14 +67,8 @@ export default function BiasSettings({
         );
     };
 
-    const onSubmit = (data: z.infer<typeof FormSchema>) => {
-        onRun(
-            clusters[0],
-            iter[0],
-            data.targetColumn,
-            data.dataType,
-            data.whichPerformanceMetricValueIsBetter === 'higher'
-        );
+    const onSubmit = () => {
+        onRun({ dataType: 'numeric', isDemo: false });
     };
 
     return (
diff --git a/src/components/synthetic-data-interfaces/BiasDetectionParameters.ts b/src/components/synthetic-data-interfaces/BiasDetectionParameters.ts
@@ -0,0 +1,4 @@
+export interface SyntheticDataParameters {
+    dataType: string;
+    isDemo: boolean;
+}
diff --git a/src/components/synthetic-data-interfaces/cluster-export.ts b/src/components/synthetic-data-interfaces/cluster-export.ts
@@ -0,0 +1,4 @@
+export interface SyntheticDataInfo {
+    dataType: string;
+    isDemo: boolean;
+}
diff --git a/src/components/synthetic-data-interfaces/csv-data.ts b/src/components/synthetic-data-interfaces/csv-data.ts
@@ -0,0 +1,6 @@
+export interface CSVData {
+    data: Record<string, string>[];
+    stringified: string;
+    fileName: string;
+    demo?: boolean;
+}
diff --git a/src/routes/SyntheticData.tsx b/src/routes/SyntheticData.tsx
@@ -4,13 +4,12 @@ import { pythonCode } from '@/assets/synthetic-data';
 import { usePython } from '@/components/pyodide/use-python';
 import { Share } from 'lucide-react';
 import { csvReader } from '@/components/CSVReader';
-import SimpleTable from '@/components/SimpleTable';
 import { cn } from '@/lib/utils';
 import ComponentMapper from '@/components/componentMapper';
-import { downloadFile } from '@/lib/download-file';
 import { useReactToPrint } from 'react-to-print';
 import Measuring from '@/components/icons/measuring.svg?react';
 import SyntheticDataSettings from '@/components/SyntheticDataSettings';
+import { SyntheticDataInfo } from '@/components/synthetic-data-interfaces/cluster-export';
 
 const PAGE_STYLE = `
     @page {
@@ -60,8 +59,10 @@ export default function SyntheticDataGeneration() {
         runPython,
         sendData,
         error,
-        clusterInfo,
-    } = usePython();
+    } = usePython<SyntheticDataInfo, SyntheticDataInfo>({
+        dataType: 'numeric',
+        isDemo: false,
+    });
 
     const onFileLoad: csvReader['onChange'] = (
         data,
@@ -85,25 +86,18 @@ export default function SyntheticDataGeneration() {
             sendData(data.stringified);
         }
         if (data.demo) {
-            onRun(3, 10, 'FP', 'numeric', false);
+            onRun({ dataType: 'numeric', isDemo: true });
         }
     }, [initialised, data]);
 
-    const onRun = (
-        clusterSize: number,
-        iterations: number,
-        targetColumn: string,
-        dataType: string,
-        higherIsBetter: boolean
-    ) => {
+    const onRun = (props: { dataType: string; isDemo: boolean }) => {
         runPython({
             type: 'start',
             params: {
-                iter: iterations,
-                clusters: clusterSize,
-                targetColumn: targetColumn,
-                dataType: dataType,
-                higherIsBetter: higherIsBetter,
+                parameters: {
+                    dataType: props.dataType,
+                    isDemo: props.isDemo,
+                },
             },
         });
     };
@@ -137,42 +131,11 @@ export default function SyntheticDataGeneration() {
                             <Share className="size-3.5 mr-2" />
                             Share
                         </Button>
-                        {clusterInfo && (
-                            <Button
-                                variant="outline"
-                                size="sm"
-                                className="p-4 text-sm"
-                                onClick={() => {
-                                    downloadFile(
-                                        JSON.stringify(
-                                            {
-                                                fileName: data.fileName,
-                                                ...clusterInfo,
-                                            },
-                                            null,
-                                            2
-                                        ),
-                                        `${data.fileName.replace('.csv', '') || 'cluster-info'}-${clusterInfo.date.toISOString()}.json`,
-                                        'application/json'
-                                    );
-                                }}
-                            >
-                                <Share className="size-3.5 mr-2" />
-                                Export to .json
-                            </Button>
-                        )}
                     </div>
                 )}
 
-                {data.data.length > 0 && (
-                    <SimpleTable
-                        data={data.data.slice(0, 5)}
-                        title="Dataset preview showing the first 5 rows."
-                    />
-                )}
-
                 {result.length > 0 ? (
-                    <ComponentMapper items={result} />
+                    <ComponentMapper items={result} data={data} />
                 ) : data.data.length > 0 ? null : (
                     <>
                         <Measuring className="max-w-96 m-auto 2xl:max-w-full" />
