synthesis works on full bar data set

Author: KoomenErina

main.py

@@ -3,14 +3,20 @@
 import pyreadr
 
 def synBar():
-    df = pd.read_csv("bar_pass_prediction.csv")[['sex', 'race1', 'ugpa', 'bar']]
+    df = pd.read_csv("bar_pass_prediction.csv")
     print(df.dtypes)
-    dtype_map = {'sex': 'float', 'race1': 'category', 'ugpa': 'float', 'bar': 'category'}
+    dtype_map = {}
 
-    for (k,v) in dtype_map.items():
-
-        if v == 'category':
-            df = df.astype({k : "category"})
+    for k in df.dtypes.keys():
+        match df.dtypes[k]:
+            case 'float64':
+                dtype_map[k] = 'float'
+            case 'category':
+                dtype_map[k] = 'category'
+                df = df.astype({k : "category"})
+            case _:
+                dtype_map[k]= 'category'
+                df = df.astype({k : "category"})
 
     print(df.dtypes)
     spop = Synthpop()
@@ -28,22 +34,35 @@ def synBar():
 def synSD2011():
     df0 = pyreadr.read_r("SD2011.rda")['SD2011']
     #pd.read_csv("bar_pass_prediction.csv")
-    print(df0.dtypes)
-    df = df0[['age', 'unempdur', 'income', 'sex']]#df0[['sex', 'race1', 'ugpa', 'bar']]
-    print(df.isna().sum())
+    #print(df0.dtypes)
+    df = df0#df0[['sex', 'race1', 'ugpa', 'bar']]
+    #print(df.isna().sum())
     #df.to_excel("inputData.xlsx")
     dtype_map ={
-        "age":"float",
-        "unempdur":"float",
-        "income":"float",
-        "sex":"category"
+        # "age":"float",
+        # "unempdur":"float",
+        # "income":"float",
+        # "sex":"category"
     }
+
+    for k in df.dtypes.keys():
+        match df.dtypes[k]:
+            case 'float64':
+                dtype_map[k] = 'float'
+            case 'category':
+                dtype_map[k] = 'category'
+                df = df.astype({k : "category"})
+            case _:
+                dtype_map[k]= 'category'
+                df = df.astype({k : "category"})
+
+    print(dtype_map)
     #{'sex': 'float', 'race1': 'category', 'ugpa': 'float', 'bar': 'category'}
     # for (k,v) in dtype_map.items():
     #     if v == 'category':
     #         df[k] = df[k].astype('category')
 
-    print(df.dtypes)
+
     r = df.dtypes.keys()
     spop = Synthpop()
     spop.fit(df,dtype_map)
@@ -53,4 +72,4 @@ def synSD2011():
     print(synth_df.head())
 
 
-synSD2011()
+synBar()

synthpop/processor/processor.py

@@ -114,10 +114,10 @@ def postprocess(self, synth_df):
             # NaNs in numerical columns
             #The code below sets changes NANs in numerical columns to a given value, and removes the NAN indicator column.
             #The NAN_indicator columns are not synthesised.
-            # elif processing_nan_col_dict['dtype'] in NUM_COLS_DTYPES:
-            #     for col_nan_flag, col_nan_value in processing_nan_col_dict['nan_flags'].items():
-            #         nan_flag_indices = synth_df[processing_nan_col_dict['col_nan_name']] == col_nan_flag #expects columnName_NAN in the synthetic data set
-            #         synth_df.loc[nan_flag_indices, col] = col_nan_value
-            #     synth_df.drop(columns=processing_nan_col_dict['col_nan_name'], inplace=True)
+            elif processing_nan_col_dict['dtype'] in NUM_COLS_DTYPES:
+                for col_nan_flag, col_nan_value in processing_nan_col_dict['nan_flags'].items():
+                    nan_flag_indices = synth_df[processing_nan_col_dict['col_nan_name']] == col_nan_flag #expects columnName_NAN in the synthetic data set
+                    synth_df.loc[nan_flag_indices, col] = col_nan_value
+                synth_df.drop(columns=processing_nan_col_dict['col_nan_name'], inplace=True)
 
         return synth_df