added support for benchmark data sets

Author: brettinanl

examples/ADRP/adrp.py

@@ -18,6 +18,7 @@
 
 additional_definitions = [
     {"name": "latent_dim", "action": "store", "type": int, "help": "latent dimensions"},
+    {"name": "benchmark_data", "action": "store", "type": candle.str2bool, "default": False, "help": "Use prepared benchmark data"},
     {
         "name": "residual",
         "type": candle.str2bool,
@@ -230,43 +231,89 @@ def get_model(params):
 
 
 def load_data(params, seed):
-    header_url = params["header_url"]
-    dh_dict, th_list = load_headers(
-        "descriptor_headers.csv", "training_headers.csv", header_url
-    )
-    offset = 6  # descriptor starts at index 6
-    desc_col_idx = [dh_dict[key] + offset for key in th_list]
-
-    url = params["data_url"]
-    file_train = (
-        "ml." + params["base_name"] + ".Orderable_zinc_db_enaHLL.sorted.4col.dd.parquet"
-    )
-    # file_train = params["train_data"]
-    train_file = candle.get_file(file_train, url + file_train, cache_subdir="Pilot1")
-    # df = (pd.read_csv(data_path,skiprows=1).values).astype('float32')
-    print("Loading data...")
-    df = pd.read_parquet(train_file)
-    print("done")
-
-    # df_y = df[:,0].astype('float32')
-    df_y = df["reg"].astype("float32")
-    # df_x = df[:, 1:PL].astype(np.float32)
-    df_x = df.iloc[:, desc_col_idx].astype(np.float32)
-
-    bins = np.arange(0, 20)
-    histogram, bin_edges = np.histogram(df_y, bins=bins, density=False)
-    print("Histogram of samples (bins, counts)")
-    print(bin_edges)
-    print(histogram)
-
-    #    scaler = MaxAbsScaler()
-
-    scaler = StandardScaler()
-    df_x = scaler.fit_transform(df_x)
-
-    X_train, X_test, Y_train, Y_test = train_test_split(
-        df_x, df_y, test_size=0.20, random_state=42
-    )
+    if 'benchmark_data' in params and params['benchmark_data'] != "":
+        if params['train_data'].endswith('.parquet'):
+            header_url = params["header_url"]
+            dh_dict, th_list = load_headers(
+                "descriptor_headers.csv", "training_headers.csv", header_url
+            )
+            offset = 6  # descriptor starts at index 6
+            desc_col_idx = [dh_dict[key] + offset for key in th_list]
+
+            url = params["data_url"]
+
+            # file_train = params["train_data"]
+            train_file = candle.get_file(params['train_data'], url + params['train_data'], cache_subdir="Pilot1")
+            test_file = candle.get_file(params['test_data'], url + params['test_data'], cache_subdir="Pilot1")
+            val_file = candle.get_file(params['val_data'], url + params['val_data'], cache_subdir="Pilot1")
+
+            # df = (pd.read_csv(data_path,skiprows=1).values).astype('float32')
+            print("Loading data...")
+            train_df = pd.read_parquet(train_file)
+            val_df = pd.read_parquet(val_file)
+            test_df = pd.read_parquet(test_file)
+            print("done")
+
+            train_df_y = train_df["reg"].astype("float32")
+            train_df_x = train_df.iloc[:, desc_col_idx].astype(np.float32)
+            test_df_y = test_df["reg"].astype("float32")
+            test_df_x = test_df.iloc[:, desc_col_idx].astype(np.float32)
+            val_df_y = val_df["reg"].astype("float32")
+            val_df_x = val_df.iloc[:, desc_col_idx].astype(np.float32)
+
+            bins = np.arange(0, 20)
+            histogram, bin_edges = np.histogram(train_df_y, bins=bins, density=False)
+            print("Histogram of samples (bins, counts)")
+            print(bin_edges)
+            print(histogram)
+
+            scaler = StandardScaler()
+            scaler.fit(train_df_x)
+            train_df_x = scaler.fit_transform(train_df_x)
+            test_df_x  = scaler.fit_transform(test_df_x)
+            val_df_x  = scaler.fit_transform(val_df_x)
+
+            return train_df_x, train_df_y, val_df_x, val_df_y, train_df_x.shape[1], histogram
+            #return X_train, Y_train, X_test, Y_test, X_train.shape[1], histogram
+
+    else:
+        header_url = params["header_url"]
+        dh_dict, th_list = load_headers(
+            "descriptor_headers.csv", "training_headers.csv", header_url
+        )
+        offset = 6  # descriptor starts at index 6
+        desc_col_idx = [dh_dict[key] + offset for key in th_list]
+
+        url = params["data_url"]
+        file_train = (
+            "ml." + params["base_name"] + ".Orderable_zinc_db_enaHLL.sorted.4col.dd.parquet"
+        )
+        # file_train = params["train_data"]
+        train_file = candle.get_file(file_train, url + file_train, cache_subdir="Pilot1")
+        # df = (pd.read_csv(data_path,skiprows=1).values).astype('float32')
+        print("Loading data...")
+        df = pd.read_parquet(train_file)
+        print("done")
+
+        # df_y = df[:,0].astype('float32')
+        df_y = df["reg"].astype("float32")
+        # df_x = df[:, 1:PL].astype(np.float32)
+        df_x = df.iloc[:, desc_col_idx].astype(np.float32)
+
+        bins = np.arange(0, 20)
+        histogram, bin_edges = np.histogram(df_y, bins=bins, density=False)
+        print("Histogram of samples (bins, counts)")
+        print(bin_edges)
+        print(histogram)
+
+        #    scaler = MaxAbsScaler()
+
+        scaler = StandardScaler()
+        df_x = scaler.fit_transform(df_x)
+
+        X_train, X_test, Y_train, Y_test = train_test_split(
+            df_x, df_y, test_size=0.20, random_state=42
+        )
 
     print("x_train shape:", X_train.shape)
     print("x_test shape:", X_test.shape)