update plangen

Author: hyoo

Pilot1/Uno/README.ChallengeProblem.md

@@ -0,0 +1,26 @@
+# Using Uno for Challenge Problem
+
+## Generate Plan file
+
+From the (master dataframe generation process)[https://github.com/ECP-CANDLE/topN_generator/blob/master/build_master.ipynb], the unique list of cell line and drug will be generated (top21_cell.txt and top21_drug.txt respectively)
+
+```
+$ python plangen.py --fs_name cell drug --fs_paths top21_cell.txt top21_drug.txt --fs_parts 4 1 --out_dir . --overwrite
+...
+plangen_cell694-p4_drug1492-p1.json JSON file written
+```
+
+## Node specific dataframe generation
+
+This part is already integrated in the Challenge Problem workflow, but for the testing purpose, you can run this command to generate a dataframe for node 1.1
+
+```
+$ python topN_to_uno.py --dataframe_from top21.parquet --plan plangen_cell694-p4_drug1492-p1.json --incremental --cell_feature_selection lincs1000 --node 1.1 --output TopN_1.1_uno.h5
+```
+
+## Running Uno with new dataframe
+
+```
+# set CANDLE_DATA_DIR to point direcotry containing topN_1.1_uno.h5
+$ python uno_baseline_keras2.py --conf uno_auc_model.txt --use_exported_data topN_1.1_uno.h5
+```

Pilot1/Uno/planargs.py

@@ -0,0 +1,73 @@
+"""
+plangen command line argument definitions
+"""
+
+import argparse
+
+partitioning_strategies = ['windows', 'undefine1', 'undefined2']         # to be completed ?????????????
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(
+        description='feature-set partioning'
+    )
+
+    parser.add_argument('--in_dir',
+                        type=str,
+                        help='Directory containing feature-set list files')
+
+    parser.add_argument('--out_dir',
+                        default='results',
+                        type=str,
+                        help='Directory to contain generated plan files')
+
+    parser.add_argument('--json',
+                        action='store_true',
+                        help='Generate plan in JSON format')
+
+    parser.add_argument('--overwrite',
+                        action='store_true',
+                        help='Accept non-empty out_dir, contents overwritten')
+
+    parser.add_argument('--partition_strategy',
+                        choices=partitioning_strategies,
+                        default=partitioning_strategies[0],
+                        help='Specify a feature-set partitioning strategy')
+
+    # The following fs_* arguments are required, the number of values specified for each
+    # must match, and at least two values are required for each
+
+    parser.add_argument('--fs_names',
+                        required=True,
+                        type=str,
+                        nargs='+',
+                        help='Specify a list of (arbitrary) feature-set names')
+
+    parser.add_argument('--fs_paths',
+                        required=True,
+                        type=str,
+                        nargs='+',
+                        help='Specify a list of feature-set file paths')
+
+    parser.add_argument('--fs_parts',
+                        required=True,
+                        type=int,
+                        nargs='+',
+                        help='Specify a list of partition counts')
+
+    parser.add_argument('--verbose',
+                        action='store_true',
+                        help='Verbosity')
+
+    parser.add_argument('--debug',
+                        action='store_true',
+                        help='Data structure dumps, etc')
+
+    parser.add_argument('--test',
+                        action='store_true',
+                        help='Test plan navigation and entry retrieval')
+
+    parser.add_argument('--maxdepth', type=int, default=0)
+    parser.add_argument('--print_tree', action='store_true')
+    args = parser.parse_args()
+    return args

Pilot1/Uno/plangen.py

@@ -4,6 +4,7 @@
 import os
 import sqlite3
 import sys
+import planargs
 from abc import ABC, abstractmethod  # abstract class support
 from collections import OrderedDict, deque, namedtuple
 from datetime import datetime
@@ -139,8 +140,8 @@ def validate_args(args):
         sys.exit("Terminating due to error")
 
     # construct a partitioning object exporting a partion() function
-    if args.partition_strategy == "leaveout":
-        generator = LeaveoutSubsetGenerator()
+    # if args.partition_strategy == "leaveout":
+    generator = LeaveoutSubsetGenerator()
 
     # return feature-set contents lists
     return generator, fs_content

Pilot1/Uno/topN_to_uno.py

@@ -242,7 +242,7 @@ def build_dataframe(args):
         x_test_1.columns = [""] * len(x_test_1.columns)
 
     # store
-    store = pd.HDFStore(args.output, "w", complevel=9, complib="blosc:snappy")
+    store = pd.HDFStore(args.output, "w")
     store.put("y_train", y_train, format="table")
     store.put("y_val", y_val, format="table")
     store.put("x_train_0", x_train_0, format="table")