make chunkszie configurable

Author: akahles

spladder/merge.py

@@ -356,9 +356,6 @@ def run_merge(options):
     if options.do_prune:
         prune_tag = '_pruned'
 
-    chunksize = 10
-    assert chunksize > 0
-
     fn_out = '%s/spladder/genes_graph_conf%i.%s%s.pickle' % (options.outdir , options.confidence, options.merge, prune_tag)
     if options.validate_sg:
         fn_out_count = '%s/spladder/genes_graph_conf%i.%s%s.validated.count.hdf5' % (options.outdir, options.confidence, options.merge, prune_tag)
@@ -370,25 +367,25 @@ def run_merge(options):
             jobinfo = []
             PAR = dict()
             PAR['options'] = options
-            levels = int(math.ceil(math.log(len(options.samples), chunksize)))
+            levels = int(math.ceil(math.log(len(options.samples), options.chunksize)))
             for level in range(1, levels + 1):
                 print('merging files on level %i' % level)
                 if level == 1:
                     merge_list = np.array(['%s/spladder/genes_graph_conf%i.%s%s.pickle' % (options.outdir, options.confidence, x, prune_tag) for x in options.samples])
                 else:
                     merge_list = np.array(level_files)
                 level_files = []
-                for c_idx in range(0, len(merge_list), chunksize):
+                for c_idx in range(0, len(merge_list), options.chunksize):
                     if level == levels:
-                        assert(len(merge_list) <= chunksize)
+                        assert len(merge_list) <= options.chunksize, 'chunksize is %i but merge_list has length %i with: %s' % (options.chunksize, len(merge_list), str(merge_list))
                         fn = fn_out
                     else:
-                        fn = '%s/spladder/genes_graph_conf%i.%s%s_level%i_chunk%i_%i.pickle' % (options.outdir, options.confidence, options.merge, prune_tag, level, c_idx, min(len(merge_list), c_idx + chunksize))
+                        fn = '%s/spladder/genes_graph_conf%i.%s%s_level%i_chunk%i_%i.pickle' % (options.outdir, options.confidence, options.merge, prune_tag, level, c_idx, min(len(merge_list), c_idx + options.chunksize))
                     level_files.append(fn)
                     if os.path.exists(fn):
                         continue
-                    print('submitting level %i chunk %i to %i' % (level, c_idx, min(len(merge_list), c_idx + chunksize)))
-                    chunk_idx = np.arange(c_idx, min(len(merge_list), c_idx + chunksize))
+                    print('submitting level %i chunk %i to %i' % (level, c_idx, min(len(merge_list), c_idx + options.chunksize)))
+                    chunk_idx = np.arange(c_idx, min(len(merge_list), c_idx + options.chunksize))
                     PAR['merge_list'] = merge_list[chunk_idx]
                     PAR['fn_out'] = fn
                     jobinfo.append(rp.rproc('merge_genes_by_splicegraph', PAR, 20000*level, options.options_rproc, 40*60))
@@ -402,7 +399,7 @@ def run_merge(options):
             chunk_end = min(len(merge_list), chunk_end)
 
             if curr_level == max_level:
-                assert(len(merge_list) <= chunksize)
+                assert len(merge_list) <= options.chunksize, 'chunksize is %i but merge_list has length %i with: %s' % (options.chunksize, len(merge_list), str(merge_list))
                 fn = fn_out
             else:
                 fn = '%s/spladder/genes_graph_conf%i.%s%s_level%i_chunk%i_%i.pickle' % (options.outdir, options.confidence, options.merge, prune_tag, curr_level, chunk_start, chunk_end)

spladder/spladder.py

@@ -67,7 +67,8 @@ def parse_options(argv):
     graph.add_argument('-c', '--confidence', dest='confidence', metavar='INT', type=int, help='confidence level (0 lowest to 3 highest) [3]', default=3)
     graph.add_argument('-I', '--iterations', dest='insert_intron_iterations', metavar='INT', type=int, help='number of iterations to insert new introns into the graph [5]', default=5)
     graph.add_argument('-M', '--merge-strat', dest='merge', metavar='<STRAT>', help='merge strategy, where <STRAT> is one of: single, merge_bams, merge_graphs, merge_all [merge_graphs]', default='merge_graphs')
-    graph.add_argument('--chunked-merge', dest='chunked_merge', metavar="LEVEL MAX_LEVEL START END", nargs='+', action='append', help='provide infor for external merge with START being 0-based and END non-inclusive', default=[])
+    graph.add_argument('--chunked-merge', dest='chunked_merge', metavar="LEVEL MAX_LEVEL START END", nargs='+', action='append', help='provide info for external merge with START being 0-based and END non-inclusive', default=[])
+    graph.add_argument('--chunksize', dest='chunksize', metavar='INT', type=int, help='chunksize for chunked merge [10]', default=10)
     graph.add_argument('--insert-ir', dest='insert_ir', action='store_true', help='insert intron retentions [on]', default=True)
     graph.add_argument('--no-insert-ir', dest='insert_ir', action='store_false', default=True)
     graph.add_argument('--insert-es', dest='insert_es', action='store_true', help='insert cassette exons [on]', default=True)