Merge pull request #4 from Hexotical/file-ids

Use file ids instead of local paths.

Author: Hexotical

toil/python/draft.py

@@ -42,7 +42,6 @@ def run(self, fileStore):
                           working_dir=tempDir,
                           parameters=[os.path.join(tempDir, "FastQCone_script.sh")],
                           entrypoint="/bin/bash",
-                          user='root',
                           stderr=True,
                           demux=True,
                           volumes={tempDir: {"bind": tempDir}})
@@ -57,7 +56,7 @@ def run(self, fileStore):
         fastqc_output_path = os.path.join(os.path.abspath(current_working_dir), 'readsone_fastqc.html')
         fileStore.exportFile(output_file_id, f'file://{fastqc_output_path}')
 
-        return {"fastqc_output_path": fastqc_output_path}
+        return {"fastqc": output_file_id}
 
 
 class FastQCtwoCls(Job):
@@ -89,7 +88,6 @@ def run(self, fileStore):
                           working_dir=tempDir,
                           parameters=[os.path.join(tempDir, "FastQCtwo_script.sh")],
                           entrypoint="/bin/bash",
-                          user='root',
                           stderr=True,
                           demux=True,
                           volumes={tempDir: {"bind": tempDir}})
@@ -104,7 +102,7 @@ def run(self, fileStore):
         fastqc_output_path = os.path.join(os.path.abspath(current_working_dir), 'readstwo_fastqc.html')
         fileStore.exportFile(output_file_id, f'file://{fastqc_output_path}')
 
-        return {"fastqc_output_path": fastqc_output_path}
+        return {"fastqc": output_file_id}
 
 
 class SalmonIndexCls(Job):
@@ -141,7 +139,6 @@ def run(self, fileStore):
                           working_dir=tempDir,
                           parameters=[os.path.join(tempDir, "SalmonIndex_script.sh")],
                           entrypoint="/bin/bash",
-                          user='root',
                           stderr=True,
                           demux=True,
                           volumes={tempDir: {"bind": tempDir}})
@@ -157,8 +154,7 @@ def run(self, fileStore):
         index_output_path = os.path.join(os.path.abspath(current_working_dir), 'index.tar.gz')
         fileStore.exportFile(output_file_id, f'file://{index_output_path}')
 
-        
-        return {"index" : index_output_path}
+        return {"index": output_file_id}
 
 
 class SalmonAlignQuantCls(Job):
@@ -181,11 +177,9 @@ def run(self, fileStore):
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
-        
-        fpath_reads1 = fileStore.readGlobalFile(self.reads1, userPath=os.path.join(tempDir, os.path.basename(self.reads1)))
 
+        fpath_reads1 = fileStore.readGlobalFile(self.reads1, userPath=os.path.join(tempDir, os.path.basename(self.reads1)))
         fpath_reads2 = fileStore.readGlobalFile(self.reads2, userPath=os.path.join(tempDir, os.path.basename(self.reads2)))
-
         fpath_index = fileStore.readGlobalFile(self.index, userPath=os.path.join(tempDir, os.path.basename(self.index)))
 
 
@@ -201,7 +195,6 @@ def run(self, fileStore):
                           working_dir=tempDir,
                           parameters=[os.path.join(tempDir, "SalmonAlignQuant_script.sh")],
                           entrypoint="/bin/bash",
-                          user='root',
                           stderr=True,
                           demux=True,
                           volumes={tempDir: {"bind": tempDir}})
@@ -215,7 +208,7 @@ def run(self, fileStore):
         quant_output_path = os.path.join(os.path.abspath(current_working_dir), 'quant.tar.gz')
         fileStore.exportFile(output_file_id, f'file://{quant_output_path}')
 
-        return {"quant": quant_output_path}
+        return {"quant": output_file_id}
 
 
 if __name__ == "__main__":
@@ -225,23 +218,28 @@ def run(self, fileStore):
     pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
 
     with Toil(options) as fileStore:
-        reads1 = fileStore.importFile('file://' + os.path.join(pkg_root, "test_data/reads_1.fq.gz"))
-        reads2 = fileStore.importFile('file://' + os.path.join(pkg_root, "test_data/reads_2.fq.gz"))
-        ref_txome = fileStore.importFile('file://' + os.path.join(pkg_root, "test_data/transcriptome.fa"))
+        # import all files into the jobstore and retrieve their file ID references
+        # that way jobs run remotely can fetch from the centralized jobstore without sharing a filesystem
+        reads1_file_id = fileStore.importFile(f'file://{os.path.join(pkg_root, "test_data/reads_1.fq.gz")}')
+        reads2_file_id = fileStore.importFile(f'file://{os.path.join(pkg_root, "test_data/reads_2.fq.gz")}')
+        ref_transcriptome_file_id = fileStore.importFile(f'file://{os.path.join(pkg_root, "test_data/transcriptome.fa")}')
 
-        FastQCone = FastQConeCls(reads=reads1)
-        fastqc_output_report_path = FastQCone.rv("fastqc_res")
+        fastqc_job_1 = FastQConeCls(reads=reads1_file_id)  # this is our root job, which runs first
+        fastqc_output_report_file_id_1 = fastqc_job_1.rv("fastqc")  # "rv" == return value
 
-        FastQCtwo = FastQCtwoCls(reads=reads2)
-        FastQCtwo_fastqc_res = FastQCtwo.rv("fastqc_res")
-        FastQCone.addChild(FastQCtwo)
-        
-        SalmonIndex = FastQCone.addChild(SalmonIndexCls(ref_txome=ref_txome))
-        SalmonIndex_index = SalmonIndex.rv("index")
+        fastqc_job_2 = FastQCtwoCls(reads=reads2_file_id)
+        fastqc_output_report_file_id_2 = fastqc_job_2.rv("fastqc")
+        fastqc_job_1.addChild(fastqc_job_2)  # fastqc_job_2 will run after our root job
 
-        SalmonAlignQuant = FastQCone.addFollowOn(SalmonAlignQuantCls(reads1=reads1, reads2=reads2, index=(SalmonIndex_index)))
-        SalmonAlignQuant_quant = SalmonAlignQuant.rv("quant")
+        salmon_index_job = SalmonIndexCls(ref_txome=ref_transcriptome_file_id)
+        index_file_id = salmon_index_job.rv("index")
+        fastqc_job_1.addChild(salmon_index_job)  # salmon_index_job will run after our root job
+
+        salmon_align_quant_job = SalmonAlignQuantCls(reads1=reads1_file_id, reads2=reads2_file_id, index=index_file_id)
+        fastqc_job_1.addFollowOn(salmon_align_quant_job)  # run after this job and all of its children
+        # we don't do anything with our results, but we could
+        salmon_align_quant_file_id = salmon_align_quant_job.rv("quant")
 
-        fileStore.start(FastQCone)
+        fileStore.start(fastqc_job_1)
 
 #/home/hexotical/bioinformatics-workflows/toil/python/index.tar.gz