Merge pull request #154 from jbernal0019/master

Fix bug when a chris link points to an empty dir

Author: jbernal0019

pfcon/storage/base_storage.py

@@ -58,8 +58,10 @@ def process_chrislink_files(self, job_incoming_dir):
 
         for folder in linked_path_top_folders:
             if folder not in self._linked_paths:
-                self.deletesrc(os.path.join(job_incoming_dir, folder))
-
+                try:
+                    self.deletesrc(os.path.join(job_incoming_dir, folder))
+                except FileNotFoundError:
+                    pass
         return self._nlinks
 
     def _process_chrislink_files(self, dir):
@@ -84,7 +86,10 @@ def _process_chrislink_files(self, dir):
                             dst_path = os.path.join(root, source_trace_dir)
 
                             if not os.path.isdir(dst_path):  # only copy once to a dest path
-                                self.copysrc(abs_path, dst_path)
+                                try:
+                                    self.copysrc(abs_path, dst_path)
+                                except FileNotFoundError:
+                                    pass
                                 self._already_copied_src_set.add(abs_path)
                                 self._process_chrislink_files(dst_path)  # recursive call
 

tests/test_resources.py

@@ -87,7 +87,7 @@ def test_post(self):
         data = {
             'jid': job_id,
             'entrypoint': ['python3', '/usr/local/bin/simplefsapp'],
-            'args': ['--saveinputmeta', '--saveoutputmeta', '--dir', '/share/incoming'],
+            'args': ['--dir', '/share/incoming'],
             'auid': 'cube',
             'number_of_workers': '1',
             'cpu_limit': '1000',
@@ -130,8 +130,8 @@ def setUp(self):
         self.image = 'fnndsc/pl-simplefsapp'
         self.env = []
 
-        self.cmd = ['python3', '/usr/local/bin/simplefsapp', '--saveinputmeta',
-                    '--saveoutputmeta', '--dir', '/share/incoming', '/share/outgoing']
+        self.cmd = ['python3', '/usr/local/bin/simplefsapp',
+                    '--dir', '/share/incoming', '/share/outgoing']
 
         self.resources_dict = {'number_of_workers': 1, 'cpu_limit': 1000,
                                'memory_limit': 200, 'gpu_limit': 0}

tests/test_resources_innetwork.py

@@ -109,7 +109,7 @@ def test_post(self):
         data = {
             'jid': job_id,
             'entrypoint': ['python3', '/usr/local/bin/simplefsapp'],
-            'args': ['--saveinputmeta', '--saveoutputmeta', '--dir', '/share/incoming'],
+            'args': ['--dir', '/share/incoming'],
             'auid': 'cube',
             'number_of_workers': '1',
             'cpu_limit': '1000',
@@ -149,7 +149,7 @@ def test_post_with_chris_links(self):
         data = {
             'jid': job_id,
             'entrypoint': ['python3', '/usr/local/bin/simpledsapp'],
-            'args': ['--saveinputmeta', '--saveoutputmeta', '--prefix', 'lo'],
+            'args': ['--prefix', 'lo'],
             'auid': 'cube',
             'number_of_workers': '1',
             'cpu_limit': '1000',
@@ -207,8 +207,8 @@ def setUp(self):
         self.image = 'fnndsc/pl-simplefsapp'
         self.env = []
 
-        self.cmd = ['python3', '/usr/local/bin/simplefsapp', '--saveinputmeta',
-                    '--saveoutputmeta', '--dir', '/share/incoming', '/share/outgoing']
+        self.cmd = ['python3', '/usr/local/bin/simplefsapp',
+                    '--dir', '/share/incoming', '/share/outgoing']
 
         self.resources_dict = {'number_of_workers': 1, 'cpu_limit': 1000,
                                'memory_limit': 200, 'gpu_limit': 0}

tests/test_resources_innetwork_fs.py

@@ -91,7 +91,7 @@ def test_post(self):
         data = {
             'jid': job_id,
             'entrypoint': ['python3', '/usr/local/bin/simplefsapp'],
-            'args': ['--saveinputmeta', '--saveoutputmeta', '--dir', '/share/incoming'],
+            'args': ['--dir', '/share/incoming'],
             'auid': 'cube',
             'number_of_workers': '1',
             'cpu_limit': '1000',
@@ -136,8 +136,8 @@ def setUp(self):
         self.image = 'fnndsc/pl-simplefsapp'
         self.env = []
 
-        self.cmd = ['python3', '/usr/local/bin/simplefsapp', '--saveinputmeta',
-                    '--saveoutputmeta', '--dir', '/share/incoming', '/share/outgoing']
+        self.cmd = ['python3', '/usr/local/bin/simplefsapp',
+                    '--dir', '/share/incoming', '/share/outgoing']
 
         self.resources_dict = {'number_of_workers': 1, 'cpu_limit': 1000,
                                'memory_limit': 200, 'gpu_limit': 0}

tests/test_resources_innetwork_fslink.py

@@ -95,7 +95,7 @@ def test_post(self):
         data = {
             'jid': job_id,
             'entrypoint': ['python3', '/usr/local/bin/simplefsapp'],
-            'args': ['--saveinputmeta', '--saveoutputmeta', '--dir', '/share/incoming'],
+            'args': ['--dir', '/share/incoming'],
             'auid': 'cube',
             'number_of_workers': '1',
             'cpu_limit': '1000',
@@ -135,7 +135,7 @@ def test_post_with_chris_links(self):
         data = {
             'jid': job_id,
             'entrypoint': ['python3', '/usr/local/bin/simpledsapp'],
-            'args': ['--saveinputmeta', '--saveoutputmeta', '--prefix', 'lo'],
+            'args': ['--prefix', 'lo'],
             'auid': 'cube',
             'number_of_workers': '1',
             'cpu_limit': '1000',
@@ -194,8 +194,8 @@ def setUp(self):
         self.image = 'fnndsc/pl-simplefsapp'
         self.env = []
 
-        self.cmd = ['python3', '/usr/local/bin/simplefsapp', '--saveinputmeta',
-                    '--saveoutputmeta', '--dir', '/share/incoming', '/share/outgoing']
+        self.cmd = ['python3', '/usr/local/bin/simplefsapp',
+                    '--dir', '/share/incoming', '/share/outgoing']
 
         self.resources_dict = {'number_of_workers': 1, 'cpu_limit': 1000,
                                'memory_limit': 200, 'gpu_limit': 0}