16
16
import uuid
17
17
from collections import Iterable # pylint: disable=unused-import
18
18
from io import open
19
- from typing import (Any , Callable , Dict , Generator , List , Optional , Set , Tuple ,
20
- Union , cast )
19
+ from typing import (Any , Callable , Dict , Generator , Iterator , List , Optional ,
20
+ Set , Tuple , Union , cast )
21
21
from typing_extensions import Text , TYPE_CHECKING # pylint: disable=unused-import
22
22
# move to a regular typing import when Python 3.3-3.6 is no longer supported
23
23
@@ -263,18 +263,6 @@ def stageFiles(pm, stageFunc=None, ignoreWritable=False, symLink=True, secret_st
263
263
n .write (p .resolved .encode ("utf-8" ))
264
264
ensure_writable (p .target )
265
265
266
- def collectFilesAndDirs (obj , out ):
267
- # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]], List[Dict[Text, Any]]) -> None
268
- if isinstance (obj , dict ):
269
- if obj .get ("class" ) in ("File" , "Directory" ):
270
- out .append (obj )
271
- else :
272
- for v in obj .values ():
273
- collectFilesAndDirs (v , out )
274
- if isinstance (obj , list ):
275
- for l in obj :
276
- collectFilesAndDirs (l , out )
277
-
278
266
279
267
def relocateOutputs (outputObj , # type: Union[Dict[Text, Any],List[Dict[Text, Any]]]
280
268
destination_path , # type: Text
@@ -289,7 +277,21 @@ def relocateOutputs(outputObj, # type: Union[Dict[Text, Any],List[Di
289
277
if action not in ("move" , "copy" ):
290
278
return outputObj
291
279
292
- def moveIt (src , dst ):
280
+ def _collectDirEntries (obj ):
281
+ # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> Iterator[Dict[Text, Any]]
282
+ if isinstance (obj , dict ):
283
+ if obj .get ("class" ) in ("File" , "Directory" ):
284
+ yield obj
285
+ else :
286
+ for sub_obj in obj .values ():
287
+ for dir_entry in _collectDirEntries (sub_obj ):
288
+ yield dir_entry
289
+ elif isinstance (obj , list ):
290
+ for sub_obj in obj :
291
+ for dir_entry in _collectDirEntries (sub_obj ):
292
+ yield dir_entry
293
+
294
+ def _relocate (src , dst ):
293
295
if action == "move" :
294
296
for a in output_dirs :
295
297
if src .startswith (a + "/" ):
@@ -298,7 +300,7 @@ def moveIt(src, dst):
298
300
# merge directories
299
301
for root , dirs , files in os .walk (src ):
300
302
for f in dirs + files :
301
- moveIt (os .path .join (root , f ), os .path .join (dst , f ))
303
+ _relocate (os .path .join (root , f ), os .path .join (dst , f ))
302
304
else :
303
305
shutil .move (src , dst )
304
306
return
@@ -313,10 +315,9 @@ def moveIt(src, dst):
313
315
else :
314
316
shutil .copy2 (src , dst )
315
317
316
- outfiles = [] # type: List[Dict[Text, Any]]
317
- collectFilesAndDirs (outputObj , outfiles )
318
+ outfiles = list (_collectDirEntries (outputObj ))
318
319
pm = PathMapper (outfiles , "" , destination_path , separateDirs = False )
319
- stageFiles (pm , stageFunc = moveIt , symLink = False )
320
+ stageFiles (pm , stageFunc = _relocate , symLink = False )
320
321
321
322
def _check_adjust (file ):
322
323
file ["location" ] = file_uri (pm .mapper (file ["location" ])[1 ])
0 commit comments