9
9
import json
10
10
import logging
11
11
import os
12
+ try :
13
+ from os import scandir
14
+ except ImportError :
15
+ from scandir import scandir
12
16
import shutil
13
17
import stat
14
18
import tempfile
@@ -266,7 +270,7 @@ def stageFiles(pm, stageFunc=None, ignoreWritable=False, symLink=True, secret_st
266
270
267
271
def relocateOutputs (outputObj , # type: Union[Dict[Text, Any],List[Dict[Text, Any]]]
268
272
destination_path , # type: Text
269
- output_dirs , # type: Set[Text]
273
+ source_directories , # type: Set[Text]
270
274
action , # type: Text
271
275
fs_access , # type: StdFsAccess
272
276
compute_checksum = True # type: bool
@@ -292,28 +296,31 @@ def _collectDirEntries(obj):
292
296
yield dir_entry
293
297
294
298
def _relocate (src , dst ):
299
+ if src == dst :
300
+ return
301
+
295
302
if action == "move" :
296
- for a in output_dirs :
297
- if src . startswith ( a + "/" ):
298
- _logger . debug ( "Moving %s to %s" , src , dst )
299
- if os . path . isdir ( src ) and os . path . isdir ( dst ):
300
- # merge directories
301
- for root , dirs , files in os . walk ( src ):
302
- for f in dirs + files :
303
- _relocate (os . path . join ( root , f ), os .path .join (dst , f ))
304
- else :
305
- shutil .move (src , dst )
303
+ # do not move anything if we are trying to move an entity from
304
+ # outside of the source directories
305
+ if any ( src . startswith ( path + "/" ) for path in source_directories ):
306
+ _logger . debug ( "Moving %s to %s" , src , dst )
307
+ if os . path . isdir ( src ) and os . path . isdir ( dst ):
308
+ # merge directories
309
+ for dir_entry in scandir ( src ) :
310
+ _relocate (dir_entry , os .path .join (dst , dir_entry . name ))
311
+ else :
312
+ shutil .move (src , dst )
306
313
return
307
- if src != dst :
308
- _logger .debug ("Copying %s to %s" , src , dst )
309
- if os .path .isdir (src ):
310
- if os .path .isdir (dst ):
311
- shutil .rmtree (dst )
312
- elif os .path .isfile (dst ):
313
- os .unlink (dst )
314
- shutil .copytree (src , dst )
315
- else :
316
- shutil .copy2 (src , dst )
314
+
315
+ _logger .debug ("Copying %s to %s" , src , dst )
316
+ if os .path .isdir (src ):
317
+ if os .path .isdir (dst ):
318
+ shutil .rmtree (dst )
319
+ elif os .path .isfile (dst ):
320
+ os .unlink (dst )
321
+ shutil .copytree (src , dst )
322
+ else :
323
+ shutil .copy2 (src , dst )
317
324
318
325
outfiles = list (_collectDirEntries (outputObj ))
319
326
pm = PathMapper (outfiles , "" , destination_path , separateDirs = False )
@@ -332,30 +339,35 @@ def _check_adjust(file):
332
339
# If there are symlinks to intermediate output directories, we want to move
333
340
# the real files into the final output location. If a file is linked more than once,
334
341
# make an internal relative symlink.
342
+ def relink (relinked , # type: Dict[Text, Text]
343
+ root_path # type: Text
344
+ ):
345
+ for dir_entry in scandir (root_path ):
346
+ path = dir_entry .path
347
+ if os .path .islink (path ):
348
+ real_path = os .path .realpath (path )
349
+ if real_path in relinked :
350
+ link_name = relinked [real_path ]
351
+ if onWindows ():
352
+ if os .path .isfile (path ):
353
+ shutil .copy (os .path .relpath (link_name , path ), path )
354
+ elif os .path .exists (path ) and os .path .isdir (path ):
355
+ shutil .rmtree (path )
356
+ copytree_with_merge (os .path .relpath (link_name , path ), path )
357
+ else :
358
+ os .unlink (path )
359
+ os .symlink (os .path .relpath (link_name , path ), path )
360
+ else :
361
+ if any (real_path .startswith (path + "/" ) for path in source_directories ):
362
+ os .unlink (path )
363
+ os .rename (real_path , path )
364
+ relinked [real_path ] = path
365
+ if os .path .isdir (path ):
366
+ relink (relinked , path )
367
+
335
368
if action == "move" :
336
369
relinked = {} # type: Dict[Text, Text]
337
- for root , dirs , files in os .walk (destination_path ):
338
- for f in dirs + files :
339
- path = os .path .join (root , f )
340
- if os .path .islink (path ):
341
- rp = os .path .realpath (path )
342
- if rp in relinked :
343
- if onWindows ():
344
- if os .path .isfile (path ):
345
- shutil .copy (os .path .relpath (relinked [rp ], path ), path )
346
- elif os .path .exists (path ) and os .path .isdir (path ):
347
- shutil .rmtree (path )
348
- copytree_with_merge (os .path .relpath (relinked [rp ], path ), path )
349
- else :
350
- os .unlink (path )
351
- os .symlink (os .path .relpath (relinked [rp ], path ), path )
352
- else :
353
- for od in output_dirs :
354
- if rp .startswith (od + "/" ):
355
- os .unlink (path )
356
- os .rename (rp , path )
357
- relinked [rp ] = path
358
- break
370
+ relink (relinked , destination_path )
359
371
360
372
return outputObj
361
373
0 commit comments