fix sleep behavior only for last iteration, also address the parent dir issue

Author: Jeffrey

open_lm/file_utils.py

@@ -72,10 +72,10 @@ def remote_sync(local_dir, remote_dir, protocol):
 
 def remote_sync_with_expon_backoff(sync_every, local_dir, remote_dir, protocol, max_retries=6):
     for i in range(max_retries):
+        time.sleep(sync_every * 2**i)
         success = remote_sync(local_dir, remote_dir, protocol)
         if success:
             return True
-        time.sleep(sync_every * 2**i)
     return False
 
 

open_lm/main.py

@@ -381,7 +381,7 @@ def main(args):
     args.tensorboard = "tensorboard" in args.report_to or "all" in args.report_to
     args.checkpoint_path = os.path.join(log_base_path, "checkpoints")
     args.failed_checkpoint_path = os.path.join(log_base_path, "checkpoints_failed")
-    if is_master(args):
+    if is_master(args, local=args.log_local):
         args.tensorboard_path = os.path.join(log_base_path, "tensorboard") if args.tensorboard else ""
         for dirname in [args.tensorboard_path, args.checkpoint_path, args.failed_checkpoint_path]:
             if dirname:
@@ -932,8 +932,9 @@ def main(args):
     if remote_sync_process is not None:
         logging.info("Final remote sync.")
         terminate_sync_process(remote_sync_process)
+        # Can just pass in sync_every=0 for last sync, otherwise will unecessarily sleep.
         result = remote_sync_with_expon_backoff(
-            args.remote_sync_frequency,
+            0,
             os.path.join(args.logs, args.name),
             os.path.join(args.remote_sync, args.name),
             args.remote_sync_protocol,