fix

Signed-off-by: oliver könig <okoenig@nvidia.com>

Author: ko3n1g

nemo_run/core/execution/dgxcloud.py

@@ -405,7 +405,11 @@ def fetch_logs(
         job_subdir = self.job_dir[len(nemo_run_home) + 1 :]  # +1 to remove the initial backslash
         self.pvc_job_dir = os.path.join(self.pvc_nemo_run_dir, job_subdir)
 
-        cmd.extend([f"{self.pvc_job_dir}/logs/output-{i}.log" for i in range(self.nodes)])
+        while files := len(glob.glob(f"{self.pvc_job_dir}/logs/output-*.log")) < self.nodes:
+            logger.info(f"Waiting for {self.nodes - len(files)} log files to be created...")
+            time.sleep(3)
+
+        cmd.extend(files)
 
         logger.info(f"Attempting to stream logs with command: {cmd}")