test

Signed-off-by: oliver könig <okoenig@nvidia.com>

Author: ko3n1g

nemo_run/core/execution/dgxcloud.py

@@ -405,21 +405,35 @@ def fetch_logs(
         job_subdir = self.job_dir[len(nemo_run_home) + 1 :]  # +1 to remove the initial backslash
         self.pvc_job_dir = os.path.join(self.pvc_nemo_run_dir, job_subdir)
 
-        cmd.extend([glob.glob(f"{self.pvc_job_dir}/logs/output-*.log")])
+        cmd.extend([f"{self.pvc_job_dir}/logs/output-{i}.log" for i in range(self.nodes)])
 
         logger.info(f"Attempting to stream logs with command: {cmd}")
 
         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True, bufsize=1)
 
-        try:
-            for line in iter(proc.stdout.readline, ""):
-                if line:
-                    yield line.rstrip("\n")
-                if proc.poll() is not None:
-                    break
-        finally:
-            proc.terminate()
-            proc.wait(timeout=2)
+        if stream:
+            while True:
+                try:
+                    for line in iter(proc.stdout.readline, ""):
+                        if line:
+                            yield line.rstrip("\n")
+                        if proc.poll() is not None:
+                            break
+                except Exception as e:
+                    logger.error(f"Error streaming logs: {e}")
+                    time.sleep(3)
+                    continue
+
+        else:
+            try:
+                for line in iter(proc.stdout.readline, ""):
+                    if line:
+                        yield line.rstrip("\n")
+                    if proc.poll() is not None:
+                        break
+            finally:
+                proc.terminate()
+                proc.wait(timeout=2)
 
     def cancel(self, job_id: str):
         # Retrieve the authentication token for the REST calls