Add error handling for executor deserialization in dgxcloud scheduler (#166)

Signed-off-by: Hemil Desai <[email protected]>

Author: hemildesai

nemo_run/run/torchx_backend/schedulers/dgxcloud.py

@@ -240,6 +240,10 @@ def _get_job_dirs() -> dict[str, dict[str, str]]:
 
     serializer = ZlibJSONSerializer()
     for app in data.values():
-        app["executor"] = fdl.build(serializer.deserialize(app["executor"]))
+        try:
+            app["executor"] = fdl.build(serializer.deserialize(app["executor"]))
+        except Exception as e:
+            log.warning(f"Failed to deserialize executor: {e}")
+            continue
 
     return data