Skip to content

Commit 0ebf9cd

Browse files
committed
add warning
Signed-off-by: Timothy Seah <tseah@anyscale.com>
1 parent 38f1ea1 commit 0ebf9cd

File tree

1 file changed

+5
-0
lines changed

1 file changed

+5
-0
lines changed

torchft/manager.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -733,6 +733,11 @@ def _async_quorum(
733733
# Only reset FR recording if available (requires NCCL Flight Recorder support)
734734
if hasattr(torch._C._distributed_c10d, "_reset_fr_recording_nccl"):
735735
torch._C._distributed_c10d._reset_fr_recording_nccl() # pyre-ignore
736+
else:
737+
self._logger.warn(
738+
"Unable to reset NCCL flight recorder recording so traces will be "
739+
"incorrect."
740+
)
736741
except Exception as e:
737742
self._logger.exception(f"got exception in pg configure: {e}")
738743
self.report_error(e)

0 commit comments

Comments
 (0)