We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 38f1ea1 commit 0ebf9cdCopy full SHA for 0ebf9cd
torchft/manager.py
@@ -733,6 +733,11 @@ def _async_quorum(
733
# Only reset FR recording if available (requires NCCL Flight Recorder support)
734
if hasattr(torch._C._distributed_c10d, "_reset_fr_recording_nccl"):
735
torch._C._distributed_c10d._reset_fr_recording_nccl() # pyre-ignore
736
+ else:
737
+ self._logger.warn(
738
+ "Unable to reset NCCL flight recorder recording so traces will be "
739
+ "incorrect."
740
+ )
741
except Exception as e:
742
self._logger.exception(f"got exception in pg configure: {e}")
743
self.report_error(e)
0 commit comments