Don't CUDA sync by default on timer (#1029)

Summary:
Pull Request resolved: #1029

Only CUDA sync when it is explicitly requested. In this case we are essentially performing CUDA sync by default

Reviewed By: JKSenthil, diego-urgell

Differential Revision: D81793986

fbshipit-source-id: 32e61c2f0ad425699b31298953dee9ea9b094bce

Author: rehm-g

tests/utils/test_timer_gpu.py

@@ -20,14 +20,32 @@ class TimerGPUTest(unittest.TestCase):
     @skip_if_not_gpu
     @patch("torch.cuda.synchronize")
     def test_timer_synchronize(self, mock_synchronize: MagicMock) -> None:
-        """Make sure that torch.cuda.synchronize() is called when GPU is present."""
+        """Make sure that torch.cuda.synchronize() is not called by default when GPU is present."""
 
         start_event = torch.cuda.Event(enable_timing=True)
         end_event = torch.cuda.Event(enable_timing=True)
         timer = Timer()
 
         # Do not explicitly call synchronize, timer must call it for test to pass.
 
+        with timer.time("action_1"):
+            start_event.record()
+            time.sleep(0.5)
+            end_event.record()
+
+        self.assertEqual(mock_synchronize.call_count, 0)
+
+    @skip_if_not_gpu
+    @patch("torch.cuda.synchronize")
+    def test_timer_synchronize_when_explicit(self, mock_synchronize: MagicMock) -> None:
+        """Make sure that torch.cuda.synchronize() is called when GPU is present and sync is explicit."""
+
+        start_event = torch.cuda.Event(enable_timing=True)
+        end_event = torch.cuda.Event(enable_timing=True)
+        timer = Timer(cuda_sync=True)
+
+        # Do not explicitly call synchronize, timer must call it for test to pass.
+
         with timer.time("action_1"):
             start_event.record()
             time.sleep(0.5)

torchtnt/utils/timer.py

@@ -146,9 +146,7 @@ def __init__(
             raise ValueError(
                 "CUDA must be available in order to enable CUDA synchronization."
             )
-        self.cuda_sync: bool = (
-            cuda_sync if cuda_sync is not None else torch.cuda.is_available()
-        )
+        self.cuda_sync: bool = cuda_sync if cuda_sync is not None else False
         self.verbose = verbose
         self.recorded_durations: Dict[str, List[float]] = defaultdict(list)