Make benchmark for training work with multiple gpus

Summary:
1. For tensorboard_trace_handler, ``worker_name`` should be unique for each worker in distributed scenario.
2. Turn off saving `profiler-trace-iter{}.json` in the output dir when `save_tensorboard=True` as it seems like we cannot have both enabled right now (discussions are here https://fb.workplace.com/groups/289111155699012/permalink/640990860511038/)

Reviewed By: ppwwyyxx

Differential Revision: D31312437

fbshipit-source-id: b50d80c9f288a35483d9a41ce07e0307c15c297f

Author: HannaMao

detectron2/engine/hooks.py

@@ -414,7 +414,8 @@ def before_step(self):
                         self._output_dir,
                         "log",
                         "profiler-tensorboard-iter{}".format(self.trainer.iter),
-                    )
+                    ),
+                    f"worker{comm.get_rank()}",
                 )
             else:
                 on_trace_ready = None
@@ -434,21 +435,22 @@ def after_step(self):
         if self._profiler is None:
             return
         self._profiler.__exit__(None, None, None)
-        PathManager.mkdirs(self._output_dir)
-        out_file = os.path.join(
-            self._output_dir, "profiler-trace-iter{}.json".format(self.trainer.iter)
-        )
-        if "://" not in out_file:
-            self._profiler.export_chrome_trace(out_file)
-        else:
-            # Support non-posix filesystems
-            with tempfile.TemporaryDirectory(prefix="detectron2_profiler") as d:
-                tmp_file = os.path.join(d, "tmp.json")
-                self._profiler.export_chrome_trace(tmp_file)
-                with open(tmp_file) as f:
-                    content = f.read()
-            with PathManager.open(out_file, "w") as f:
-                f.write(content)
+        if not self._save_tensorboard:
+            PathManager.mkdirs(self._output_dir)
+            out_file = os.path.join(
+                self._output_dir, "profiler-trace-iter{}.json".format(self.trainer.iter)
+            )
+            if "://" not in out_file:
+                self._profiler.export_chrome_trace(out_file)
+            else:
+                # Support non-posix filesystems
+                with tempfile.TemporaryDirectory(prefix="detectron2_profiler") as d:
+                    tmp_file = os.path.join(d, "tmp.json")
+                    self._profiler.export_chrome_trace(tmp_file)
+                    with open(tmp_file) as f:
+                        content = f.read()
+                with PathManager.open(out_file, "w") as f:
+                    f.write(content)
 
 
 class AutogradProfiler(TorchProfiler):