diff --git a/src/accelerate/tracking.py b/src/accelerate/tracking.py
index ecd722569fc..52d8fa3405d 100644
--- a/src/accelerate/tracking.py
+++ b/src/accelerate/tracking.py
@@ -340,16 +340,7 @@ def store_init_configuration(self, values: dict):
         """
         import wandb
 
-        if os.environ.get("WANDB_MODE") == "offline":
-            # In offline mode, restart wandb with config included
-            if hasattr(self, "run") and self.run:
-                self.run.finish()
-
-            init_kwargs = self.init_kwargs.copy()
-            init_kwargs["config"] = values
-            self.run = wandb.init(project=self.run_name, **init_kwargs)
-        else:
-            wandb.config.update(values, allow_val_change=True)
+        wandb.config.update(values, allow_val_change=True)
         logger.debug("Stored initial configuration hyperparameters to WandB")
 
     @on_main_process