support mtp in ep64 (#4280)

Author: freeliuzc

fastdeploy/spec_decode/base.py

@@ -38,13 +38,21 @@ def __init__(self, cfg: FDConfig):
         Init Speculative proposer
         """
         cfg.parallel_config.tp_group = None
+        cfg.parallel_config.ep_group = None
         self.cfg = deepcopy(cfg)
         cfg.parallel_config.tp_group = dist.get_group(
             cfg.parallel_config.data_parallel_rank + envs.FD_TP_GROUP_GID_OFFSET
         )
+        cfg.parallel_config.ep_group = dist.get_group(
+            cfg.parallel_config.data_parallel_size + envs.FD_TP_GROUP_GID_OFFSET
+        )
         self.cfg.parallel_config.tp_group = dist.get_group(
             cfg.parallel_config.data_parallel_rank + envs.FD_TP_GROUP_GID_OFFSET
         )
+        self.cfg.parallel_config.ep_group = dist.get_group(
+            cfg.parallel_config.data_parallel_size + envs.FD_TP_GROUP_GID_OFFSET
+        )
+
         self.parallel_config = self.cfg.parallel_config
         self.model_config = self.cfg.model_config
         self.speculative_config = self.cfg.speculative_config

fastdeploy/spec_decode/mtp.py

@@ -695,6 +695,9 @@ def _propose(self, target_hidden_states):
 
                 if substep != self.num_model_steps - 1:
                     target_hidden_states = self._get_self_hidden_states(hidden_states)
+            else:
+                if hasattr(self.model, "empty_input_forward"):
+                    self.model.empty_input_forward()
 
     def _get_self_hidden_states(self, hidden_states):
         target_hidden_states = eagle_get_self_hidden_states(