fix MultimodalRegistry (#3699)

Author: yuanlehome

fastdeploy/engine/sched/resource_manager_v1.py

@@ -137,11 +137,7 @@ def _get_num_new_tokens(self, request, token_budget):
             return num_new_tokens
 
         inputs = request.multimodal_inputs
-        if (
-            inputs["image_feature_urls"] is not None
-            or inputs["video_feature_urls"] is not None
-            or inputs["audio_feature_urls"] is not None
-        ):
+        if inputs["patch_idx"] is not None and inputs["patch_map"] is not None:
             pre_end_idx = request.num_computed_tokens
             new_end_idx = pre_end_idx + num_new_tokens
             # start

fastdeploy/multimodal/registry.py

@@ -14,32 +14,13 @@
 # limitations under the License.
 """
 
-from typing import Callable
-
 
 class MultimodalRegistry:
     """
     A registry for multimodal models
     """
 
-    mm_models: set[str] = {"Ernie4_5_VLMoeForConditionalGeneration"}
-
-    @classmethod
-    def register_model(cls, name: str = "") -> Callable:
-        """
-        Register model with the given name, class name is used if name is not provided.
-        """
-
-        def _register(model):
-            nonlocal name
-            if len(name) == 0:
-                name = model.__name__
-            if name in cls.mm_models:
-                raise ValueError(f"multimodal model {name} is already registered")
-            cls.mm_models.add(name)
-            return model
-
-        return _register
+    mm_models: set[str] = {"Ernie4_5_VLMoeForConditionalGeneration", "Ernie5MoeForCausalLM"}
 
     @classmethod
     def contains_model(cls, name: str) -> bool: