Support deepseekv3 (#3449) (#3455)

Author: blzheng

examples/cpu/llm/inference/distributed/run_generation_with_deepspeed.py

@@ -294,7 +294,9 @@ def get_repo_root(model_name_or_path):
 def get_checkpoint_files(model_name_or_path):
     cached_repo_dir = get_repo_root(model_name_or_path)
     glob_pattern = "*.[bp][it][n]"
-    if re.search("deepseek-v2", model_name_or_path, re.IGNORECASE):
+    if re.search("deepseek-v2", model_name_or_path, re.IGNORECASE) or re.search(
+        "deepseek-v3", model_name_or_path, re.IGNORECASE
+    ):
         glob_pattern = "*.[sbp][ait][fn][e][t][e][n][s][o][r][s]"
     # extensions: .bin | .pt
     # creates a list of paths from all downloaded files in cache dir

examples/cpu/llm/inference/run.py

@@ -594,6 +594,7 @@ def main(args_in: Optional[List[str]] = None) -> None:
                 "maira": ("/maira2_local_shard"),
                 "jamba": ("/jamba_local_shard"),
                 "deepseek-v2": ("/deepseekv2_local_shard"),
+                "deepseek-v3": ("/deepseekv3_local_shard"),
             }
             model_type = next(
                 (

examples/cpu/llm/inference/single_instance/run_quantization.py

@@ -45,7 +45,7 @@
 from llm.inference.utils.model_class.whisper import WhisperConfig
 from llm.inference.utils.model_class.maira2 import MAIRA2Config
 from llm.inference.utils.model_class.jamba import JambaConfig
-from llm.inference.utils.model_class.deepseek import DeepseekV2Config
+from llm.inference.utils.model_class.deepseek import DeepseekV2Config, DeepseekV3Config
 
 parser = argparse.ArgumentParser("LLM generation script (int8 path)", add_help=False)
 parser.add_argument(
@@ -437,6 +437,8 @@ def download_and_open(url: str) -> Image.Image:
     model = JambaConfig(args.model_id)
 elif re.search("deepseekv2", config.architectures[0], re.IGNORECASE):
     model = DeepseekV2Config(args.model_id)
+elif re.search("deepseekv3", config.architectures[0], re.IGNORECASE):
+    model = DeepseekV3Config(args.model_id)
 else:
     raise AssertionError("Not support %s." % (args.model_id))
 

examples/cpu/llm/inference/utils/model_class/deepseek.py

@@ -44,3 +44,43 @@ def get_user_model(self, config, benchmark):
                 trust_remote_code=True,
             )
         return self.model
+
+
+class DeepseekV3Config(LLMConfig):
+    def __init__(self, model_id):
+        self.name = "deepseekv3"
+        self.model_id = model_id
+        self.to_channels_last = False
+        self.example_inputs_mode = EXAMPLE_INPUTS_MODE.MASK_POS_KV
+
+        self.use_global_past_key_value = True
+        self.use_ipex_autotune = True
+
+    def get_user_model(self, config, benchmark):
+        if benchmark:
+            try:
+                with ipex.OnDevice(dtype=torch.float, device="meta"):
+                    self.model = AutoModelForCausalLM.from_config(
+                        config, trust_remote_code=True
+                    )
+            except (RuntimeError, AttributeError):
+                self.model = AutoModelForCausalLM.from_config(
+                    config, trust_remote_code=True
+                )
+            except Exception:
+                self.model = AutoModelForCausalLM.from_pretrained(
+                    self.model_id,
+                    torch_dtype=torch.bfloat16,
+                    config=config,
+                    low_cpu_mem_usage=True,
+                    trust_remote_code=True,
+                )
+        else:
+            self.model = AutoModelForCausalLM.from_pretrained(
+                self.model_id,
+                torch_dtype=torch.bfloat16,
+                config=config,
+                low_cpu_mem_usage=True,
+                trust_remote_code=True,
+            )
+        return self.model

examples/cpu/llm/inference/utils/supported_models.py

@@ -36,6 +36,7 @@
     "maira-2": (AutoModelForCausalLM, AutoProcessor),
     "jamba": (AutoModelForCausalLM, AutoTokenizer),
     "deepseek-v2": (AutoModelForCausalLM, AutoTokenizer),
+    "deepseek-v3": (AutoModelForCausalLM, AutoTokenizer),
     "auto": (AutoModelForCausalLM, AutoTokenizer),
 }
 

intel_extension_for_pytorch/transformers/generation/beam_sample.py

@@ -178,6 +178,7 @@ def _beam_sample(
             "Maira2ForConditionalGeneration",
             "JambaForCausalLM",
             "DeepseekV2ForCausalLM",
+            "DeepseekV3ForCausalLM",
         ]:
             first_token = False
             if hasattr(self.config, "kv_cache_dtype"):

intel_extension_for_pytorch/transformers/generation/beam_search.py

@@ -205,6 +205,7 @@ def _beam_search(
             "Maira2ForConditionalGeneration",
             "JambaForCausalLM",
             "DeepseekV2ForCausalLM",
+            "DeepseekV3ForCausalLM",
         ]:
             first_token = False
             has_position_id = model_inputs.get("position_ids", None) is not None

intel_extension_for_pytorch/transformers/generation/greedy_search.py

@@ -170,6 +170,7 @@ def _greedy_search(
             "Maira2ForConditionalGeneration",
             "JambaForCausalLM",
             "DeepseekV2ForCausalLM",
+            "DeepseekV3ForCausalLM",
         ]:
             first_token = False
             if hasattr(self.config, "kv_cache_dtype"):

intel_extension_for_pytorch/transformers/generation/sample.py

@@ -197,6 +197,7 @@ def _sample(
             "Maira2ForConditionalGeneration",
             "JambaForCausalLM",
             "DeepseekV2ForCausalLM",
+            "DeepseekV3ForCausalLM",
         ]:
             first_token = False
             if hasattr(self.config, "kv_cache_dtype"):

intel_extension_for_pytorch/transformers/models/cpu/modules/decoder.py

@@ -62,6 +62,7 @@ def __init__(self, module, config, tpp=False, woq=False):
             "Maira2ForConditionalGeneration",
             "JambaForCausalLM",
             "DeepseekV2ForCausalLM",
+            "DeepseekV3ForCausalLM",
         ]:
             if not self.distributed:
                 if hasattr(module, "linear_add"):
@@ -143,7 +144,10 @@ def __init__(self, module, config, tpp=False, woq=False):
                     tpp=tpp,
                     woq=woq,
                 )
-            if self.model_backbone == "DeepseekV2ForCausalLM":
+            if self.model_backbone in [
+                "DeepseekV2ForCausalLM",
+                "DeepseekV3ForCausalLM",
+            ]:
                 if hasattr(self.mlp, "experts"):
                     # 0: Default, 1: TPP, 2: DNNL, 3: MKL, 4: WOQ
                     self.moe_linear_type = 0