Fix predictor bug (#8947)

* fix bug

* update

Author: yuanlehome

llm/predict/predictor.py

@@ -773,12 +773,8 @@ def __init__(self, config: PredictorArgument, tokenizer: PretrainedTokenizer):
 
         self.dtype = config.dtype or self.model_config.dtype
 
-        try:
-            self.rope_theta = self.model_config.rope_theta
-            self.rope_scaling = self.model_config.rope_scaling
-        except:
-            self.rope_theta = 10000.0
-            self.rope_scaling = None
+        self.rope_theta = self.model_config.get("rope_theta", 10000.0)
+        self.rope_scaling = self.model_config.get("rope_scaling", None)
 
         self.pre_cache_length = 0
 

paddlenlp/experimental/transformers/llama/modeling.py

@@ -980,7 +980,6 @@ def set_state_dict(self, state_dict):
                 act_scale_map_dict = scale_map_dict["act_scale"]
                 weight_scale_map_dict = scale_map_dict["weight_scale"]
                 cache_scale_map_dict = scale_map_dict["cachekv_scale"]
-                # TODO(RichardWooSJTU): support multi-cards
 
                 act_scale_json_path = os.path.join(self.quant_model_path, "act_scales.json")
                 weight_scale_json_path = os.path.join(self.quant_model_path, "weight_scales.json")
@@ -1008,7 +1007,7 @@ def set_state_dict(self, state_dict):
                     cache_scale_json_path = os.path.join(self.quant_model_path, "cachekv_scales.json")
                     if self.config.tensor_parallel_degree > 1 and not self.config.single_card_ptq:
                         cache_scale_json_path = os.path.join(
-                            self.quant_model_path, f"cachekv_act_scales_{self.config.tensor_parallel_rank}.json"
+                            self.quant_model_path, f"cachekv_scales_{self.config.tensor_parallel_rank}.json"
                         )
                     cache_scales_loader = CacheScaleLoader(
                         cache_scale_json_path,