[INFER] Fix tune_cublaslt_int8_gemm.py and remove dist_config (#9520)

Author: yuanlehome

csrc/utils/tune_cublaslt_int8_gemm.py

@@ -23,7 +23,7 @@
 n1 = [6144, 4096, 28672, 4096]
 
 # llama3.1-405b mp=8
-k2 = [16384, 16384, 16384, 6656]
+k2 = [16384, 2048, 16384, 6656]
 n2 = [2560, 16384, 13312, 16384]
 
 # qwen2-1.5b
@@ -43,5 +43,5 @@
 
 # shape 计算公式
 # [qkv, out_linear, ffn1, ffn2]
-# k = [hidden_size, hidden_size, hidden_size, intermediate_size//mp_size]
+# k = [hidden_size, hidden_size//mp_size, hidden_size, intermediate_size//mp_size]
 # n = [((num_attention_heads//mp_size)+2*(num_key_value_heads//mp_size))*(hidden_size//num_attention_heads), hidden_size, 2*(intermediate_size//mp_size), hidden_size]

llm/predict/predictor.py

@@ -673,18 +673,6 @@ def _create_predictor(self, predictor_args: PredictorArgument):
             config.enable_use_gpu(100, device_id)
         config.enable_new_executor()
 
-        if self.tensor_parallel_degree > 1:
-            trainer_endpoints = fleet.worker_endpoints()
-            current_endpoint = trainer_endpoints[self.tensor_parallel_rank]
-
-            dist_config = config.dist_config()
-            dist_config.set_ranks(self.tensor_parallel_degree, self.tensor_parallel_rank)
-            dist_config.set_endpoints(trainer_endpoints, current_endpoint)
-            dist_config.enable_dist_model(True)
-
-            dist_config.set_comm_init_config(os.path.join(predictor_args.model_name_or_path, "rank_mapping.csv"))
-            config.set_dist_config(dist_config)
-
         predictor = paddle.inference.create_predictor(config)
         return predictor
 
@@ -1178,18 +1166,6 @@ def _create_predictor(self, predictor_args: PredictorArgument):
             pass_builder = config.pass_builder()
             passes.addPasses(pass_builder, self.model_config.model_type, self.model_config.quant_type)
 
-        if self.tensor_parallel_degree > 1:
-            trainer_endpoints = fleet.worker_endpoints()
-            current_endpoint = trainer_endpoints[self.tensor_parallel_rank]
-
-            dist_config = config.dist_config()
-            dist_config.set_ranks(self.tensor_parallel_degree, self.tensor_parallel_rank)
-            dist_config.set_endpoints(trainer_endpoints, current_endpoint)
-            dist_config.enable_dist_model(True)
-
-            dist_config.set_comm_init_config(os.path.join(predictor_args.model_name_or_path, "rank_mapping.csv"))
-            config.set_dist_config(dist_config)
-
         self.predictor = paddle.inference.create_predictor(config)
 
     def predict(self, input_texts: list[str], return_tokens=False):