Fix cuda memory leak (#655)

* fix mem leak

Author: FrostML

paddlenlp/ops/faster_transformer/src/fusion_decoding_op.cu

@@ -340,9 +340,11 @@ std::vector<paddle::Tensor> DecodingCUDAForward(
   cublasCreate(&cublas_handle_);
   cublasSetStream(cublas_handle_, stream);
 
+  std::vector<paddle::Tensor> ret;
+
   switch (input.type()) {
     case paddle::DataType::FLOAT16: {
-      return decoding_kernel<paddle::DataType::FLOAT16>(
+      ret = decoding_kernel<paddle::DataType::FLOAT16>(
           input,
           mem_seq_len,
           word_embedding,
@@ -393,9 +395,10 @@ std::vector<paddle::Tensor> DecodingCUDAForward(
           beam_search_diversity_rate,
           cublas_handle_,
           stream);
+      break;
     }
     case paddle::DataType::FLOAT32: {
-      return decoding_kernel<paddle::DataType::FLOAT32>(
+      ret = decoding_kernel<paddle::DataType::FLOAT32>(
           input,
           mem_seq_len,
           word_embedding,
@@ -446,11 +449,16 @@ std::vector<paddle::Tensor> DecodingCUDAForward(
           beam_search_diversity_rate,
           cublas_handle_,
           stream);
+      break;
     }
     default: {
       PD_THROW(
           "NOT supported data type. "
           "Only float16 and float32 are supported. ");
+      break;
     }
   }
+
+  cublasDestroy(cublas_handle_);
+  return ret;
 }

paddlenlp/ops/faster_transformer/src/fusion_gpt_op.cu

@@ -190,75 +190,80 @@ std::vector<paddle::Tensor> GPT2CUDAForward(
   cublasCreate(&cublas_handle_);
   cublasSetStream(cublas_handle_, stream);
 
+  std::vector<paddle::Tensor> ret;
+
   if (use_fp16) {
-    return gpt2_kernel<paddle::DataType::FLOAT16>(input,
-                                                  word_embedding,
-                                                  self_ln_weight,
-                                                  self_ln_bias,
-                                                  self_q_weight,
-                                                  self_q_bias,
-                                                  self_k_weight,
-                                                  self_k_bias,
-                                                  self_v_weight,
-                                                  self_v_bias,
-                                                  self_out_weight,
-                                                  self_out_bias,
-                                                  ffn_ln_weight,
-                                                  ffn_ln_bias,
-                                                  ffn_inter_weight,
-                                                  ffn_inter_bias,
-                                                  ffn_out_weight,
-                                                  ffn_out_bias,
-                                                  decoder_ln_weight,
-                                                  decoder_ln_bias,
-                                                  positional_embedding_weight,
-                                                  emb_weight,
-                                                  output_ids,
-                                                  topk,
-                                                  topp,
-                                                  max_len,
-                                                  n_head,
-                                                  size_per_head,
-                                                  num_layer,
-                                                  bos_id,
-                                                  eos_id,
-                                                  temperature,
-                                                  cublas_handle_,
-                                                  stream);
+    ret = gpt2_kernel<paddle::DataType::FLOAT16>(input,
+                                                 word_embedding,
+                                                 self_ln_weight,
+                                                 self_ln_bias,
+                                                 self_q_weight,
+                                                 self_q_bias,
+                                                 self_k_weight,
+                                                 self_k_bias,
+                                                 self_v_weight,
+                                                 self_v_bias,
+                                                 self_out_weight,
+                                                 self_out_bias,
+                                                 ffn_ln_weight,
+                                                 ffn_ln_bias,
+                                                 ffn_inter_weight,
+                                                 ffn_inter_bias,
+                                                 ffn_out_weight,
+                                                 ffn_out_bias,
+                                                 decoder_ln_weight,
+                                                 decoder_ln_bias,
+                                                 positional_embedding_weight,
+                                                 emb_weight,
+                                                 output_ids,
+                                                 topk,
+                                                 topp,
+                                                 max_len,
+                                                 n_head,
+                                                 size_per_head,
+                                                 num_layer,
+                                                 bos_id,
+                                                 eos_id,
+                                                 temperature,
+                                                 cublas_handle_,
+                                                 stream);
   } else {
-    return gpt2_kernel<paddle::DataType::FLOAT32>(input,
-                                                  word_embedding,
-                                                  self_ln_weight,
-                                                  self_ln_bias,
-                                                  self_q_weight,
-                                                  self_q_bias,
-                                                  self_k_weight,
-                                                  self_k_bias,
-                                                  self_v_weight,
-                                                  self_v_bias,
-                                                  self_out_weight,
-                                                  self_out_bias,
-                                                  ffn_ln_weight,
-                                                  ffn_ln_bias,
-                                                  ffn_inter_weight,
-                                                  ffn_inter_bias,
-                                                  ffn_out_weight,
-                                                  ffn_out_bias,
-                                                  decoder_ln_weight,
-                                                  decoder_ln_bias,
-                                                  positional_embedding_weight,
-                                                  emb_weight,
-                                                  output_ids,
-                                                  topk,
-                                                  topp,
-                                                  max_len,
-                                                  n_head,
-                                                  size_per_head,
-                                                  num_layer,
-                                                  bos_id,
-                                                  eos_id,
-                                                  temperature,
-                                                  cublas_handle_,
-                                                  stream);
+    ret = gpt2_kernel<paddle::DataType::FLOAT32>(input,
+                                                 word_embedding,
+                                                 self_ln_weight,
+                                                 self_ln_bias,
+                                                 self_q_weight,
+                                                 self_q_bias,
+                                                 self_k_weight,
+                                                 self_k_bias,
+                                                 self_v_weight,
+                                                 self_v_bias,
+                                                 self_out_weight,
+                                                 self_out_bias,
+                                                 ffn_ln_weight,
+                                                 ffn_ln_bias,
+                                                 ffn_inter_weight,
+                                                 ffn_inter_bias,
+                                                 ffn_out_weight,
+                                                 ffn_out_bias,
+                                                 decoder_ln_weight,
+                                                 decoder_ln_bias,
+                                                 positional_embedding_weight,
+                                                 emb_weight,
+                                                 output_ids,
+                                                 topk,
+                                                 topp,
+                                                 max_len,
+                                                 n_head,
+                                                 size_per_head,
+                                                 num_layer,
+                                                 bos_id,
+                                                 eos_id,
+                                                 temperature,
+                                                 cublas_handle_,
+                                                 stream);
   }
+
+  cublasDestroy(cublas_handle_);
+  return ret;
 }