@@ -100,8 +100,7 @@ int ggml_cuda_get_device() {
100100static cudaError_t ggml_cuda_device_malloc (void ** ptr, size_t size, int device) {
101101 ggml_cuda_set_device (device);
102102 cudaError_t err;
103- if (getenv (" GGML_CUDA_ENABLE_UNIFIED_MEMORY" nullptr )
104- {
103+ if (getenv (" GGML_CUDA_ENABLE_UNIFIED_MEMORY" nullptr ) {
105104 err = cudaMallocManaged (ptr, size);
106105#if defined(GGML_USE_HIP)
107106 if (err == hipSuccess) {
@@ -119,9 +118,7 @@ static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device)
119118 err = cudaMalloc (ptr, size);
120119 }
121120#endif // defined(GGML_USE_HIP)
122- }
123- else
124- {
121+ } else {
125122 err = cudaMalloc (ptr, size);
126123 }
127124 return err;
@@ -1946,16 +1943,14 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
19461943 && ggml_nbytes (src0) != ggml_backend_buffer_get_alloc_size (src0->buffer , src0) && src0->view_src ;
19471944
19481945 bool use_mul_mat_vec = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16)
1949- && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32
1950- && src0->ne [0 ] % 2 == 0 && src1->ne [1 ] == 1 ;
1946+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
19511947 bool use_mul_mat_vec_q = ggml_is_quantized (src0->type ) && !bad_padding_clear
19521948 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32
19531949 && src1->ne [1 ] <= MMVQ_MAX_BATCH_SIZE;
19541950 bool use_mul_mat_q = ggml_is_quantized (src0->type ) && !bad_padding_clear
19551951 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
19561952
1957- bool any_gpus_with_slow_fp16 = false ;
1958- bool any_gpus_without_fp16_mma = false ;
1953+ bool any_gpus_with_slow_fp16 = false ;
19591954
19601955 if (split) {
19611956 ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer ->buft ->context ;
@@ -1966,16 +1961,16 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
19661961 continue ;
19671962 }
19681963
1969- const int cc = ggml_cuda_info ().devices [id].cc ;
1970- use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq (src0->type , cc, src1->ne [1 ]);
1971- any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || ! fast_fp16_hardware_available (cc );
1972- any_gpus_without_fp16_mma = any_gpus_without_fp16_mma || !fp16_mma_hardware_available (cc);
1964+ const int cc = ggml_cuda_info ().devices [id].cc ;
1965+ use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq (src0->type , cc, src1->ne [1 ]);
1966+ use_mul_mat_vec = use_mul_mat_vec && ggml_cuda_should_use_mmv (src0-> type , cc, src0-> ne , src1-> ne [ 1 ] );
1967+ any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available (cc);
19731968 }
19741969 } else {
1975- const int cc = ggml_cuda_info ().devices [ctx.device ].cc ;
1976- use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq (src0->type , cc, src1->ne [1 ]);
1977- any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || ! fast_fp16_hardware_available (cc );
1978- any_gpus_without_fp16_mma = any_gpus_without_fp16_mma || !fp16_mma_hardware_available (cc);
1970+ const int cc = ggml_cuda_info ().devices [ctx.device ].cc ;
1971+ use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq (src0->type , cc, src1->ne [1 ]);
1972+ use_mul_mat_vec = use_mul_mat_vec && ggml_cuda_should_use_mmv (src0-> type , cc, src0-> ne , src1-> ne [ 1 ] );
1973+ any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available (cc);
19791974 }
19801975
19811976 // debug helpers
@@ -1986,7 +1981,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
19861981 // printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name);
19871982 // printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name);
19881983
1989- if (!split && use_mul_mat_vec && (src0-> ne [ 1 ] <= MMV_MAX_ROWS || any_gpus_without_fp16_mma) ) {
1984+ if (!split && use_mul_mat_vec) {
19901985 // the custom F16 vector kernel can be used over batched cuBLAS GEMM
19911986 // but this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention)
19921987 ggml_cuda_mul_mat_vec (ctx, src0, src1, nullptr , dst);
0 commit comments