llama : update WavTokenizer to non-causal attn

Author: ggerganov

convert_hf_to_gguf.py

@@ -2069,6 +2069,8 @@ def set_gguf_parameters(self):
         self.gguf_writer.add_convnext_embedding_length(self.hparams["convnext"]["n_embd"])
         self.gguf_writer.add_convnext_block_count     (self.hparams["convnext"]["n_layer"])
 
+        self.gguf_writer.add_causal_attention(False)
+
 
 @Model.register("Qwen2MoeForCausalLM")
 class Qwen2MoeModel(Model):

src/llama.cpp

@@ -6393,6 +6393,7 @@ static void llm_load_hparams(
                 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);
                 ml.get_key(LLM_KV_ATTENTION_GROUPNORM_EPS,    hparams.f_norm_group_eps);
                 ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);
+                ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn);
             } break;
         default: (void)0;
     }