ntk alpha freq_base

Author: ngxson

src/llama-model.cpp

@@ -1511,6 +1511,11 @@ void llama_model::load_hparams(llama_model_loader & ml) {
                 ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);
                 ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
 
+                // TODO: read from gguf
+                float n_dim = hparams.n_embd_head_k;
+                float alpha = 1000.0f; // NTK-Aware
+                hparams.rope_freq_base_train = 10000.0f * std::powf(alpha, n_dim / (n_dim - 2.0f));
+
                 switch (hparams.n_layer) {
                     case 32: type = LLM_TYPE_A13B; break;
                     default: type = LLM_TYPE_UNKNOWN;