fix 4b rope bug

Author: wdl339

convert_hf_to_gguf.py

@@ -7590,8 +7590,8 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
 
 
 
-@ModelBase.register("SmallthinkerForCausalLM")
-class SmallthinkerModel(TextModel):
+@ModelBase.register("SmallThinkerForCausalLM")
+class SmallThinkerModel(TextModel):
     model_arch = gguf.MODEL_ARCH.SMALLTHINKER
 
     def set_gguf_parameters(self):
@@ -7602,10 +7602,8 @@ def set_gguf_parameters(self):
             self.gguf_writer.add_expert_used_count(n_experts_used)
         if (moe_intermediate_size := self.hparams.get("moe_ffn_hidden_size")) is not None:
             self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
+            self.gguf_writer.add_feed_forward_length(moe_intermediate_size)
             logger.info(f"gguf: expert feed forward length = {moe_intermediate_size}")
-        if (shared_expert_intermediate_size := self.hparams.get('shared_expert_intermediate_size')) is not None:
-            self.gguf_writer.add_expert_shared_feed_forward_length(shared_expert_intermediate_size)
-            logger.info(f"gguf: expert shared feed forward length = {shared_expert_intermediate_size}")
         if (self.hparams.get('moe_primary_router_apply_softmax')):
             self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
         else:
@@ -7618,29 +7616,13 @@ def set_gguf_parameters(self):
             self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
             self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
 
-        sliding_window = self.hparams.get("sliding_window")
         sliding_window_layout = self.hparams.get("sliding_window_layout")
-        if sliding_window and sliding_window_layout:
+        if sliding_window_layout:
             for i in sliding_window_layout:
                 if i != 0:
+                    sliding_window = self.hparams.get("sliding_window_size")
                     self.gguf_writer.add_sliding_window(sliding_window)
                     break
-        elif sliding_window:
-            self.gguf_writer.add_sliding_window(sliding_window)
-
-        intermediate_size = self.hparams.get("ffn_hidden_size")
-        moe_intermediate_size = self.hparams.get("moe_ffn_hidden_size")
-        moe_layer_layout = self.hparams.get("moe_layer_layout")
-        ffn_layout = []
-        for i, layout in enumerate(moe_layer_layout):
-            if layout == 0:
-                ffn_layout.append(intermediate_size)
-            elif layout == 1:
-                ffn_layout.append(moe_intermediate_size)
-            else:
-                raise ValueError(f"Unknown moe layer layout: {layout}")
-        self.gguf_writer.add_feed_forward_length(ffn_layout)
-        # def add_feed_forward_length(self, length: int | Sequence[int]) -> None:
 
     _experts: list[dict[str, Tensor]] | None = None
 

src/llama-model.cpp

@@ -5200,7 +5200,6 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
 
                     for (int i = 0; i < n_layer; ++i) {
                         auto &        layer    = layers[i];
-                        const int64_t n_ff_cur = hparams.n_ff_arr[i];
 
                         layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0);
 
@@ -5220,16 +5219,10 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
 
                         // MoE branch
                         const int64_t n_ff_exp = hparams.n_ff_exp;
-                        if (n_ff_exp == n_ff_cur) {
-                            layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0);
-                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0);
-                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, 0);
-                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0);
-                        } else {
-                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff_cur }, 0);
-                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff_cur,n_embd }, 0);
-                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff_cur }, 0);
-                        }
+                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0);
+                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0);
+                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, 0);
+                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0);
                     }
                 } break;
             default:
@@ -17134,7 +17127,7 @@ struct llm_build_smallthinker : public llm_graph_context{
                 Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
                 Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
 
-                if(il % hparams.n_no_rope_layer_step) {
+                if(hparams.n_no_rope_layer_step == n_layer || il % hparams.n_no_rope_layer_step != 0) {
                     Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
                                      ext_factor, attn_factor, beta_fast, beta_slow);