Set is_recurrent from h_head_kv

Author: tdakhran

convert_hf_to_gguf.py

@@ -6970,13 +6970,12 @@ def _add_feed_forward_length(self):
 
 
     def set_gguf_parameters(self):
-        # set only for attention layers before calling super().set_gguf_parameters()
+        # set num_key_value_heads only for attention layers
         self.hparams["num_key_value_heads"] = [(self.hparams["num_key_value_heads"] if x in self.hparams["full_attn_idxs"] else 0) for x in range(self.block_count)]
 
         super().set_gguf_parameters()
         self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
         self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])
-        self.gguf_writer.add_is_recurrent_layer([x not in self.hparams["full_attn_idxs"] for x in range(self.block_count)])
         self.gguf_writer.add_layer_norm_rms_eps(self.hparams["norm_eps"])
         self._add_feed_forward_length()
 

gguf-py/gguf/constants.py

@@ -122,7 +122,6 @@ class LLM:
         ALTUP_ACTIVE_IDX                  = "{arch}.altup.active_idx"
         ALTUP_NUM_INPUTS                  = "{arch}.altup.num_inputs"
         EMBD_LENGTH_PER_LAYER_INP         = "{arch}.embedding_length_per_layer_input"
-        IS_RECURRENT_LAYER                = "{arch}.is_recurrent_layer"
 
     class Attention:
         HEAD_COUNT                   = "{arch}.attention.head_count"
@@ -2335,6 +2334,7 @@ class MODEL_TENSOR(IntEnum):
         MODEL_TENSOR.FFN_GATE,
         MODEL_TENSOR.FFN_DOWN,
         MODEL_TENSOR.FFN_UP,
+    ],
     MODEL_ARCH.LFM2: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.TOKEN_EMBD_NORM,

gguf-py/gguf/gguf_writer.py

@@ -651,9 +651,6 @@ def add_convnext_block_count(self, length: int) -> None:
     def add_shortconv_l_cache(self, length: int) -> None:
         self.add_uint32(Keys.ShortConv.L_CACHE.format(arch=self.arch), length)
 
-    def add_is_recurrent_layer(self, value: Sequence[bool]) -> None:
-        self.add_array(Keys.LLM.IS_RECURRENT_LAYER.format(arch=self.arch), value)
-
     def add_block_count(self, length: int) -> None:
         self.add_uint32(Keys.LLM.BLOCK_COUNT.format(arch=self.arch), length)
 

src/llama-arch.cpp

@@ -191,8 +191,6 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
 
     { LLM_KV_SHORTCONV_L_CACHE, "%s.shortconv.l_cache" },
 
-    { LLM_KV_IS_RECURRENT_LAYER, "%s.is_recurrent_layer" },
-
     { LLM_KV_TOKENIZER_MODEL,                "tokenizer.ggml.model"                    },
     { LLM_KV_TOKENIZER_PRE,                  "tokenizer.ggml.pre"                      },
     { LLM_KV_TOKENIZER_LIST,                 "tokenizer.ggml.tokens"                   },

src/llama-arch.h

@@ -230,8 +230,6 @@ enum llm_kv {
 
     LLM_KV_SHORTCONV_L_CACHE,
 
-    LLM_KV_IS_RECURRENT_LAYER,
-
     // deprecated:
     LLM_KV_TOKENIZER_PREFIX_ID,
     LLM_KV_TOKENIZER_SUFFIX_ID,

src/llama-model-loader.cpp

@@ -305,11 +305,10 @@ namespace GGUFMeta {
             case GGUF_TYPE_UINT32:
             case GGUF_TYPE_INT32:   GGML_ASSERT((std::is_same<T,     int32_t>::value) ||
                                                 (std::is_same<T,    uint32_t>::value)); break;
-            case GGUF_TYPE_BOOL:    GGML_ASSERT((std::is_same<T,        bool>::value)); break;
             case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T,       float>::value)); break;
             case GGUF_TYPE_STRING:  GGML_ASSERT((std::is_same<T, std::string>::value)); break;
             default:
-                throw std::runtime_error(format("%s is not a string/float32/uint32/int32/bool array", key.c_str()));
+                throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));
         }
 
         if constexpr (std::is_same<T, std::string>::value) {
@@ -347,11 +346,10 @@ namespace GGUFMeta {
             case GGUF_TYPE_UINT32:
             case GGUF_TYPE_INT32:   GGML_ASSERT((std::is_same<T,     int32_t>::value) ||
                                                 (std::is_same<T,    uint32_t>::value)); break;
-            case GGUF_TYPE_BOOL:    GGML_ASSERT((std::is_same<T,        bool>::value)); break;
             case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T,       float>::value)); break;
             case GGUF_TYPE_STRING:  GGML_ASSERT((std::is_same<T, std::string>::value)); break;
             default:
-                throw std::runtime_error(format("%s is not a string/float32/uint32/int32/bool array", key.c_str()));
+                throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));
         }
 
         if (arr_info.length > N_MAX) {
@@ -466,7 +464,6 @@ namespace GGUFMeta {
     // TODO: this is not very clever - figure out something better
     template bool llama_model_loader::get_key_or_arr<std::array<int, 4>>(enum llm_kv kid, std::array<int, 4> & result, uint32_t n, bool required);
     template bool llama_model_loader::get_key_or_arr<std::array<uint32_t, 512>>(enum llm_kv kid, std::array<uint32_t, 512> & result, uint32_t n, bool required);
-    template bool llama_model_loader::get_key_or_arr<std::array<bool, 512>>(enum llm_kv kid, std::array<bool, 512> & result, uint32_t n, bool required);
 
 llama_model_loader::llama_model_loader(
         const std::string & fname,

src/llama-model.cpp

@@ -498,7 +498,6 @@ void llama_model::load_hparams(llama_model_loader & ml) {
     hparams.n_head_kv_arr = hparams.n_head_arr;
 
     ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, hparams.n_layer, false);
-    ml.get_key_or_arr(LLM_KV_IS_RECURRENT_LAYER, hparams.recurrent_layer_arr, hparams.n_layer, false);
 
     bool rope_finetuned = false;
     ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);
@@ -1630,6 +1629,9 @@ void llama_model::load_hparams(llama_model_loader & ml) {
             {
                 ml.get_key(LLM_KV_SHORTCONV_L_CACHE,           hparams.n_shortconv_l_cache);
                 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+                for (uint32_t il = 0; il < hparams.n_layer; ++il) {
+                    hparams.recurrent_layer_arr[il] = hparams.n_head_kv(il) == 0;
+                }
                 switch (hparams.n_embd) {
                     case 1024: type = LLM_TYPE_350M; break;
                     case 1536: type = LLM_TYPE_700M; break;