Update

Author: sergiopaniego

scripts/generate_tiny_models.py

@@ -239,8 +239,8 @@ def init_weights_tiny_model(model):
     num_key_value_heads=2,
     intermediate_size=32,
     layers_block_type=["mamba", "attention"],  # 2 layers: one Mamba + one Attention
-    mamba_num_heads=4,
-    mamba_head_dim=8,
+    mamba_num_heads=8,
+    mamba_head_dim=4,
     mamba_n_groups=1,
     ssm_state_size=16,
     mamba_d_conv=4,