overload set_swa_pattern for modern bert

Author: huydt-bti

src/llama-hparams.cpp

@@ -8,6 +8,12 @@ void llama_hparams::set_swa_pattern(uint32_t n_pattern) {
     }
 }
 
+void llama_hparams::set_swa_pattern(uint32_t n_pattern, uint32_t remainder) {
+    for (uint32_t il = 0; il < n_layer; ++il) {
+        swa_layers[il] = n_pattern == 0 || (il % n_pattern != remainder);
+    }
+}
+
 bool llama_hparams::is_swa_any() const {
     for (uint32_t il = 0; il < n_layer; ++il) {
         if (swa_layers[il]) {

src/llama-hparams.h

@@ -162,6 +162,18 @@ struct llama_hparams {
     //   etc ...
     void set_swa_pattern(uint32_t n_pattern);
 
+    // Overload that allows specifying which position in the pattern is dense
+    // The remainder parameter specifies which position in the pattern is dense
+    // example: n_pattern = 3, remainder = 2
+    //   il == 0: swa     (0 % 3 = 0, which is not equal to 2)
+    //   il == 1: swa     (1 % 3 = 1, which is not equal to 2) 
+    //   il == 2: dense   (2 % 3 = 2, which equals 2)
+    //   il == 3: swa     (3 % 3 = 0, which is not equal to 2)
+    //   il == 4: swa     (4 % 3 = 1, which is not equal to 2)
+    //   il == 5: dense   (5 % 3 = 2, which equals 2)
+    //   etc ...
+    void set_swa_pattern(uint32_t n_pattern, uint32_t remainder);
+
     // return true if one of the layers is SWA
     bool is_swa_any() const;
 

src/llama-model.cpp

@@ -714,7 +714,7 @@ void llama_model::load_hparams(llama_model_loader & ml) {
                 hparams.n_swa = 128;
 
                 hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
-                hparams.set_swa_pattern(3);
+                hparams.set_swa_pattern(3, 0);
 
                 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);
                 ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn);