Added MOE options - llama.cpp PR 15077

--cpu-moe to keep all MoE weights in the CPU
--n-cpu-moe N to keep the MoE weights of the first N layers in the CPU

ggml-org/llama.cpp#15077

Author: thad0ctor

.gitignore

@@ -3,8 +3,14 @@ llama_cpp_launcher_configs.json
 __pycache__
 .gitignore
 llama.log
-llama.log
 log.txt
 tensor_overrides/Kimi-K2-Instruct-UD-IQ1_S-00001-of-00006_Kimi-K2-Instruct-UD-IQ1_S_kv=q4_0_vv=q4_0_th=24_tb=48_ctx=131072_fa_tensor_params.txt
 tensor_overrides/Kimi-K2-Instruct-UD-IQ1_S-00001-of-00006_Kimi-K2-Instruct-UD-IQ1_S_kv=q4_0_vv=q4_0_th=24_tb=48_ctx=131072_fa_tensor_params_balanced.txt
 tensor_overrides/Kimi-K2-Instruct-UD-IQ1_S-00001-of-00006_Kimi-K2-Instruct-UD-IQ1_S_kv=q4_0_vv=q4_0_th=24_tb=48_ctx=131072_fa_tensor_params_original.txt
+debug_scan.py
+debug_regex.py
+debug_glm_file.py
+/tensor_overrides
+test_fix.py
+test_more_patterns.py
+test_regex_fix.py

config.py

@@ -82,6 +82,8 @@ def generate_default_config_name(self):
             "no_mmap":       False, # Default for --no-mmap flag
             "mlock":         False, # Default for --mlock flag
             "no_kv_offload": False, # Default for --no-kv-offload flag
+            "cpu_moe":       False, # Default for --cpu-moe flag
+            "n_cpu_moe":     "",    # Default for --n-cpu-moe (empty)
 
             # Chat template parameters and custom parameters are deliberately excluded from default name generation
         }
@@ -107,6 +109,8 @@ def generate_default_config_name(self):
             "no_mmap":       self.launcher.no_mmap.get(),   # bool
             "mlock":         self.launcher.mlock.get(),    # bool
             "no_kv_offload": self.launcher.no_kv_offload.get(), # bool
+            "cpu_moe":       self.launcher.cpu_moe.get(),  # bool
+            "n_cpu_moe":     self.launcher.n_cpu_moe.get().strip(),
 
         }
 
@@ -142,6 +146,7 @@ def generate_default_config_name(self):
                          "no_kv_offload": "no-kv-offload",
 
                          "ignore_eos": "no-eos",
+                         "cpu_moe": "cpu-moe",
                       }
                       parts.append(flag_name_map.get(key, key.replace('_', '-'))) # Use mapped name or just key
             # Handle other string parameters
@@ -270,6 +275,9 @@ def current_cfg(self):
             # --- NEW: Add new parameters to config ---
             "ignore_eos":    self.launcher.ignore_eos.get(),
             "n_predict":     self.launcher.n_predict.get(),
+            # --- MoE CPU parameters ---
+            "cpu_moe":       self.launcher.cpu_moe.get(),
+            "n_cpu_moe":     self.launcher.n_cpu_moe.get(),
             # --- CHANGES FOR JSON TEMPLATES / DEFAULT OPTION ---
             # Save the new template source variable
             "template_source": self.launcher.template_source.get(),
@@ -338,6 +346,9 @@ def load_configuration(self):
         # --- NEW: Load new parameters ---
         self.launcher.ignore_eos.set(cfg.get("ignore_eos", False))
         self.launcher.n_predict.set(cfg.get("n_predict", "-1")) # Default -1 for backward compatibility
+        # --- MoE CPU parameters ---
+        self.launcher.cpu_moe.set(cfg.get("cpu_moe", False))
+        self.launcher.n_cpu_moe.set(cfg.get("n_cpu_moe", ""))
         # --- NEW: Load Custom Parameters ---
         # Default to empty list [] for backward compatibility with older configs
         self.launcher.custom_parameters_list = cfg.get("custom_parameters", [])

launch.py

@@ -194,6 +194,11 @@ def build_cmd(self):
 
         # Performance options
         self.add_arg(cmd, "--prio", self.launcher.prio.get(), "0") # Omit if 0 (default)
+        
+        # --- MoE CPU options (only for llama.cpp backend) ---
+        if backend != "ik_llama":
+            self.add_arg(cmd, "--cpu-moe", self.launcher.cpu_moe.get()) # Omit if False (default)
+            self.add_arg(cmd, "--n-cpu-moe", self.launcher.n_cpu_moe.get(), "") # Omit if empty (default)
 
         # --- NEW: Generation options ---
         self.add_arg(cmd, "--ignore-eos", self.launcher.ignore_eos.get()) # Omit if False (default)

llamacpp-server-launcher.py

@@ -277,6 +277,10 @@ def __init__(self, root: tk.Tk):
         # --- New Parameters ---
         self.ignore_eos      = tk.BooleanVar(value=False) # --ignore-eos
         self.n_predict       = tk.StringVar(value="-1")   # --n-predict
+        
+        # --- MoE CPU Parameters ---
+        self.cpu_moe         = tk.BooleanVar(value=False) # --cpu-moe
+        self.n_cpu_moe       = tk.StringVar(value="")     # --n-cpu-moe
 
         # --- Chat Template Selection Variables ---
         # Controls which template source is used: 'default', 'predefined', or 'custom'.
@@ -417,6 +421,9 @@ def __init__(self, root: tk.Tk):
         self.n_predict.trace_add("write", lambda *args: self._update_default_config_name_if_needed())
         # Bind trace to ignore_eos to update default config name if needed
         self.ignore_eos.trace_add("write", lambda *args: self._update_default_config_name_if_needed())
+        # Bind trace to MoE CPU parameters to update default config name if needed
+        self.cpu_moe.trace_add("write", lambda *args: self._update_default_config_name_if_needed())
+        self.n_cpu_moe.trace_add("write", lambda *args: self._update_default_config_name_if_needed())
         # Bind trace to other variables that affect the default config name
         self.cache_type_k.trace_add("write", lambda *args: self._update_default_config_name_if_needed())
         self.threads.trace_add("write", lambda *args: self._update_default_config_name_if_needed())
@@ -1147,6 +1154,21 @@ def _setup_advanced_tab(self, parent):
         self.prio_combo.grid(column=1, row=r, sticky="w", padx=5, pady=3); r += 1
         ttk.Label(inner, text="0=Normal, 1=Medium, 2=High, 3=Realtime (OS dependent)", font=("TkSmallCaptionFont"))\
             .grid(column=2, row=r-1, columnspan=2, sticky="w", padx=5, pady=3); # Re-grid label
+        
+        # --- MoE CPU Settings --- (same row)
+        ttk.Label(inner, text="MoE CPU Settings:")\
+            .grid(column=0, row=r, sticky="w", padx=10, pady=3)
+        moe_frame = ttk.Frame(inner)
+        moe_frame.grid(column=1, row=r, columnspan=3, sticky="w", padx=5, pady=3)
+        
+        self.cpu_moe_check = ttk.Checkbutton(moe_frame, text="Keep all MoE in CPU (--cpu-moe)", variable=self.cpu_moe, state=tk.NORMAL)
+        self.cpu_moe_check.pack(side="left", padx=(0, 10))
+        
+        ttk.Label(moe_frame, text="First N layers in CPU (--n-cpu-moe):")\
+            .pack(side="left", padx=(0, 5))
+        self.n_cpu_moe_entry = ttk.Entry(moe_frame, textvariable=self.n_cpu_moe, width=8, state=tk.NORMAL)
+        self.n_cpu_moe_entry.pack(side="left")
+        r += 1
 
 
         # --- NEW: Generation Settings ---
@@ -1647,7 +1669,8 @@ def _scan_model_dirs(self):
         print("DEBUG: _scan_model_dirs thread started", file=sys.stderr)
         found = {} # {display_name: full_path_obj}
         # Pattern to match multi-part files like model-00001-of-00005.gguf or model-F1.gguf
-        multipart_pattern = re.compile(r"^(.*?)(?:-\d{5}-of-\d{5}|-F\d+)\.gguf$", re.IGNORECASE)
+        # Note: -F[1-9] only matches single-digit F parts to avoid matching precision indicators like F16, F32
+        multipart_pattern = re.compile(r"^(.*?)(?:-\d{5}-of-\d{5}|-F[1-9])\.gguf$", re.IGNORECASE)
         # Pattern to match the FIRST part of a multi-part file (e.g., model-00001-of-00005.gguf or model-F1.gguf)
         first_part_pattern = re.compile(r"^(.*?)-(?:00001-of-\d{5}|F1)\.gguf$", re.IGNORECASE)
 

version

@@ -1 +1 @@
-2025-07-22-0
+2025-08-05-0