config.yaml

healthCheckTimeout: 600
logLevel: info
startPort: 10001

macros:
  "llama-server-base": |
    /usr/local/bin/llama-server
    --host 0.0.0.0
    --port ${PORT}
    --no-warmup
    --threads -1
    --flash-attn on
    --batch-size 512
    --ubatch-size 128
    --parallel 1

  # Offload profiles (flag-only)
  "gpu-max": |
    --n-gpu-layers -1
  "gpu-none": |
    --n-gpu-layers 0

  # Context/KV presets (flag-only)
  "ctx-16k": |
    --ctx-size 16384  -ctk q4_0 -ctv q4_0
  "ctx-32k": |
    --ctx-size 32768  -ctk q4_0 -ctv q4_0
  "ctx-64k": |
    --ctx-size 65536  -ctk q4_0 -ctv q4_0
  "ctx-128k": |
    --ctx-size 131072 -ctk q4_0 -ctv q4_0
  "ctx-262k": |
    --ctx-size 262144 -ctk q4_0 -ctv q4_0

  # Sampling presets (flag-only)
  "samp-instruct": |
    --temp 0.7 --min-p 0.0 --top-p 0.8  --top-k 20
  "samp-thinking": |
    --temp 0.6 --min-p 0.0 --top-p 0.95 --top-k 20
  "samp-creative": |
    --temp 1.0 --min-p 0.0 --top-p 1.0  --top-k 0

  # Selective CPU ops for very large models (flag-only)
  "ffn-exp-cpu": |
    -ot ".ffn_.*_exps.=CPU"

models:
  # ~30B class — comfortable on 32 GB with 64k ctx. Use parallel ~3 for headroom.
  "qwen3-30b-instruct-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-max}
      ${ctx-64k}
      ${samp-instruct}
      --model /models/Qwen3-30B-A3B-Instruct-2507-GGUF/Qwen3-30B-A3B-Instruct-2507-UD-Q4_K_XL.gguf
    ttl: 300

  "qwen3-30b-thinking-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-max}
      ${ctx-64k}
      ${samp-thinking}
      --model /models/Qwen3-30B-A3B-Thinking-2507-GGUF/Qwen3-30B-A3B-Thinking-2507-UD-Q4_K_XL.gguf
    ttl: 300

  "qwen3-30b-coder-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-max}
      ${ctx-64k}
      ${samp-instruct}
      --repeat-penalty 1.05
      --model /models/Qwen3-Coder-30B-A3B-Instruct-GGUF/Qwen3-Coder-30B-A3B-Instruct-UD-Q4_K_XL.gguf
    ttl: 86400

  # >= 100B class — keep GPU layers off to avoid surprise OOM; use smaller ctx on CPU.
  "qwen3-235b-instruct-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-none}
      ${ctx-32k}
      ${samp-instruct}
      ${ffn-exp-cpu}
      --model /models/Qwen3-235B-A22B-Instruct-2507-GGUF/UD-Q4_K_XL/Qwen3-235B-A22B-Instruct-2507-UD-Q4_K_XL-00001-of-00003.gguf
    ttl: 300

  "qwen3-235b-thinking-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-none}
      ${ctx-32k}
      ${samp-thinking}
      ${ffn-exp-cpu}
      --model /models/Qwen3-235B-A22B-Thinking-2507-GGUF/UD-Q4_K_XL/Qwen3-235B-A22B-Thinking-2507-UD-Q4_K_XL-00001-of-00003.gguf
    ttl: 300

  "qwen3-480b-coder-q2":
    cmd: |
      ${llama-server-base}
      ${gpu-none}
      ${ctx-32k}
      ${samp-instruct}
      ${ffn-exp-cpu}
      --repeat-penalty 1.05
      --model /models/Qwen3-Coder-480B-A35B-Instruct-GGUF/UD-Q2_K_XL/Qwen3-Coder-480B-A35B-Instruct-UD-Q2_K_XL-00001-of-00004.gguf
    ttl: 300

  # GLM family
  "glm-4-32b-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-max}
      ${ctx-128k}
      --jinja
      --model /models/GLM-4-32B-0414-GGUF/GLM-4-32B-0414-UD-Q4_K_XL.gguf
    ttl: 300

  "glm-4.5-air-106b-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-none}
      ${ctx-32k}
      ${ffn-exp-cpu}
      --jinja
      --model /models/GLM-4.5-Air-GGUF/UD-Q4_K_XL/GLM-4.5-Air-UD-Q4_K_XL-00001-of-00002.gguf
    ttl: 300

  "glm-4.5-355b-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-none}
      ${ctx-32k}
      ${ffn-exp-cpu}
      --jinja
      --model /models/GLM-4.5-GGUF/UD-Q4_K_XL/GLM-4.5-UD-Q4_K_XL-00001-of-00005.gguf
    ttl: 300

  # GPT-OSS examples
  "gpt-oss-20b":
    cmd: |
      ${llama-server-base}
      ${gpu-max}
      ${ctx-16k}
      ${samp-creative}
      --jinja
      --model /models/gpt-oss-20b-GGUF/gpt-oss-20b-UD-Q8_K_XL.gguf
    ttl: 300

  "gpt-oss-120b":
    cmd: |
      ${llama-server-base}
      ${gpu-none}
      ${ctx-128k}
      ${ffn-exp-cpu}
      --jinja
      --model /models/gpt-oss-120b-GGUF/UD-Q8_K_XL/gpt-oss-120b-UD-Q8_K_XL-00001-of-00002.gguf
    ttl: 300

  # DeepSeek V3.1 (very large)
  "deepseek-v31-671b-tq1":
    cmd: |
      ${llama-server-base}
      ${gpu-none}
      ${ctx-16k}
      ${ffn-exp-cpu}
      --jinja
      --cache-type-k q4_0
      --temp 0.6 --top-p 0.95 --min-p 0.01
      --model /models/DeepSeek-V3.1-GGUF/DeepSeek-V3.1-UD-TQ1_0.gguf
    ttl: 300

  "deepseek-v31-671b-q2":
    cmd: |
      ${llama-server-base}
      ${gpu-none}
      ${ctx-16k}
      ${ffn-exp-cpu}
      --jinja
      --cache-type-k q4_0
      --temp 0.6 --top-p 0.95 --min-p 0.01
      --model /models/DeepSeek-V3.1-GGUF/UD-Q2_K_XL/DeepSeek-V3.1-UD-Q2_K_XL-00001-of-00006.gguf
    ttl: 300

  # Grok 2 (example)
  "grok-2-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-none}
      ${ctx-16k}
      --jinja
      --flash-attn off
      --temp 1.0 --top-p 0.95 --min-p 0.01
      --model /models/grok-2-GGUF/UD-Q4_K_XL/grok-2-UD-Q4_K_XL-00001-of-00004.gguf
    ttl: 300

  "magistral-small-2509-q4":
    cmd: |
      ${llama-server-base}
      ${gpu-max}
      ${ctx-128k}
      --jinja
      --special
      --temp 0.7 --top-k -1 --top-p 0.95
      --model /models/Magistral-Small-2509-GGUF/Magistral-Small-2509-UD-Q4_K_XL.gguf
    ttl: 300

groups:
  "main-swap-group":
    swap: true
    exclusive: true
    members:
      - "qwen3-30b-instruct-q4"
      - "qwen3-30b-thinking-q4"
      - "qwen3-30b-coder-q4"
      - "qwen3-235b-instruct-q4"
      - "qwen3-235b-thinking-q4"
      - "qwen3-480b-coder-q2"
      - "glm-4-32b-q4"
      - "glm-4.5-air-106b-q4"
      - "glm-4.5-355b-q4"
      - "gpt-oss-20b"
      - "gpt-oss-120b"
      - "deepseek-v31-671b-tq1"
      - "deepseek-v31-671b-q2"
      - "grok-2-q4"
      - "magistral-small-2509-q4"

hooks:
  on_startup:
    preload:
      - "qwen3-30b-coder-q4"