fix(inference): scale llama-server ready timeout for large GGUF

- Derive wait from model file size (2m + ~1m/GiB, max 45m); CSGHUB_LITE_LLAMA_READY_TIMEOUT override
- SSE/CLI heartbeat while loading (seconds elapsed); larger llama log tail on errors

Made-with: Cursor

Author: ganisback

internal/cli/client.go

@@ -189,6 +189,9 @@ func preloadModel(serverURL, modelID string) error {
 			if lr.Total > 0 {
 				pct := lr.Current * 100 / lr.Total
 				fmt.Fprintf(os.Stderr, "\r\033[K  %s (%d/%d) %d%%", lr.Step, lr.Current, lr.Total, pct)
+			} else if lr.Current > 0 {
+				// Heartbeat (e.g. seconds waiting for llama-server while loading a large GGUF).
+				fmt.Fprintf(os.Stderr, "\r\033[K  %s (%ds)", lr.Step, lr.Current)
 			} else if lr.Step != lastStep {
 				if lastStep != "" {
 					fmt.Fprintf(os.Stderr, "\n")

internal/inference/engine.go

@@ -52,14 +52,14 @@ func LoadEngineWithProgress(modelDir string, lm *model.LocalModel, progress Conv
 
 	switch format {
 	case model.FormatGGUF:
-		return newLlamaEngine(modelFile, lm.FullName(), verbose, mmproj)
+		return newLlamaEngine(modelFile, lm.FullName(), verbose, progress, mmproj)
 
 	case model.FormatSafeTensors:
 		ggufPath, err := convertSafeTensors(modelDir, progress)
 		if err != nil {
 			return nil, fmt.Errorf("auto-converting SafeTensors to GGUF: %w", err)
 		}
-		eng, err := newLlamaEngine(ggufPath, lm.FullName(), verbose, mmproj)
+		eng, err := newLlamaEngine(ggufPath, lm.FullName(), verbose, progress, mmproj)
 		if err != nil {
 			log.Printf("removing invalid converted GGUF: %s", ggufPath)
 			os.Remove(ggufPath)

internal/inference/llama.go

@@ -14,6 +14,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -99,7 +100,34 @@ func findFreePort() (int, error) {
 	return l.Addr().(*net.TCPAddr).Port, nil
 }
 
-func newLlamaEngine(modelPath, modelName string, verbose bool, mmproj ...string) (*llamaEngine, error) {
+// llamaReadyTimeout returns how long to wait for llama-server /health after start.
+// Large GGUF files can take many minutes to mmap / load to GPU.
+func llamaReadyTimeout(modelPath string) time.Duration {
+	if v := strings.TrimSpace(os.Getenv("CSGHUB_LITE_LLAMA_READY_TIMEOUT")); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			return d
+		}
+		if sec, err := strconv.Atoi(v); err == nil && sec > 0 {
+			return time.Duration(sec) * time.Second
+		}
+	}
+	fi, err := os.Stat(modelPath)
+	if err != nil {
+		return 20 * time.Minute
+	}
+	gb := float64(fi.Size()) / (1024 * 1024 * 1024)
+	// 2 min base + ~1 min per GiB (F16 9B is ~17GiB on disk → ~19 min).
+	sec := int(120 + gb*60)
+	if sec < 120 {
+		sec = 120
+	}
+	if sec > 45*60 {
+		sec = 45 * 60
+	}
+	return time.Duration(sec) * time.Second
+}
+
+func newLlamaEngine(modelPath, modelName string, verbose bool, progress ConvertProgressFunc, mmproj ...string) (*llamaEngine, error) {
 	binary := findLlamaBinary()
 	if binary == "" {
 		return nil, fmt.Errorf("llama-server not found in PATH.\n" +
@@ -141,7 +169,8 @@ func newLlamaEngine(modelPath, modelName string, verbose bool, mmproj ...string)
 		engine.cmd.Stdout = os.Stderr
 		engine.cmd.Stderr = os.Stderr
 	} else {
-		w := newCappedWriter(8192)
+		// Large models print long tensor/KV lists; keep more tail for error diagnosis.
+		w := newCappedWriter(64 * 1024)
 		engine.cmd.Stdout = w
 		engine.cmd.Stderr = w
 		engine.logBuf = w
@@ -164,15 +193,19 @@ func newLlamaEngine(modelPath, modelName string, verbose bool, mmproj ...string)
 		return nil, fmt.Errorf("starting llama-server: %w", err)
 	}
 
-	if err := engine.waitForReady(30 * time.Second); err != nil {
+	readyTimeout := llamaReadyTimeout(modelPath)
+	if progress != nil {
+		progress("Starting llama-server", 0, 0)
+	}
+	if err := engine.waitForReady(readyTimeout, progress); err != nil {
 		engine.Close()
 		return nil, fmt.Errorf("llama-server failed to start: %w", err)
 	}
 
 	return engine, nil
 }
 
-func (e *llamaEngine) waitForReady(timeout time.Duration) error {
+func (e *llamaEngine) waitForReady(timeout time.Duration, progress ConvertProgressFunc) error {
 	deadline := time.Now().Add(timeout)
 	url := fmt.Sprintf("http://127.0.0.1:%d/health", e.port)
 
@@ -182,7 +215,15 @@ func (e *llamaEngine) waitForReady(timeout time.Duration) error {
 	exited := make(chan error, 1)
 	go func() { exited <- e.cmd.Wait() }()
 
+	start := time.Now()
+	lastBeat := time.Time{}
+
 	for time.Now().Before(deadline) {
+		if progress != nil && time.Since(lastBeat) >= 2*time.Second {
+			progress("Loading model with llama-server", int(time.Since(start).Seconds()), 0)
+			lastBeat = time.Now()
+		}
+
 		select {
 		case err := <-exited:
 			msg := "llama-server exited unexpectedly"
@@ -209,7 +250,7 @@ func (e *llamaEngine) waitForReady(timeout time.Duration) error {
 		time.Sleep(500 * time.Millisecond)
 	}
 
-	msg := "timeout waiting for llama-server to be ready"
+	msg := fmt.Sprintf("timeout waiting for llama-server to be ready (waited %v; large models need more time — try CSGHUB_LITE_LLAMA_READY_TIMEOUT=45m)", timeout)
 	if e.logBuf != nil {
 		if tail := strings.TrimSpace(e.logBuf.String()); tail != "" {
 			msg += "\n\nllama-server output:\n" + tail