chore(tests): Mock LLM in tests for PRs

This saves time when testing on CPU which is the only sensible thing
to do on GitHub CI for PRs. For releases or once the commit is merged
we could use an external runner with GPU or just wait.

Signed-off-by: Richard Palethorpe <io@richiejp.com>

Author: richiejp

.github/workflows/tests.yml

@@ -48,7 +48,11 @@ jobs:
 
     - name: Run tests
       run: |
-        make tests
+        if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+          make tests-mock
+        else
+          make tests
+        fi
     #sudo mv coverage/coverage.txt coverage.txt
     #sudo chmod 777 coverage.txt
 

.github/workflows/tests_fragile.yml

@@ -0,0 +1,49 @@
+name: Run Fragile Go Tests
+
+on:
+  pull_request:
+    branches:
+      - '**'
+
+concurrency:
+  group: ci-non-blocking-tests-${{ github.head_ref || github.ref }}-${{ github.repository }}
+  cancel-in-progress: true
+
+jobs:
+  llm-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - run: |
+          # Add Docker's official GPG key:
+          sudo apt-get update
+          sudo apt-get install -y ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+          # Add the repository to Apt sources:
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+            $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin make
+          docker version
+
+          docker run --rm hello-world
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '>=1.17.0'
+      - name: Free up disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo apt-get clean
+          docker system prune -af || true
+          df -h
+      - name: Run tests
+        run: |
+          make tests

Makefile

@@ -3,6 +3,8 @@ IMAGE_NAME?=webui
 MCPBOX_IMAGE_NAME?=mcpbox
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
+.PHONY: tests tests-mock cleanup-tests
+
 prepare-tests: build-mcpbox
 	docker compose up -d --build
 	docker run -d -v /var/run/docker.sock:/var/run/docker.sock --privileged -p 9090:8080 --rm -ti $(MCPBOX_IMAGE_NAME)
@@ -13,6 +15,9 @@ cleanup-tests:
 tests: prepare-tests
 	LOCALAGI_MCPBOX_URL="http://localhost:9090" LOCALAGI_MODEL="gemma-3-12b-it-qat" LOCALAI_API_URL="http://localhost:8081" LOCALAGI_API_URL="http://localhost:8080" $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --fail-fast -v -r ./...
 
+tests-mock: prepare-tests
+	LOCALAGI_MCPBOX_URL="http://localhost:9090" LOCALAI_API_URL="http://localhost:8081" LOCALAGI_API_URL="http://localhost:8080" $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --fail-fast -v -r ./...
+
 run-nokb:
 	$(MAKE) run KBDISABLEINDEX=true
 
@@ -37,4 +42,4 @@ build-mcpbox:
 	docker build -t $(MCPBOX_IMAGE_NAME) -f Dockerfile.mcpbox .
 
 run-mcpbox:
-	docker run -v /var/run/docker.sock:/var/run/docker.sock --privileged -p 9090:8080 -ti mcpbox
+	docker run -v /var/run/docker.sock:/var/run/docker.sock --privileged -p 9090:8080 -ti mcpbox

core/agent/agent.go

@@ -29,7 +29,7 @@ type Agent struct {
 	sync.Mutex
 	options   *options
 	Character Character
-	client    *openai.Client
+	client    llm.LLMClient
 	jobQueue  chan *types.Job
 	context   *types.ActionContext
 
@@ -63,7 +63,12 @@ func New(opts ...Option) (*Agent, error) {
 		return nil, fmt.Errorf("failed to set options: %v", err)
 	}
 
-	client := llm.NewClient(options.LLMAPI.APIKey, options.LLMAPI.APIURL, options.timeout)
+	var client llm.LLMClient
+	if options.llmClient != nil {
+		client = options.llmClient
+	} else {
+		client = llm.NewClient(options.LLMAPI.APIKey, options.LLMAPI.APIURL, options.timeout)
+	}
 
 	c := context.Background()
 	if options.context != nil {
@@ -125,6 +130,11 @@ func (a *Agent) SharedState() *types.AgentSharedState {
 	return a.sharedState
 }
 
+// LLMClient returns the agent's LLM client (for testing)
+func (a *Agent) LLMClient() llm.LLMClient {
+	return a.client
+}
+
 func (a *Agent) startNewConversationsConsumer() {
 	go func() {
 		for {

core/agent/agent_suite_test.go

@@ -1,6 +1,7 @@
 package agent_test
 
 import (
+	"net/url"
 	"os"
 	"testing"
 
@@ -13,15 +14,19 @@ func TestAgent(t *testing.T) {
 	RunSpecs(t, "Agent test suite")
 }
 
-var testModel = os.Getenv("LOCALAGI_MODEL")
-var apiURL = os.Getenv("LOCALAI_API_URL")
-var apiKeyURL = os.Getenv("LOCALAI_API_KEY")
+var (
+	testModel      = os.Getenv("LOCALAGI_MODEL")
+	apiURL         = os.Getenv("LOCALAI_API_URL")
+	apiKey         = os.Getenv("LOCALAI_API_KEY")
+	useRealLocalAI bool
+	clientTimeout  = "10m"
+)
+
+func isValidURL(u string) bool {
+	parsed, err := url.ParseRequestURI(u)
+	return err == nil && parsed.Scheme != "" && parsed.Host != ""
+}
 
 func init() {
-	if testModel == "" {
-		testModel = "hermes-2-pro-mistral"
-	}
-	if apiURL == "" {
-		apiURL = "http://192.168.68.113:8080"
-	}
+	useRealLocalAI = isValidURL(apiURL) && apiURL != "" && testModel != ""
 }