chore(tests): Mock LLM in tests for PRs

This saves time when testing on CPU which is the only sensible thing
to do on GitHub CI for PRs. For releases or once the commit is merged
we could use an external runner with GPU or just wait.

Signed-off-by: Richard Palethorpe <io@richiejp.com>

Author: richiejp

.github/workflows/tests.yml

@@ -48,7 +48,11 @@ jobs:
 
     - name: Run tests
       run: |
-        make tests
+        if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+          make tests-mock
+        else
+          make tests
+        fi
     #sudo mv coverage/coverage.txt coverage.txt
     #sudo chmod 777 coverage.txt
 

.github/workflows/tests_fragile.yml

@@ -0,0 +1,50 @@
+name: Non-Blocking LLM Tests
+
+on:
+  pull_request:
+    branches:
+      - '**'
+
+concurrency:
+  group: ci-non-blocking-tests-${{ github.head_ref || github.ref }}-${{ github.repository }}
+  cancel-in-progress: true
+
+jobs:
+  llm-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - run: |
+          # Add Docker's official GPG key:
+          sudo apt-get update
+          sudo apt-get install -y ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+          # Add the repository to Apt sources:
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+            $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin make
+          docker version
+
+          docker run --rm hello-world
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '>=1.17.0'
+      - name: Free up disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo apt-get clean
+          docker system prune -af || true
+          df -h
+      - name: Run non-blocking tests
+        run: |
+          make tests
+        continue-on-error: true

Makefile

@@ -13,6 +13,9 @@ cleanup-tests:
 tests: prepare-tests
 	LOCALAGI_MCPBOX_URL="http://localhost:9090" LOCALAGI_MODEL="gemma-3-12b-it-qat" LOCALAI_API_URL="http://localhost:8081" LOCALAGI_API_URL="http://localhost:8080" $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --fail-fast -v -r ./...
 
+tests-mock: prepare-tests
+	LOCALAGI_MCPBOX_URL="http://localhost:9090" LOCALAI_API_URL="http://localhost:8081" LOCALAGI_API_URL="http://localhost:8080" $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --fail-fast -v -r ./...
+
 run-nokb:
 	$(MAKE) run KBDISABLEINDEX=true
 

core/agent/agent.go

@@ -29,7 +29,7 @@ type Agent struct {
 	sync.Mutex
 	options   *options
 	Character Character
-	client    *openai.Client
+	client    llm.LLMClient
 	jobQueue  chan *types.Job
 	context   *types.ActionContext
 
@@ -61,7 +61,12 @@ func New(opts ...Option) (*Agent, error) {
 		return nil, fmt.Errorf("failed to set options: %v", err)
 	}
 
-	client := llm.NewClient(options.LLMAPI.APIKey, options.LLMAPI.APIURL, options.timeout)
+	var client llm.LLMClient
+	if options.llmClient != nil {
+		client = options.llmClient
+	} else {
+		client = llm.NewClient(options.LLMAPI.APIKey, options.LLMAPI.APIURL, options.timeout)
+	}
 
 	c := context.Background()
 	if options.context != nil {
@@ -118,6 +123,11 @@ func New(opts ...Option) (*Agent, error) {
 	return a, nil
 }
 
+// LLMClient returns the agent's LLM client (for testing)
+func (a *Agent) LLMClient() llm.LLMClient {
+	return a.client
+}
+
 func (a *Agent) startNewConversationsConsumer() {
 	go func() {
 		for {

core/agent/agent_suite_test.go

@@ -1,6 +1,7 @@
 package agent_test
 
 import (
+	"net/url"
 	"os"
 	"testing"
 
@@ -13,15 +14,18 @@ func TestAgent(t *testing.T) {
 	RunSpecs(t, "Agent test suite")
 }
 
-var testModel = os.Getenv("LOCALAGI_MODEL")
-var apiURL = os.Getenv("LOCALAI_API_URL")
-var apiKeyURL = os.Getenv("LOCALAI_API_KEY")
+var (
+	testModel      = os.Getenv("LOCALAGI_MODEL")
+	apiURL         = os.Getenv("LOCALAI_API_URL")
+	apiKeyURL      = os.Getenv("LOCALAI_API_KEY")
+	useRealLocalAI bool
+)
+
+func isValidURL(u string) bool {
+	parsed, err := url.ParseRequestURI(u)
+	return err == nil && parsed.Scheme != "" && parsed.Host != ""
+}
 
 func init() {
-	if testModel == "" {
-		testModel = "hermes-2-pro-mistral"
-	}
-	if apiURL == "" {
-		apiURL = "http://192.168.68.113:8080"
-	}
+	useRealLocalAI = isValidURL(apiURL) && apiURL != "" && testModel != ""
 }