Binary e2e tests

.github/workflows/ci.yml

@@ -20,10 +20,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
       - name: Set up Go
-        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5  # v6.2.0
+        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
         with:
           go-version: "1.26.0"
           cache: true
@@ -43,10 +43,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
       - name: Set up Go
-        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5  # v6.2.0
+        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
         with:
           go-version: "1.26.0"
           cache: true
@@ -57,16 +57,18 @@ jobs:
           version: 3.x
 
       - name: Run tests
-        run: task test
+        run: |
+          task test
+          task test-binary
 
   license-check:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
       - name: Set up Go
-        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5  # v6.2.0
+        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
         with:
           go-version: "1.26.0"
           cache: true
@@ -82,10 +84,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
       - name: Set up Go
-        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5  # v6.2.0
+        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
         with:
           go-version: "1.26.0"
           cache: true
@@ -106,7 +108,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
       - name: Hub login
         if: github.event_name != 'pull_request'
@@ -116,11 +118,11 @@ jobs:
           password: ${{ secrets.DOCKERPUBLICBOT_WRITE_PAT }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f  # v3.12.0
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
 
       - name: Docker metadata
         id: meta
-        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051  # v5.10.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
         with:
           images: docker/cagent
           tags: |

Taskfile.yml

@@ -25,7 +25,7 @@ tasks:
     desc: Build the application binary
     cmds:
       - go build -ldflags "{{.LDFLAGS}}" -o {{.BUILD_DIR}}/{{.BINARY_NAME}} {{.MAIN_PKG}}
-      - ln -sf {{.USER_WORKING_DIR}}/{{.BUILD_DIR}}/{{.BINARY_NAME}} {{.HOME}}/bin/{{.BINARY_NAME}}
+      - '{{if ne .CI "true"}}ln -sf {{.USER_WORKING_DIR}}/{{.BUILD_DIR}}/{{.BINARY_NAME}} {{.HOME}}/bin/{{.BINARY_NAME}}{{end}}'
     sources:
       - "{{.GO_SOURCES}}"
       - "**/*.template"
@@ -40,7 +40,9 @@ tasks:
     desc: Deploy the docker agent cli-plugin
     deps: ["build"]
     cmds:
+      - mkdir -p ~/.docker/cli-plugins
       - cp "{{.BUILD_DIR}}/{{.BINARY_NAME}}" ~/.docker/cli-plugins/{{.CLI_PLUGIN_BINARY_NAME}}
+      - cp "{{.BUILD_DIR}}/{{.BINARY_NAME}}" {{.BUILD_DIR}}/{{.CLI_PLUGIN_BINARY_NAME}}
 
   lint:
     desc: Run golangci-lint
@@ -63,7 +65,12 @@ tasks:
   test:
     aliases: [t]
     desc: Run tests
-    cmd: CAGENT_MODELS_GATEWAY= OPENAI_API_KEY= ANTHROPIC_API_KEY= GOOGLE_API_KEY= GOOGLE_GENAI_USE_VERTEXAI= MISTRAL_API_KEY= GITHUB_TOKEN= go test {{.CLI_ARGS}} ./...
+    cmd: CAGENT_MODELS_GATEWAY= OPENAI_API_KEY= ANTHROPIC_API_KEY= GOOGLE_API_KEY= GOOGLE_GENAI_USE_VERTEXAI= MISTRAL_API_KEY= GITHUB_TOKEN= go test {{.CLI_ARGS}}  ./...
+
+  test-binary:
+    deps: ["deploy-local"]
+    desc: Run tests on build binary
+    cmd: CAGENT_MODELS_GATEWAY= OPENAI_API_KEY= ANTHROPIC_API_KEY= GOOGLE_API_KEY= GOOGLE_GENAI_USE_VERTEXAI= MISTRAL_API_KEY= GITHUB_TOKEN= go test -tags=binary_required {{.CLI_ARGS}}  ./e2e/binary/...
 
   build-local:
     desc: Build binaries for local host platform

cmd/root/root.go

@@ -186,15 +186,7 @@ We collect anonymous usage data to help improve cagent. To disable:
 			}
 			return nil
 		}
-		if rootCmd.RunE != nil {
-			originalRunE := rootCmd.RunE
-			rootCmd.RunE = func(cmd *cobra.Command, args []string) error {
-				if err := originalRunE(cmd, args); err != nil {
-					return processErr(cmd.Context(), err, stderr, rootCmd)
-				}
-				return nil
-			}
-		}
+		setErrorHandlingRecursive(rootCmd, processErr)
 		return rootCmd
 	}, metadata.Metadata{
 		SchemaVersion: "0.1.0",
@@ -212,6 +204,22 @@ func setContextRecursive(ctx context.Context, cmd *cobra.Command) {
 	}
 }
 
+func setErrorHandlingRecursive(cmd *cobra.Command, processErr func(context.Context, error, io.Writer, *cobra.Command) error) {
+	if cmd.RunE != nil {
+		originalRunE := cmd.RunE
+		cmd.RunE = func(cmd *cobra.Command, args []string) error {
+			if err := originalRunE(cmd, args); err != nil {
+				return processErr(cmd.Context(), err, cmd.ErrOrStderr(), cmd)
+			}
+			return nil
+		}
+	}
+
+	for _, child := range cmd.Commands() {
+		setErrorHandlingRecursive(child, processErr)
+	}
+}
+
 // defaultToRun prepends "run" to the argument list when no subcommand is
 // specified so that bare "cagent" (or "cagent --debug", etc.) launches the
 // default agent. Help flags (--help / -h) are left alone.

e2e/binary/binary_test.go

@@ -0,0 +1,55 @@
+//go:build binary_required
+// +build binary_required
+
+package binary
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+const binDir = "../../bin"
+
+func TestHelpInAllExecMode(t *testing.T) {
+	t.Run("cli plugin help", func(t *testing.T) {
+		res, err := Exec("docker", "agent", "help")
+		require.NoError(t, err)
+		require.Contains(t, res.Stdout, "docker agent run ./agent.yaml")
+	})
+
+	t.Run("cagent help", func(t *testing.T) {
+		res, err := Exec(binDir+"/cagent", "help")
+		require.NoError(t, err)
+		require.Contains(t, res.Stdout, "cagent run ./agent.yaml")
+	})
+
+	t.Run("docker-agent help", func(t *testing.T) {
+		res, err := Exec(binDir+"/docker-agent", "help")
+		require.NoError(t, err)
+		require.Contains(t, res.Stdout, "docker-agent run ./agent.yaml")
+	})
+}
+
+func TestExecMissingKeys(t *testing.T) {
+	t.Run("cli plugin exec", func(t *testing.T) {
+		res, err := Exec("docker", "agent", "run", "--exec", "./test-agent.yaml")
+		require.Error(t, err)
+		require.Contains(t, res.Stderr, "environment variables must be set")
+		require.Contains(t, res.Stderr, "OPENAI_API_KEY")
+	})
+
+	t.Run("cagent exec", func(t *testing.T) {
+		res, err := Exec(binDir+"/cagent", "run", "--exec", "./test-agent.yaml")
+		require.Error(t, err)
+		require.Contains(t, res.Stderr, "environment variables must be set")
+		require.Contains(t, res.Stderr, "OPENAI_API_KEY")
+	})
+
+	t.Run("docker-agent exec", func(t *testing.T) {
+		res, err := Exec(binDir+"/docker-agent", "run", "--exec", "./test-agent.yaml")
+		require.Error(t, err)
+		require.Contains(t, res.Stderr, "environment variables must be set")
+		require.Contains(t, res.Stderr, "OPENAI_API_KEY")
+	})
+}

e2e/binary/shellout.go

@@ -0,0 +1,55 @@
+package binary
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// CmdResult output of a command, including stdout and stderr
+type CmdResult struct {
+	Stdout string
+	Stderr string
+}
+
+func ExecWithContext(ctx context.Context, cmd string, args ...string) (CmdResult, error) {
+	return ExecWithContextInDir(ctx, "", cmd, args, nil)
+}
+
+func Exec(cmd string, args ...string) (CmdResult, error) {
+	return ExecWithContextInDir(context.Background(), "", cmd, args, nil)
+}
+
+func ExecWithContextAndEnv(ctx context.Context, env []string, cmd string, args ...string) (CmdResult, error) {
+	return ExecWithContextInDir(ctx, "", cmd, args, env)
+}
+
+func ExecWithContextInDir(ctx context.Context, dir, cmd string, args, env []string) (CmdResult, error) {
+	command := exec.CommandContext(ctx, cmd, args...)
+	command.Dir = dir
+	command.Env = append(os.Environ(), env...)
+	res, err := runCmd(command)
+	if err != nil {
+		return res, fmt.Errorf("executing '%s %s' : %s: %s", cmd, strings.Join(args, " "), err.Error(), res.Stdout+"\n"+res.Stderr)
+	}
+	return res, nil
+}
+
+func runCmd(c *exec.Cmd) (CmdResult, error) {
+	if c.Stdout != nil {
+		return CmdResult{}, errors.New("exec: Stdout already set")
+	}
+	if c.Stderr != nil {
+		return CmdResult{}, errors.New("exec: Stderr already set")
+	}
+	var outBuffer bytes.Buffer
+	var errBuffer bytes.Buffer
+	c.Stdout = &outBuffer
+	c.Stderr = &errBuffer
+	err := c.Run()
+	return CmdResult{outBuffer.String(), errBuffer.String()}, err
+}

e2e/binary/test-agent.yaml

@@ -0,0 +1,4 @@
+version: "2"
+agents:
+  root:
+    model: openai/gpt-4o