test: consolidates e2e-ish aigw cli tests into e2e-aigw (#1359)

Author: mathetake

.github/workflows/build_and_test.yaml

@@ -67,32 +67,7 @@ jobs:
             ~/go/pkg/mod
             ~/go/bin
           key: unittest-${{ hashFiles('**/go.mod', '**/go.sum', '**/Makefile') }}-${{ matrix.os }}
-
-      # This runs ollama server to be used in `aigw run` end-to-end tests.
-      # The test case using it will be skipped if ollama is not available.
-      # Since installing it and pulling the model takes a while, we do it only for Linux runners.
-      - name: Start Ollama server
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-          curl -fsSL https://ollama.com/install.sh | sh && sudo systemctl stop ollama
-          nohup ollama serve > ollama.log 2>&1 &
-          timeout 30 sh -c 'until nc -z localhost 11434; do sleep 1; done'
-          grep _MODEL .env.ollama | cut -d= -f2 | xargs -I{} ollama pull {}
-        env:
-          OLLAMA_CONTEXT_LENGTH: 131072 # Larger context for goose
-          OLLAMA_HOST: 0.0.0.0
-      # Download Envoy via func-e using implicit default version `aigw` would
-      # otherwise need to download during test runs.
-      - name: Download Envoy via func-e
-        run: go tool -modfile=tools/go.mod func-e run --version
-        env:
-          FUNC_E_HOME: /tmp/envoy-gateway # hard-coded directory in EG
-      - env:
-          TEST_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BEDROCK_USER_AWS_ACCESS_KEY_ID }}
-          TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BEDROCK_USER_AWS_SECRET_ACCESS_KEY }}
-          TEST_OPENAI_API_KEY: ${{ secrets.ENVOY_AI_GATEWAY_OPENAI_API_KEY }}
-          TEST_GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: make test-coverage
+      - run: make test-coverage
       - if: failure()
         run: cat ollama.log || true
       - name: Upload coverage to Codecov
@@ -391,7 +366,10 @@ jobs:
           GOOSE_VERSION: v1.8.0
         run: |
           curl -fsSL https://github.com/block/goose/releases/download/stable/download_cli.sh | CONFIGURE=false bash
-      - run: make test-e2e-aigw
+      - env:
+          # This is used to access the GitHub MCP server.
+          TEST_GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: make test-e2e-aigw
       - if: failure()
         run: cat ollama.log || true
 

cmd/aigw/run_test.go

@@ -24,9 +24,6 @@ import (
 	"time"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
-	"github.com/modelcontextprotocol/go-sdk/mcp"
-	"github.com/openai/openai-go"
-	"github.com/openai/openai-go/option"
 	"github.com/stretchr/testify/require"
 	"github.com/tetratelabs/func-e/api"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,84 +32,29 @@ import (
 
 	"github.com/envoyproxy/ai-gateway/cmd/extproc/mainlib"
 	"github.com/envoyproxy/ai-gateway/internal/aigw"
-	"github.com/envoyproxy/ai-gateway/internal/autoconfig"
 	"github.com/envoyproxy/ai-gateway/internal/filterapi"
 	internaltesting "github.com/envoyproxy/ai-gateway/internal/testing"
 )
 
-// startupRegexp ensures the status message is written to stderr as we know we are healthy!
-var startupRegexp = `Envoy AI Gateway listening on http://localhost:1975 \(admin http://localhost:\d+\) after [^\n]+`
-
+// TestRun verifies that the main run function starts up correctly without making any actual requests.
+//
+// The real e2e tests are in tests/e2e-aigw.
 func TestRun(t *testing.T) {
-	ollamaModel, err := internaltesting.GetOllamaModel(internaltesting.ChatModel)
-	if err == nil {
-		err = internaltesting.CheckIfOllamaReady(ollamaModel)
-	}
-	if err != nil {
-		t.Skip(err)
-	}
-
 	ports := internaltesting.RequireRandomPorts(t, 1)
 	// TODO: parameterize the main listen port 1975
 	adminPort := ports[0]
 
-	t.Setenv("OPENAI_BASE_URL", "http://localhost:11434/v1")
+	// Note: we do not make any real requests here!
+	t.Setenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
 	t.Setenv("OPENAI_API_KEY", "unused")
 
 	buffers := internaltesting.DumpLogsOnFail(t, "aigw Stdout", "aigw Stderr")
 	stdout, stderr := buffers[0], buffers[1]
 	ctx, cancel := context.WithCancel(t.Context())
 	defer cleanupRun(t, cancel)
 
-	go func() {
-		opts := runOpts{extProcLauncher: mainlib.Main}
-		require.NoError(t, run(ctx, cmdRun{Debug: true, AdminPort: adminPort}, opts, stdout, stderr))
-	}()
-
-	client := openai.NewClient(option.WithBaseURL("http://localhost:1975/v1/"))
-	chatReq := openai.ChatCompletionNewParams{
-		Messages: []openai.ChatCompletionMessageParamUnion{
-			openai.UserMessage("Say this is a test"),
-		},
-		Model: ollamaModel,
-	}
-
-	t.Run("chat completion", func(t *testing.T) {
-		internaltesting.RequireEventuallyNoError(t, func() error {
-			chatCompletion, err := client.Chat.Completions.New(ctx, chatReq)
-			if err != nil {
-				return fmt.Errorf("chat completion failed: %w", err)
-			}
-			for _, choice := range chatCompletion.Choices {
-				if choice.Message.Content != "" {
-					return nil
-				}
-			}
-			return fmt.Errorf("no content in response")
-		}, 1*time.Minute, 2*time.Second,
-			"chat completion never succeeded")
-	})
-
-	// By now, we're listening
-	require.Regexp(t, startupRegexp, stderr.String())
-
-	t.Run("access metrics", func(t *testing.T) {
-		internaltesting.RequireEventuallyNoError(t, func() error {
-			req, err := http.NewRequest(http.MethodGet,
-				fmt.Sprintf("http://localhost:%d/metrics", adminPort), nil)
-			require.NoError(t, err)
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				return err
-			}
-			defer resp.Body.Close()
-			if resp.StatusCode != http.StatusOK {
-				return fmt.Errorf("status %d", resp.StatusCode)
-			}
-			return nil
-		}, 1*time.Minute, time.Second,
-			"metrics endpoint never became available")
-	})
+	opts := runOpts{extProcLauncher: func(context.Context, []string, io.Writer) error { return nil }}
+	require.NoError(t, run(ctx, cmdRun{Debug: true, AdminPort: adminPort}, opts, stdout, stderr))
 }
 
 func cleanupRun(t testing.TB, cancel context.CancelFunc) {
@@ -126,73 +68,6 @@ func cleanupRun(t testing.TB, cancel context.CancelFunc) {
 	}
 }
 
-// TestRunMCP runs the AIGW with MCP configured and verifies calling a tool.
-// It uses the same MCP config as docker-compose.yaml to ensure consistency.
-func TestRunMCP(t *testing.T) {
-	ports := internaltesting.RequireRandomPorts(t, 1)
-	// TODO: parameterize the main listen port 1975
-	adminPort := ports[0]
-
-	mcpServers := &autoconfig.MCPServers{
-		McpServers: map[string]autoconfig.MCPServer{
-			"kiwi": {
-				Type: "http",
-				URL:  "https://mcp.kiwi.com",
-			},
-		},
-	}
-
-	buffers := internaltesting.DumpLogsOnFail(t, "aigw Stdout", "aigw Stderr")
-	stdout, stderr := buffers[0], buffers[1]
-	ctx, cancel := context.WithCancel(t.Context())
-	defer cleanupRun(t, cancel)
-
-	go func() {
-		opts := runOpts{extProcLauncher: mainlib.Main}
-		require.NoError(t, run(ctx, cmdRun{Debug: true, AdminPort: adminPort, mcpConfig: mcpServers}, opts, stdout, stderr))
-	}()
-
-	url := fmt.Sprintf("http://localhost:%d/mcp", 1975)
-	mcpClient := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "0.1.0"},
-		&mcp.ClientOptions{})
-
-	// Calculate departure date as one week from now
-	departureDate := time.Now().AddDate(0, 0, 7).Format("02/01/2006")
-
-	callTool := &mcp.CallToolParams{
-		Name: "kiwi__search-flight",
-		Arguments: map[string]any{
-			"flyFrom":       "NYC",
-			"flyTo":         "LAX",
-			"departureDate": departureDate,
-		},
-	}
-
-	t.Run("call tool", func(t *testing.T) {
-		internaltesting.RequireEventuallyNoError(t, func() error {
-			session, err := mcpClient.Connect(t.Context(),
-				&mcp.StreamableClientTransport{Endpoint: url}, nil)
-			if err != nil {
-				return fmt.Errorf("connect failed: %w", err)
-			}
-			defer session.Close()
-
-			resp, err := session.CallTool(t.Context(), callTool)
-			if err != nil {
-				return fmt.Errorf("call tool failed: %w", err)
-			}
-			if resp.IsError {
-				return fmt.Errorf("tool returned error response")
-			}
-			return nil
-		}, 2*time.Minute, 2*time.Second,
-			"MCP tool call never succeeded")
-	})
-
-	// By now, we're listening
-	require.Regexp(t, startupRegexp, stderr.String())
-}
-
 func TestRunExtprocStartFailure(t *testing.T) {
 	t.Setenv("OPENAI_BASE_URL", "http://localhost:11434/v1")
 	t.Setenv("OPENAI_API_KEY", "unused")

internal/extensionserver/mcproute_test.go

@@ -124,13 +124,69 @@ func TestServer_createRoutesForBackendListener(t *testing.T) {
 			},
 			expectedRoute: nil,
 		},
+		{
+			name: "with MCP routes",
+			routes: []*routev3.RouteConfiguration{
+				{
+					VirtualHosts: []*routev3.VirtualHost{
+						{
+							Name:   "test-vh",
+							Routes: []*routev3.Route{{Name: "normal"}},
+						},
+					},
+				},
+				{
+					VirtualHosts: []*routev3.VirtualHost{
+						{
+							Name:    "mcp-vh",
+							Domains: []string{"*"},
+							Routes: []*routev3.Route{
+								{
+									Name: internalapi.MCPPerBackendRefHTTPRoutePrefix + "foo/rule/0",
+									Action: &routev3.Route_Route{
+										Route: &routev3.RouteAction{ClusterSpecifier: &routev3.RouteAction_Cluster{}},
+									},
+								},
+								{
+									Name: internalapi.MCPPerBackendRefHTTPRoutePrefix + "bar/rule/1",
+									Action: &routev3.Route_Route{
+										Route: &routev3.RouteAction{ClusterSpecifier: &routev3.RouteAction_Cluster{}},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			expectedRoute: &routev3.RouteConfiguration{
+				Name: "aigateway-mcp-backend-listener-route-config",
+				VirtualHosts: []*routev3.VirtualHost{
+					{
+						Domains: []string{"*"},
+						Name:    "aigateway-mcp-backend-listener-wildcard",
+						Routes: []*routev3.Route{
+							{Name: internalapi.MCPPerBackendRefHTTPRoutePrefix + "foo/rule/0", Action: &routev3.Route_Route{
+								Route: &routev3.RouteAction{ClusterSpecifier: &routev3.RouteAction_Cluster{}},
+							}},
+							{Name: internalapi.MCPPerBackendRefHTTPRoutePrefix + "bar/rule/1", Action: &routev3.Route_Route{
+								Route: &routev3.RouteAction{ClusterSpecifier: &routev3.RouteAction_Cluster{}},
+							}},
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			s := &Server{log: testr.New(t)}
 			route := s.createRoutesForBackendListener(tt.routes)
-			require.Equal(t, tt.expectedRoute, route)
+			if tt.expectedRoute == nil {
+				require.Nil(t, route)
+			} else {
+				require.Empty(t, cmp.Diff(tt.expectedRoute, route, protocmp.Transform()))
+			}
 		})
 	}
 }

internal/mcpproxy/handlers_test.go

@@ -286,6 +286,18 @@ func TestServePOST_JSONRPCRequest(t *testing.T) {
 			params:        &mcp.ListToolsParams{},
 			expStatusCode: 202,
 		},
+		{
+			name:          "initialize invalid param",
+			method:        "initialize",
+			params:        "invalid-param",
+			expStatusCode: 400,
+		},
+		{
+			name:          "initialize without route header",
+			method:        "initialize",
+			params:        &mcp.InitializeParams{},
+			expStatusCode: 500,
+		},
 		{
 			method:           "tools/list",
 			upstreamResponse: `{"jsonrpc":"2.0","id":"1","result":{"tools":[{"name":"my-tool"},{"name":"test-tool"}]}}`,
@@ -298,6 +310,18 @@ func TestServePOST_JSONRPCRequest(t *testing.T) {
 				require.Equal(t, "backend1__test-tool", result.Tools[0].Name)
 			},
 		},
+		{
+			method:           "notifications/roots/list_changed",
+			upstreamResponse: `{"jsonrpc":"2.0","id":"1","result":{}}`,
+			params:           &mcp.RootsListChangedParams{},
+			expStatusCode:    202,
+		},
+		{
+			name:          "notifications/roots/list_changed invalid param",
+			method:        "notifications/roots/list_changed",
+			params:        "invalid-param",
+			expStatusCode: 400,
+		},
 		{
 			method:           "prompts/list",
 			upstreamResponse: `{"jsonrpc":"2.0","id":"1","result":{"prompts":[{"name":"my-prompt"}]}}`,
@@ -310,6 +334,12 @@ func TestServePOST_JSONRPCRequest(t *testing.T) {
 				require.Equal(t, "backend1__my-prompt", result.Prompts[0].Name)
 			},
 		},
+		{
+			name:          "prompts/list invalid param type",
+			method:        "prompts/list",
+			params:        "invalid",
+			expStatusCode: 400,
+		},
 		{
 			method:           "resources/list",
 			upstreamResponse: `{"jsonrpc":"2.0","id":"1","result":{"resources":[{"name":"my-resource"}]}}`,

tests/e2e-aigw/aigw_test.go

@@ -57,7 +57,7 @@ func buildAigwOnDemand() (string, error) {
 }
 
 // startAIGWCLI starts the aigw CLI as a subprocess with the given config file.
-func startAIGWCLI(t *testing.T, aigwBin string, env []string, arg ...string) {
+func startAIGWCLI(t *testing.T, aigwBin string, env []string, arg ...string) (adminPort int) {
 	// aigw has many fixed ports: some are in the envoy subprocess
 	gatewayPort := 1975
 
@@ -151,6 +151,7 @@ func startAIGWCLI(t *testing.T, aigwBin string, env []string, arg ...string) {
 		"MCP endpoint never became available")
 
 	t.Log("aigw CLI is ready with MCP endpoint")
+	return envoyAdmin.Port()
 }
 
 // Function to check if a port is in use (returns true if listening).

tests/e2e-aigw/examples_test.go renamed to tests/e2e-aigw/examples_mcp_test.go

@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"os"
 	"path"
-	"slices"
 	"sort"
 	"testing"
 	"time"
@@ -34,21 +33,6 @@ var (
 		"kiwi__feedback-to-devs",
 		"kiwi__search-flight",
 	}
-	allGithubTools = []string{
-		"github__get_issue",
-		"github__get_issue_comments",
-		"github__list_issue_types",
-		"github__list_issues",
-		"github__list_pull_requests",
-		"github__list_sub_issues",
-		"github__pull_request_read",
-		"github__search_issues",
-		"github__search_pull_requests",
-	}
-	allTools = func() []string {
-		combined := append(append([]string{}, allNonGithubTools...), allGithubTools...)
-		return slices.Sorted(slices.Values(combined))
-	}()
 
 	// Filtered tools based on mcp_example.yaml selectors
 	filteredNonGithubTools = []string{