Merge pull request #620 from apollographql/AMS-238

ci: run e2e tests in CI

Author: DaleSeo

.github/workflows/ci.yml

@@ -70,3 +70,24 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: codecov.json
           fail_ci_if_error: true
+
+  e2e:
+    name: End-to-End Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Build release binary
+        run: cargo build --release
+
+      - name: Run e2e tests (local-operations)
+        run: ./e2e/mcp-server-tester/run_tests.sh local-operations
+
+      - name: Run e2e tests (pq-manifest)
+        run: ./e2e/mcp-server-tester/run_tests.sh pq-manifest

.gitignore

@@ -22,3 +22,6 @@ target/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# Generated e2e test configs
+e2e/**/config.generated.yaml

e2e/mcp-server-tester/local-operations/config.yaml

@@ -4,10 +4,10 @@ transport:
 operations:
   source: local
   paths:
-    - ./local-operations/operations
+    - <test-dir>/operations
 schema:
   source: local
-  path: ./local-operations/api.graphql
+  path: <test-dir>/api.graphql
 overrides:
   mutation_mode: all
 introspection:

e2e/mcp-server-tester/pq-manifest/config.yaml

@@ -3,10 +3,10 @@ transport:
   type: stdio
 operations:
   source: manifest
-  path: ./pq-manifest/apollo.json
+  path: <test-dir>/apollo.json
 schema:
   source: local
-  path: ./pq-manifest/api.graphql
+  path: <test-dir>/api.graphql
 overrides:
   mutation_mode: all
 introspection:

e2e/mcp-server-tester/run_tests.sh

@@ -30,7 +30,6 @@ else
   TEST_DIR="$(cd -P -- "$SCRIPT_DIR/$RAW_DIR_ARG" && pwd)"
 fi
 
-TEST_DIR="${1%/}"  # strip trailing slash if present
 TESTS="$TEST_DIR/tool-tests.yaml"
 MCP_CONFIG="$TEST_DIR/config.yaml"
 
@@ -62,13 +61,21 @@ cleanup() { rm -rf "$TMP_DIR"; }
 trap cleanup EXIT INT TERM # cleanup before exiting
 GEN_CONFIG="$TMP_DIR/server-config.generated.json"
 
-# Safe replacement for <test-dir> with absolute path (handles /, &, and |)
+# Safe replacement for paths with absolute path (handles /, &, and |)
 safe_dir="${TEST_DIR//\\/\\\\}"
 safe_dir="${safe_dir//&/\\&}"
 safe_dir="${safe_dir//|/\\|}"
 
-# Replace the literal token "<test-dir>" everywhere
-sed "s|<test-dir>|$safe_dir|g" "$TEMPLATE_PATH" > "$GEN_CONFIG"
+safe_bin="${BIN_PATH//\\/\\\\}"
+safe_bin="${safe_bin//&/\\&}"
+safe_bin="${safe_bin//|/\\|}"
 
-# Run the command
-npx -y mcp-server-tester@1.4.0 tools "$TESTS" --server-config "$GEN_CONFIG"
+# Generate MCP config.yaml with absolute paths
+GEN_MCP_CONFIG="$TEST_DIR/config.generated.yaml"
+sed -e "s|<test-dir>|$safe_dir|g" "$MCP_CONFIG" > "$GEN_MCP_CONFIG"
+
+# Replace the literal tokens "<test-dir>" and "<bin-path>" in server config
+sed -e "s|<test-dir>|$safe_dir|g" -e "s|<bin-path>|$safe_bin|g" "$TEMPLATE_PATH" > "$GEN_CONFIG"
+
+# Run the command (ignore-scripts needed due to mcp-server-tester's postinstall requiring patch-package)
+npm_config_ignore_scripts=true npx -y mcp-server-tester@1.4.1 tools "$TESTS" --server-config "$GEN_CONFIG"

e2e/mcp-server-tester/server-config.template.json

@@ -1,8 +1,8 @@
 {
   "mcpServers": {
     "mcp-server": {
-      "command": "../../target/release/apollo-mcp-server",
-      "args": ["./<test-dir>/config.yaml"]
+      "command": "<bin-path>",
+      "args": ["<test-dir>/config.generated.yaml"]
     }
   }
 }