diff --git a/.envoy-version b/.envoy-version new file mode 100644 index 000000000..c6a567b8e --- /dev/null +++ b/.envoy-version @@ -0,0 +1 @@ +1.36.2 diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index eee51604a..207ba2650 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -136,20 +136,13 @@ jobs: test_extproc: needs: changes if: ${{ needs.changes.outputs.code == 'true' }} - name: External Processor Test (Envoy v${{ matrix.version }} on ${{ matrix.os }}) + name: External Processor Test (${{ matrix.os }}) strategy: fail-fast: false matrix: - # Note: we cannot run the latest Envoy version on macOS due to https://github.com/tetratelabs/archive-envoy/issues/12. - # Once it's supported, the following "binary installation" steps below can be just removed and - # we can simply exec.Cmd with "go tool -modfile=tools/go.mod func-e run" with the envoy version configured via ENVOY_VERSION env var. - include: - - version: 1.35.0 # NOTE: when updating this, also update the comment in the CONTRIBUTING.md file. - os: ubuntu-latest - - version: 1.35.0 # NOTE: when updating this, also update the comment in the CONTRIBUTING.md file. - os: macos-latest - - version: latest - os: ubuntu-latest + os: + - ubuntu-latest + - macos-latest runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -164,19 +157,6 @@ jobs: ~/go/pkg/mod ~/go/bin key: extproc-tests-${{ hashFiles('**/go.mod', '**/go.sum', '**/Makefile') }} - - name: Install stable Envoy via func-e - if: matrix.version != 'latest' - run: | - go tool -modfile=tools/go.mod func-e use ${{ matrix.version }} - echo $HOME/.func-e/versions/${{ matrix.version }}/bin >> $GITHUB_PATH - - name: Install latest Envoy - if: matrix.version == 'latest' - run: | - export ENVOY_BIN_DIR=$HOME/envoy/bin - mkdir -p $ENVOY_BIN_DIR - docker run -v $ENVOY_BIN_DIR:/tmp/ci -w /tmp/ci \ - --entrypoint /bin/cp envoyproxy/envoy-dev:latest /usr/local/bin/envoy . - echo $ENVOY_BIN_DIR >> $GITHUB_PATH - env: TEST_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BEDROCK_USER_AWS_ACCESS_KEY_ID }} TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BEDROCK_USER_AWS_SECRET_ACCESS_KEY }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b1af85a3..9f6968734 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,12 +43,6 @@ For example, - The latest `kubectl` binary for running `make test-e2e`. - See: https://kubernetes.io/docs/tasks/tools/ -- The latest `envoy` binary for running `make test-extproc`. The current required version is v1.35 or later. - - On Linux, you can download the latest Envoy binary as described in https://www.envoyproxy.io/docs/envoy/latest/start/install. - Alternatively, you can use `func-e` on Linux as well like on macOS below. - - On macOS, since `brew envoy` tends to behind the latest version, it is recommended use `func-e` to run the latest Envoy. See https://func-e.io/. - - `alias envoy='func-e run'` is a convenient way to run the latest Envoy binary via `func-e` on both macOS and Linux. - For example, `func-e use 1.34` can be used to switch to a specific version of Envoy to be run with `func-e run`. Other than that, everything will be automatically managed and installed via `make` targets, and you should not need to worry about the dependencies (tell us if you do). diff --git a/Makefile b/Makefile index 38b24dd8e..0ef47613d 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,8 @@ test-crdcel: apigen ## Run the integration tests of CEL validation in CRD defini .PHONY: test-extproc # This requires the extproc binary to be built. test-extproc: build.extproc ## Run the integration tests for extproc without controller or k8s at all. @$(MAKE) build.testupstream CMD_PATH_PREFIX=tests/internal/testupstreamlib + @echo "Ensure func-e is built and Envoy is installed" + @@$(GO_TOOL) func-e run --version >/dev/null 2>&1 @echo "Run ExtProc test" @EXTPROC_BIN=$(OUTPUT_DIR)/extproc-$(shell go env GOOS)-$(shell go env GOARCH) go test ./tests/extproc/... $(GO_TEST_E2E_ARGS) diff --git a/tests/extproc/extproc_test.go b/tests/extproc/extproc_test.go index 3a7b9442f..5992b6556 100644 --- a/tests/extproc/extproc_test.go +++ b/tests/extproc/extproc_test.go @@ -59,9 +59,7 @@ var ( Region: "gcp-region", ProjectName: "gcp-project-name", }}} - // This always failing backend is configured to have AWS Bedrock schema so that - // we can test that the extproc can fallback to the different schema. E.g. Primary AWS and then OpenAI. - alwaysFailingBackend = filterapi.Backend{Name: "always-failing-backend", Schema: awsBedrockSchema} + alwaysFailingBackend = filterapi.Backend{Name: "always-failing-backend", Schema: openAISchema} // envoyConfig is the embedded Envoy configuration template. // diff --git a/tests/internal/testenvironment/test_environment.go b/tests/internal/testenvironment/test_environment.go index a86842b38..f49edf088 100644 --- a/tests/internal/testenvironment/test_environment.go +++ b/tests/internal/testenvironment/test_environment.go @@ -13,6 +13,7 @@ import ( "net" "os" "os/exec" + "path/filepath" "runtime" "strconv" "strings" @@ -24,6 +25,7 @@ import ( "github.com/envoyproxy/ai-gateway/cmd/extproc/mainlib" internaltesting "github.com/envoyproxy/ai-gateway/internal/testing" + testsinternal "github.com/envoyproxy/ai-gateway/tests/internal" ) // TestEnvironment holds all the services needed for tests. @@ -257,7 +259,16 @@ func requireEnvoy(t testing.TB, envoyYamlPath := t.TempDir() + "/envoy.yaml" require.NoError(t, os.WriteFile(envoyYamlPath, []byte(processedConfig), 0o600)) - cmd := exec.CommandContext(t.Context(), "envoy", + // Note: do not pass t.Context() to CommandContext, as it's canceled + // *before* t.Cleanup functions are called. + // + // > Context returns a context that is canceled just before + // > Cleanup-registered functions are called. + // + // That means the subprocess gets killed before we can send it an interrupt + // signal for graceful shutdown, which results in orphaned subprocesses. + ctx, cancel := context.WithCancel(context.Background()) + cmd := testsinternal.GoToolCmdContext(ctx, "func-e", "run", "-c", envoyYamlPath, "--concurrency", strconv.Itoa(max(runtime.NumCPU(), 2)), // This allows multiple Envoy instances to run in parallel. @@ -265,6 +276,25 @@ func requireEnvoy(t testing.TB, // Add debug logging for http. "--component-log-level", "http:debug", ) + // func-e will use the version specified in the project root's .envoy-version file. + cmd.Dir = internaltesting.FindProjectRoot() + version, err := os.ReadFile(filepath.Join(cmd.Dir, ".envoy-version")) + require.NoError(t, err) + t.Logf("Starting Envoy version %s", strings.TrimSpace(string(version))) + cmd.WaitDelay = 3 * time.Second // auto-kill after 3 seconds. + t.Cleanup(func() { + defer cancel() + // Graceful shutdown, should kill the Envoy subprocess, too. + if err := cmd.Process.Signal(os.Interrupt); err != nil { + t.Logf("Failed to send interrupt to aigw process: %v", err) + } + // Wait for the process to exit gracefully, in worst case this is + // killed in 3 seconds by WaitDelay above. In that case, you may + // have a zombie Envoy process left behind! + if _, err := cmd.Process.Wait(); err != nil { + t.Logf("Failed to wait for aigw process to exit: %v", err) + } + }) // wait for the ready message or exit. StartAndAwaitReady(t, cmd, stdout, stderr, "starting main dispatch loop")