Skip to content

Commit ba79bfa

Browse files
mathetakemathetake
authored
controller: removes openAIPrefix config (envoyproxy#1063)
**Description** The configurability for "/v1" was introduced in envoyproxy#1020. However, it is unnecessary configuration no one asked for at the moment given that we keep the rootPrefix for the separation concern between AIGatewayRoutes vs HTTPRoutes. This partially reverts envoyproxy#1020, and removes the config so that we can have a simpler config overall. We can revisit this if anyone asks for it later. If so, I think it will be a time to think about Gateway level CRD. **Related Issues/PRs (if applicable)** Follow up on envoyproxy#1020 --------- Signed-off-by: Takeshi Yoneda <[email protected]>
1 parent f0eeed8 commit ba79bfa

File tree

9 files changed

+67
-99
lines changed

9 files changed

+67
-99
lines changed

‎cmd/controller/main.go‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ type flags struct {
4444
caBundleName string
4545
metricsRequestHeaderLabels string
4646
rootPrefix string
47-
openAIPrefix string
4847
extProcExtraEnvVars string
4948
}
5049

@@ -123,11 +122,6 @@ func parseAndValidateFlags(args []string) (flags, error) {
123122
"/",
124123
`The root prefix for all supported endpoints. Default is "/"`,
125124
)
126-
openAIPrefix := fs.String(
127-
"openAIPrefix",
128-
"/v1",
129-
`The prefix for OpenAI endpoints following *after* the root prefix. Default is "/v1".`,
130-
)
131125
extProcExtraEnvVars := fs.String(
132126
"extProcExtraEnvVars",
133127
"",
@@ -185,7 +179,6 @@ func parseAndValidateFlags(args []string) (flags, error) {
185179
caBundleName: *caBundleName,
186180
metricsRequestHeaderLabels: *metricsRequestHeaderLabels,
187181
rootPrefix: *rootPrefix,
188-
openAIPrefix: *openAIPrefix,
189182
extProcExtraEnvVars: *extProcExtraEnvVars,
190183
}, nil
191184
}
@@ -261,7 +254,6 @@ func main() {
261254
UDSPath: extProcUDSPath,
262255
MetricsRequestHeaderLabels: flags.metricsRequestHeaderLabels,
263256
RootPrefix: flags.rootPrefix,
264-
OpenAIPrefix: flags.openAIPrefix,
265257
ExtProcExtraEnvVars: flags.extProcExtraEnvVars,
266258
}); err != nil {
267259
setupLog.Error(err, "failed to start controller")

‎cmd/extproc/mainlib/main.go‎

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,8 @@ type extProcFlags struct {
4545
metricsPort int // HTTP port for the metrics server.
4646
healthPort int // HTTP port for the health check server.
4747
metricsRequestHeaderLabels string // comma-separated key-value pairs for mapping HTTP request headers to Prometheus metric labels.
48-
// openAIPrefix is the OpenAI API prefix to be used for the external processor.
49-
openAIPrefix string
50-
// anthropicPrefix is the Anthropic API prefix to be used for the external processor.
51-
anthropicPrefix string
48+
// rootPrefix is the root prefix for all the processors.
49+
rootPrefix string
5250
}
5351

5452
// parseAndValidateFlags parses and validates the flags passed to the external processor.
@@ -82,17 +80,10 @@ func parseAndValidateFlags(args []string) (extProcFlags, error) {
8280
"",
8381
"Comma-separated key-value pairs for mapping HTTP request headers to Prometheus metric labels. Format: x-team-id:team_id,x-user-id:user_id.",
8482
)
85-
fs.StringVar(&flags.openAIPrefix,
86-
"openAIPrefix",
87-
"/v1",
88-
"OpenAI endpoint prefix to be used for the external processor. This is used to route requests to the correct handler. "+
89-
"Defaults to /v1, which is the standard OpenAI API prefix.",
90-
)
91-
fs.StringVar(&flags.anthropicPrefix,
92-
"anthropicPrefix",
93-
"/anthropic/v1",
94-
"Anthropic endpoint prefix to be used for the external processor. This is used to route requests to the correct handler. "+
95-
"Defaults to /anthropic/v1, which provides clear separation from OpenAI endpoints.",
83+
fs.StringVar(&flags.rootPrefix,
84+
"rootPrefix",
85+
"/",
86+
"The root path prefix for all the processors.",
9687
)
9788

9889
if err := fs.Parse(args); err != nil {
@@ -180,10 +171,10 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) {
180171
if err != nil {
181172
return fmt.Errorf("failed to create external processor server: %w", err)
182173
}
183-
server.Register(path.Join(flags.openAIPrefix, "/chat/completions"), extproc.ChatCompletionProcessorFactory(chatCompletionMetrics))
184-
server.Register(path.Join(flags.openAIPrefix, "/embeddings"), extproc.EmbeddingsProcessorFactory(embeddingsMetrics))
185-
server.Register(path.Join(flags.openAIPrefix, "/models"), extproc.NewModelsProcessor)
186-
server.Register(path.Join(flags.anthropicPrefix, "/messages"), extproc.MessagesProcessorFactory(chatCompletionMetrics))
174+
server.Register(path.Join(flags.rootPrefix, "/v1/chat/completions"), extproc.ChatCompletionProcessorFactory(chatCompletionMetrics))
175+
server.Register(path.Join(flags.rootPrefix, "/v1/embeddings"), extproc.EmbeddingsProcessorFactory(embeddingsMetrics))
176+
server.Register(path.Join(flags.rootPrefix, "/v1/models"), extproc.NewModelsProcessor)
177+
server.Register(path.Join(flags.rootPrefix, "/anthropic/v1/messages"), extproc.MessagesProcessorFactory(chatCompletionMetrics))
187178

188179
if err := extproc.StartConfigWatcher(ctx, flags.configPath, server, l, time.Second*5); err != nil {
189180
return fmt.Errorf("failed to start config watcher: %w", err)

‎cmd/extproc/mainlib/main_test.go‎

Lines changed: 46 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -33,76 +33,76 @@ import (
3333
func Test_parseAndValidateFlags(t *testing.T) {
3434
t.Run("ok extProcFlags", func(t *testing.T) {
3535
for _, tc := range []struct {
36-
name string
37-
args []string
38-
configPath string
39-
addr string
40-
openAIPRefix string
41-
logLevel slog.Level
36+
name string
37+
args []string
38+
configPath string
39+
addr string
40+
rootPrefix string
41+
logLevel slog.Level
4242
}{
4343
{
44-
name: "minimal extProcFlags",
45-
args: []string{"-configPath", "/path/to/config.yaml"},
46-
configPath: "/path/to/config.yaml",
47-
addr: ":1063",
48-
openAIPRefix: "/v1",
49-
logLevel: slog.LevelInfo,
44+
name: "minimal extProcFlags",
45+
args: []string{"-configPath", "/path/to/config.yaml"},
46+
configPath: "/path/to/config.yaml",
47+
addr: ":1063",
48+
rootPrefix: "/",
49+
logLevel: slog.LevelInfo,
5050
},
5151
{
52-
name: "custom addr",
53-
args: []string{"-configPath", "/path/to/config.yaml", "-extProcAddr", "unix:///tmp/ext_proc.sock"},
54-
configPath: "/path/to/config.yaml",
55-
addr: "unix:///tmp/ext_proc.sock",
56-
openAIPRefix: "/v1",
57-
logLevel: slog.LevelInfo,
52+
name: "custom addr",
53+
args: []string{"-configPath", "/path/to/config.yaml", "-extProcAddr", "unix:///tmp/ext_proc.sock"},
54+
configPath: "/path/to/config.yaml",
55+
addr: "unix:///tmp/ext_proc.sock",
56+
rootPrefix: "/",
57+
logLevel: slog.LevelInfo,
5858
},
5959
{
60-
name: "log level debug",
61-
args: []string{"-configPath", "/path/to/config.yaml", "-logLevel", "debug"},
62-
configPath: "/path/to/config.yaml",
63-
addr: ":1063",
64-
openAIPRefix: "/v1",
65-
logLevel: slog.LevelDebug,
60+
name: "log level debug",
61+
args: []string{"-configPath", "/path/to/config.yaml", "-logLevel", "debug"},
62+
configPath: "/path/to/config.yaml",
63+
addr: ":1063",
64+
rootPrefix: "/",
65+
logLevel: slog.LevelDebug,
6666
},
6767
{
68-
name: "log level warn",
69-
args: []string{"-configPath", "/path/to/config.yaml", "-logLevel", "warn"},
70-
configPath: "/path/to/config.yaml",
71-
addr: ":1063",
72-
openAIPRefix: "/v1",
73-
logLevel: slog.LevelWarn,
68+
name: "log level warn",
69+
args: []string{"-configPath", "/path/to/config.yaml", "-logLevel", "warn"},
70+
configPath: "/path/to/config.yaml",
71+
addr: ":1063",
72+
rootPrefix: "/",
73+
logLevel: slog.LevelWarn,
7474
},
7575
{
76-
name: "log level error",
77-
args: []string{"-configPath", "/path/to/config.yaml", "-logLevel", "error"},
78-
configPath: "/path/to/config.yaml",
79-
addr: ":1063",
80-
openAIPRefix: "/v1",
81-
logLevel: slog.LevelError,
76+
name: "log level error",
77+
args: []string{"-configPath", "/path/to/config.yaml", "-logLevel", "error"},
78+
configPath: "/path/to/config.yaml",
79+
addr: ":1063",
80+
rootPrefix: "/",
81+
logLevel: slog.LevelError,
8282
},
8383
{
8484
name: "all extProcFlags",
8585
args: []string{
8686
"-configPath", "/path/to/config.yaml",
8787
"-extProcAddr", "unix:///tmp/ext_proc.sock",
8888
"-logLevel", "debug",
89-
"-openAIPrefix", "/foo/bar/v1",
89+
"-rootPrefix", "/foo/bar/",
9090
},
91-
configPath: "/path/to/config.yaml",
92-
addr: "unix:///tmp/ext_proc.sock",
93-
openAIPRefix: "/foo/bar/v1",
94-
logLevel: slog.LevelDebug,
91+
configPath: "/path/to/config.yaml",
92+
addr: "unix:///tmp/ext_proc.sock",
93+
rootPrefix: "/foo/bar/",
94+
logLevel: slog.LevelDebug,
9595
},
9696
{
9797
name: "with header mapping",
9898
args: []string{
9999
"-configPath", "/path/to/config.yaml",
100100
"-metricsRequestHeaderLabels", "x-team-id:team_id,x-user-id:user_id",
101101
},
102-
configPath: "/path/to/config.yaml",
103-
openAIPRefix: "/v1",
104-
addr: ":1063",
105-
logLevel: slog.LevelInfo,
102+
configPath: "/path/to/config.yaml",
103+
rootPrefix: "/",
104+
addr: ":1063",
105+
logLevel: slog.LevelInfo,
106106
},
107107
} {
108108
t.Run(tc.name, func(t *testing.T) {
@@ -111,6 +111,7 @@ func Test_parseAndValidateFlags(t *testing.T) {
111111
assert.Equal(t, tc.configPath, flags.configPath)
112112
assert.Equal(t, tc.addr, flags.extProcAddr)
113113
assert.Equal(t, tc.logLevel, flags.logLevel)
114+
assert.Equal(t, tc.rootPrefix, flags.rootPrefix)
114115
})
115116
}
116117
})

‎internal/controller/controller.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ package controller
88
import (
99
"context"
1010
"fmt"
11-
"path"
1211

1312
egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
1413
"github.com/go-logr/logr"
@@ -74,8 +73,6 @@ type Options struct {
7473
MetricsRequestHeaderLabels string
7574
// RootPrefix is the root prefix for all the routes handled by the AI Gateway.
7675
RootPrefix string
77-
// OpenAIEndpointsPrefix is the prefix for OpenAI endpoints that follow after the root prefix.
78-
OpenAIPrefix string
7976
// ExtProcExtraEnvVars is the semicolon-separated key=value pairs for extra environment variables in extProc container.
8077
ExtProcExtraEnvVars string
8178
}
@@ -186,7 +183,7 @@ func StartControllers(ctx context.Context, mgr manager.Manager, config *rest.Con
186183
options.ExtProcLogLevel,
187184
options.UDSPath,
188185
options.MetricsRequestHeaderLabels,
189-
path.Join(options.RootPrefix, options.OpenAIPrefix),
186+
options.RootPrefix,
190187
options.ExtProcExtraEnvVars,
191188
))
192189
mgr.GetWebhookServer().Register("/mutate", &webhook.Admission{Handler: h})

‎internal/controller/gateway_mutator.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@ type gatewayMutator struct {
3737
extProcLogLevel string
3838
udsPath string
3939
metricsRequestHeaderLabels string
40-
openAIPrefix string
40+
rootPrefix string
4141
extProcExtraEnvVars []corev1.EnvVar
4242
}
4343

4444
func newGatewayMutator(c client.Client, kube kubernetes.Interface, logger logr.Logger,
4545
extProcImage string, extProcImagePullPolicy corev1.PullPolicy, extProcLogLevel,
46-
udsPath, metricsRequestHeaderLabels, openAIPrefix, extProcExtraEnvVars string,
46+
udsPath, metricsRequestHeaderLabels, rootPrefix, extProcExtraEnvVars string,
4747
) *gatewayMutator {
4848
var parsedEnvVars []corev1.EnvVar
4949
if extProcExtraEnvVars != "" {
@@ -63,7 +63,7 @@ func newGatewayMutator(c client.Client, kube kubernetes.Interface, logger logr.L
6363
logger: logger,
6464
udsPath: udsPath,
6565
metricsRequestHeaderLabels: metricsRequestHeaderLabels,
66-
openAIPrefix: openAIPrefix,
66+
rootPrefix: rootPrefix,
6767
extProcExtraEnvVars: parsedEnvVars,
6868
}
6969
}
@@ -95,7 +95,7 @@ func (g *gatewayMutator) buildExtProcArgs(filterConfigFullPath string, extProcMe
9595
"-extProcAddr", "unix://" + g.udsPath,
9696
"-metricsPort", fmt.Sprintf("%d", extProcMetricsPort),
9797
"-healthPort", fmt.Sprintf("%d", extProcHealthPort),
98-
"-openAIPrefix", g.openAIPrefix,
98+
"-rootPrefix", g.rootPrefix,
9999
}
100100

101101
// Add metrics header label mapping if configured.

‎internal/extproc/server.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ func (s *Server) LoadConfig(ctx context.Context, config *filterapi.Config) error
102102

103103
// Register a new processor for the given request path.
104104
func (s *Server) Register(path string, newProcessor ProcessorFactory) {
105+
s.logger.Info("Registering processor", slog.String("path", path))
105106
s.processorFactories[path] = newProcessor
106107
}
107108

‎manifests/charts/ai-gateway-helm/templates/deployment.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ spec:
5959
- --enableLeaderElection=true
6060
{{- end }}
6161
- --rootPrefix={{ .Values.endpointConfig.rootPrefix }}
62-
- --openAIPrefix={{ .Values.endpointConfig.openAIPrefix }}
6362
livenessProbe:
6463
grpc:
6564
port: 1063

‎manifests/charts/ai-gateway-helm/values.yaml‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,14 @@
66
# Default values for ai-gateway-helm.
77

88
# Global configuration for the endpoints supported by the AI Gateway.
9-
#
10-
# By default, the AI Gateway will assume that the downstream client's OpenAI SDK will talk to the Gateway using the base_url
11-
# set to "http://<gateway-hostname>/v1" which has the default "/v1" prefix in the base_url.
12-
#
13-
# By using this configuration, you can change the prefix for the OpenAI endpoints as well as the future non-OpenAI endpoints.
14-
# For example, when you can configure the rootPrefix to "/ai" and the openAIPrefix to "/openai/v1" which will result in the
15-
# OpenAI endpoints being served at "http://<gateway-hostname>/ai/openai/v1". This *will* become useful when you add support for
16-
# other input schemas like Anthropic, Google Gemini, etc. and you want to serve them under a different prefix to avoid conflicts.
17-
# Follow the issues https://github.com/envoyproxy/ai-gateway/issues/847 as well as https://github.com/envoyproxy/ai-gateway/issues/948 for detail.
189
endpointConfig:
1910
# The prefix for all the routes served by the AI Gateway. Defaulting to "/". All the generated routes will have this prefix.
11+
#
12+
# With the default "/", the AI Gateway will assume that the downstream client's OpenAI SDK will talk to the Gateway using the base_url
13+
# set to "http://<gateway-hostname>/v1" which has the default "/v1" prefix in the base_url.
14+
#
15+
# This can be used for providing a separation between AIGatewayRoutes and normal HTTPRoutes when the top level "/v1/" is not desired.
2016
rootPrefix: "/"
21-
# The prefix for the OpenAI endpoints. Defaulting to "/v1". This comes **after** the rootPrefix. E.g. if the rootPrefix is "/ai" and the openAIPrefix is "/v1",
22-
# the OpenAI endpoints will be served at "/ai/v1" which requires the base_url set to "http://<gateway-hostname>/ai/v1".
23-
openAIPrefix: "/v1"
24-
# TODO: addr more input schemas. E.g. Anthropic https://github.com/envoyproxy/ai-gateway/issues/847
2517

2618
extProc:
2719
image:

‎site/docs/capabilities/llm-integrations/supported-endpoints.md‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,6 @@ The Envoy AI Gateway provides OpenAI-compatible API endpoints for routing and ma
1010

1111
The Envoy AI Gateway acts as a proxy that accepts OpenAI-compatible requests and routes them to various AI providers. While it maintains compatibility with the OpenAI API specification, it currently supports a subset of the full OpenAI API.
1212

13-
:::tip
14-
`/v1` prefix on OpenAI API endpoints is configurable via Envoy AI Gateway installation options. The default is `/v1` unless specified otherwise.
15-
Please refer to the `endpointConfig` option in the [helm values file](https://github.com/envoyproxy/ai-gateway/blob/main/manifests/charts/ai-gateway-helm/values.yaml) for details.
16-
:::
17-
1813
## Supported Endpoints
1914

2015
### Chat Completions

0 commit comments

Comments
 (0)