mcp: make session encryption seed rotatable (#1357)

Author: mathetake

cmd/controller/main.go

@@ -55,9 +55,10 @@ type flags struct {
 	// extProcMaxRecvMsgSize is the maximum message size in bytes that the gRPC server can receive.
 	extProcMaxRecvMsgSize int
 	// maxRecvMsgSize is the maximum message size in bytes that the gRPC extension server can receive.
-	maxRecvMsgSize   int
-	watchNamespaces  []string
-	cacheSyncTimeout time.Duration
+	maxRecvMsgSize           int
+	mcpSessionEncryptionSeed string
+	watchNamespaces          []string
+	cacheSyncTimeout         time.Duration
 }
 
 // parsePullPolicy parses string into a k8s PullPolicy.
@@ -187,6 +188,13 @@ func parseAndValidateFlags(args []string) (flags, error) {
 		2*time.Minute, // This is the controller-runtime default
 		"Maximum time to wait for k8s caches to sync",
 	)
+	mcpSessionEncryptionSeed := fs.String(
+		"mcpSessionEncryptionSeed",
+		"seed",
+		"Arbitrary string seed used to derive the MCP session encryption key. "+
+			"Do not include commas as they are used as separators. You can optionally pass \"fallback\" seed after the first one to allow for key rotation. "+
+			"For example: \"new-seed,old-seed-for-fallback\". The fallback seed is only used for decryption.",
+	)
 
 	if err := fs.Parse(args); err != nil {
 		err = fmt.Errorf("failed to parse flags: %w", err)
@@ -268,6 +276,7 @@ func parseAndValidateFlags(args []string) (flags, error) {
 		maxRecvMsgSize:                 *maxRecvMsgSize,
 		watchNamespaces:                parseWatchNamespaces(*watchNamespaces),
 		cacheSyncTimeout:               *cacheSyncTimeout,
+		mcpSessionEncryptionSeed:       *mcpSessionEncryptionSeed,
 	}, nil
 }
 
@@ -355,6 +364,7 @@ func main() {
 		ExtProcExtraEnvVars:            parsedFlags.extProcExtraEnvVars,
 		ExtProcImagePullSecrets:        parsedFlags.extProcImagePullSecrets,
 		ExtProcMaxRecvMsgSize:          parsedFlags.extProcMaxRecvMsgSize,
+		MCPSessionEncryptionSeed:       parsedFlags.mcpSessionEncryptionSeed,
 	}); err != nil {
 		setupLog.Error(err, "failed to start controller")
 	}

cmd/extproc/mainlib/main.go

@@ -105,7 +105,9 @@ func parseAndValidateFlags(args []string) (extProcFlags, error) {
 	fs.StringVar(&flags.mcpSessionEncryptionSeed,
 		"mcpSessionEncryptionSeed",
 		"mcp",
-		"seed used to derive the MCP session encryption key. This should be set to a secure value in production.",
+		"Arbitrary string seed used to derive the MCP session encryption key. "+
+			"Do not include commas as they are used as separators. You can optionally pass \"fallback\" seed after the first one to allow for key rotation. "+
+			"For example: \"new-seed,old-seed-for-fallback\". The fallback seed is only used for decryption.",
 	)
 	fs.DurationVar(&flags.mcpWriteTimeout, "mcpWriteTimeout", 120*time.Second,
 		"The maximum duration before timing out writes of the MCP response")
@@ -259,7 +261,8 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) {
 
 	var mcpServer *http.Server
 	if mcpLis != nil {
-		mcpSessionCrypto := mcpproxy.DefaultSessionCrypto(flags.mcpSessionEncryptionSeed)
+		seed, fallbackSeed, _ := strings.Cut(flags.mcpSessionEncryptionSeed, ",")
+		mcpSessionCrypto := mcpproxy.DefaultSessionCrypto(seed, fallbackSeed)
 		var mcpProxyMux *http.ServeMux
 		var mcpProxy *mcpproxy.MCPProxy
 		mcpProxy, mcpProxyMux, err = mcpproxy.NewMCPProxy(l.With("component", "mcp-proxy"), mcpMetrics,

internal/controller/controller.go

@@ -85,6 +85,8 @@ type Options struct {
 	ExtProcImagePullSecrets string
 	// ExtProcMaxRecvMsgSize is the maximum message size in bytes that the gRPC server can receive for extProc.
 	ExtProcMaxRecvMsgSize int
+	// MCPSessionEncryptionSeed is the seed used to derive the encryption key for MCP session encryption.
+	MCPSessionEncryptionSeed string
 }
 
 // StartControllers starts the controllers for the AI Gateway.
@@ -215,6 +217,7 @@ func StartControllers(ctx context.Context, mgr manager.Manager, config *rest.Con
 			options.ExtProcImagePullSecrets,
 			options.ExtProcMaxRecvMsgSize,
 			isKubernetes133OrLater(versionInfo, logger),
+			options.MCPSessionEncryptionSeed,
 		))
 		mgr.GetWebhookServer().Register("/mutate", &webhook.Admission{Handler: h})
 	}

internal/controller/gateway_mutator.go

@@ -45,6 +45,9 @@ type gatewayMutator struct {
 	extProcImagePullSecrets        []corev1.LocalObjectReference
 	extProcMaxRecvMsgSize          int
 
+	// mcpSessionEncryptionSeed is the seed used to derive the encryption key for MCP session data.
+	mcpSessionEncryptionSeed string
+
 	// Whether to run the extProc container as a sidecar (true) as a normal container (false).
 	// This is essentially a workaround for old k8s versions, and we can remove this in the future.
 	extProcAsSideCar bool
@@ -54,6 +57,7 @@ func newGatewayMutator(c client.Client, kube kubernetes.Interface, logger logr.L
 	extProcImage string, extProcImagePullPolicy corev1.PullPolicy, extProcLogLevel,
 	udsPath, metricsRequestHeaderAttributes, spanRequestHeaderAttributes, rootPrefix, extProcExtraEnvVars, extProcImagePullSecrets string, extProcMaxRecvMsgSize int,
 	extProcAsSideCar bool,
+	mcpSessionEncryptionSeed string,
 ) *gatewayMutator {
 	var parsedEnvVars []corev1.EnvVar
 	if extProcExtraEnvVars != "" {
@@ -90,6 +94,7 @@ func newGatewayMutator(c client.Client, kube kubernetes.Interface, logger logr.L
 		extProcImagePullSecrets:        parsedImagePullSecrets,
 		extProcMaxRecvMsgSize:          extProcMaxRecvMsgSize,
 		extProcAsSideCar:               extProcAsSideCar,
+		mcpSessionEncryptionSeed:       mcpSessionEncryptionSeed,
 	}
 }
 
@@ -123,7 +128,8 @@ func (g *gatewayMutator) buildExtProcArgs(filterConfigFullPath string, extProcAd
 		"-maxRecvMsgSize", fmt.Sprintf("%d", g.extProcMaxRecvMsgSize),
 	}
 	if needMCP {
-		args = append(args, "-mcpAddr", ":"+strconv.Itoa(internalapi.MCPProxyPort))
+		args = append(args, "-mcpAddr", ":"+strconv.Itoa(internalapi.MCPProxyPort),
+			"-mcpSessionEncryptionSeed", g.mcpSessionEncryptionSeed)
 	}
 
 	// Add metrics header label mapping if configured.

internal/controller/gateway_mutator_test.go

@@ -7,6 +7,7 @@ package controller
 
 import (
 	"fmt"
+	"strconv"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -21,6 +22,7 @@ import (
 	gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 
 	aigv1a1 "github.com/envoyproxy/ai-gateway/api/v1alpha1"
+	"github.com/envoyproxy/ai-gateway/internal/internalapi"
 )
 
 func TestGatewayMutator_Default(t *testing.T) {
@@ -51,6 +53,7 @@ func TestGatewayMutator_mutatePod(t *testing.T) {
 		extProcImagePullSecrets        string
 		extprocTest                    func(t *testing.T, container corev1.Container)
 		podTest                        func(t *testing.T, pod corev1.Pod)
+		needMCP                        bool
 	}{
 		{
 			name: "basic extproc container",
@@ -61,6 +64,25 @@ func TestGatewayMutator_mutatePod(t *testing.T) {
 				require.Empty(t, pod.Spec.ImagePullSecrets)
 			},
 		},
+		{
+			name:    "basic extproc container with MCPRoute",
+			needMCP: true,
+			extprocTest: func(t *testing.T, container corev1.Container) {
+				var foundMCPAddr, foundMCPSeed bool
+				for i, arg := range container.Args {
+					switch arg {
+					case "-mcpAddr":
+						foundMCPAddr = true
+						require.Equal(t, ":"+strconv.Itoa(internalapi.MCPProxyPort), container.Args[i+1])
+					case "-mcpSessionEncryptionSeed":
+						foundMCPSeed = true
+						require.Equal(t, "seed", container.Args[i+1])
+					}
+				}
+				require.True(t, foundMCPAddr)
+				require.True(t, foundMCPSeed)
+			},
+		},
 		{
 			name:                "with extra env vars",
 			extProcExtraEnvVars: "OTEL_SERVICE_NAME=ai-gateway-extproc;OTEL_TRACES_EXPORTER=otlp",
@@ -190,6 +212,22 @@ func TestGatewayMutator_mutatePod(t *testing.T) {
 					})
 					require.NoError(t, err)
 
+					if tt.needMCP {
+						err = fakeClient.Create(t.Context(), &aigv1a1.MCPRoute{
+							ObjectMeta: metav1.ObjectMeta{Name: "test-mcp", Namespace: gwNamespace},
+							Spec: aigv1a1.MCPRouteSpec{
+								ParentRefs: []gwapiv1a2.ParentReference{
+									{
+										Name:  gwName,
+										Kind:  ptr.To(gwapiv1a2.Kind("Gateway")),
+										Group: ptr.To(gwapiv1a2.Group("gateway.networking.k8s.io")),
+									},
+								},
+							},
+						})
+						require.NoError(t, err)
+					}
+
 					pod := &corev1.Pod{
 						ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "test-namespace"},
 						Spec: corev1.PodSpec{
@@ -227,7 +265,9 @@ func TestGatewayMutator_mutatePod(t *testing.T) {
 
 					require.Equal(t, "ai-gateway-extproc", extProcContainer.Name)
 					tt.extprocTest(t, extProcContainer)
-					tt.podTest(t, *pod)
+					if tt.podTest != nil {
+						tt.podTest(t, *pod)
+					}
 				})
 			}
 		})
@@ -239,7 +279,7 @@ func newTestGatewayMutator(fakeClient client.Client, fakeKube *fake2.Clientset,
 	return newGatewayMutator(
 		fakeClient, fakeKube, ctrl.Log, "docker.io/envoyproxy/ai-gateway-extproc:latest", corev1.PullIfNotPresent,
 		"info", "/tmp/extproc.sock", metricsRequestHeaderAttributes, spanRequestHeaderAttributes, "/v1", extProcExtraEnvVars, extProcImagePullSecrets, 512*1024*1024,
-		sidecar,
+		sidecar, "seed",
 	)
 }
 

internal/mcpproxy/crypto.go

@@ -26,13 +26,25 @@ type SessionCrypto interface {
 }
 
 // DefaultSessionCrypto returns a SessionCrypto implementation using PBKDF2 for key derivation and AES-GCM for encryption.
-func DefaultSessionCrypto(seed string) SessionCrypto {
-	return pbkdf2AesGcm{
+func DefaultSessionCrypto(seed, fallbackSeed string) SessionCrypto {
+	primary := &pbkdf2AesGcm{
 		seed:       seed,    // Seed used to derive the encryption key.
 		saltSize:   16,      // Salt size for PBKDF2.
 		keyLength:  32,      // Key length for AES-256.
 		iterations: 100_000, // Number of PBKDF2 iterations (trade security vs performance).
 	}
+	if fallbackSeed == "" {
+		return primary
+	}
+	return &fallbackEnabledSessionCrypto{
+		primary: primary,
+		fallback: &pbkdf2AesGcm{
+			seed:       fallbackSeed,
+			saltSize:   16,
+			keyLength:  32,
+			iterations: 100_000,
+		},
+	}
 }
 
 // pbkdf2AesGcm implements SessionCrypto using PBKDF2 for key derivation and AES-GCM for encryption.
@@ -119,3 +131,26 @@ func (p pbkdf2AesGcm) Decrypt(encrypted string) (string, error) {
 	}
 	return string(plaintext), nil
 }
+
+// fallbackEnabledSessionCrypto tries to decrypt using the primary SessionCrypto first for decryption.
+// If that fails and a fallback SessionCrypto is provided, it tries to decrypt using the fallback.
+type fallbackEnabledSessionCrypto struct {
+	primary, fallback SessionCrypto
+}
+
+// Encrypt always uses the primary SessionCrypto.
+func (f fallbackEnabledSessionCrypto) Encrypt(plaintext string) (string, error) {
+	return f.primary.Encrypt(plaintext)
+}
+
+// Decrypt tries the primary SessionCrypto first, and if that fails and a fallback is provided, it tries the fallback.
+func (f fallbackEnabledSessionCrypto) Decrypt(encrypted string) (string, error) {
+	plaintext, err := f.primary.Decrypt(encrypted)
+	if err == nil {
+		return plaintext, nil
+	}
+	if f.fallback != nil {
+		return f.fallback.Decrypt(encrypted)
+	}
+	return "", err
+}

internal/mcpproxy/crypto_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestSessionEncryption(t *testing.T) {
-	sc := DefaultSessionCrypto("test")
+	sc := DefaultSessionCrypto("test", "")
 
 	enc, err := sc.Encrypt("plaintext")
 	require.NoError(t, err)
@@ -23,7 +23,7 @@ func TestSessionEncryption(t *testing.T) {
 }
 
 func TestEncryptionIsSalted(t *testing.T) {
-	sc := DefaultSessionCrypto("test")
+	sc := DefaultSessionCrypto("test", "")
 
 	enc1, err := sc.Encrypt("plaintext")
 	require.NoError(t, err)
@@ -34,8 +34,8 @@ func TestEncryptionIsSalted(t *testing.T) {
 }
 
 func TestDecryptWrongSeed(t *testing.T) {
-	sc1 := DefaultSessionCrypto("test1")
-	sc2 := DefaultSessionCrypto("test2")
+	sc1 := DefaultSessionCrypto("test1", "")
+	sc2 := DefaultSessionCrypto("test2", "")
 
 	enc, err := sc1.Encrypt("plaintext")
 	require.NoError(t, err)
@@ -45,9 +45,34 @@ func TestDecryptWrongSeed(t *testing.T) {
 	require.Empty(t, dec)
 }
 
+func TestDecryptFallbackSeed(t *testing.T) {
+	sc1 := DefaultSessionCrypto("test1", "")
+	sc2 := DefaultSessionCrypto("test2", "test1")
+
+	// Decrypting should work with the fallback seed.
+	enc, err := sc1.Encrypt("plaintext")
+	require.NoError(t, err)
+	dec, err := sc2.Decrypt(enc)
+	require.NoError(t, err)
+	require.Equal(t, "plaintext", dec)
+
+	// Encrypting should happen with the latest seed.
+	enc2, err := sc2.Encrypt("plaintext2")
+	require.NoError(t, err)
+	require.NotEqual(t, enc, enc2)
+
+	dec2, err := sc1.Decrypt(enc2)
+	require.Error(t, err)
+	require.Empty(t, dec2)
+
+	dec2, err = sc2.Decrypt(enc2)
+	require.NoError(t, err)
+	require.Equal(t, "plaintext2", dec2)
+}
+
 func TestDecryptDifferentInstancesSameSeed(t *testing.T) {
-	sc1 := DefaultSessionCrypto("test")
-	sc2 := DefaultSessionCrypto("test")
+	sc1 := DefaultSessionCrypto("test", "")
+	sc2 := DefaultSessionCrypto("test", "")
 
 	enc, err := sc1.Encrypt("plaintext")
 	require.NoError(t, err)

internal/mcpproxy/handlers_test.go

@@ -42,7 +42,7 @@ func newTestMCPProxy() *MCPProxy {
 
 func newTestMCPProxyWithTracer(t tracingapi.MCPTracer) *MCPProxy {
 	return &MCPProxy{
-		sessionCrypto: DefaultSessionCrypto("test"),
+		sessionCrypto: DefaultSessionCrypto("test", ""),
 		mcpProxyConfig: &mcpProxyConfig{
 			backendListenerAddr: "http://test-backend",
 			routes: map[filterapi.MCPRouteName]*mcpProxyConfigRoute{

internal/mcpproxy/mcpproxy_test.go

@@ -61,7 +61,7 @@ var noopTracer = tracing.NoopMCPTracer{}
 
 func TestNewMCPProxy(t *testing.T) {
 	l := slog.Default()
-	proxy, mux, err := NewMCPProxy(l, stubMetrics{}, noopTracer, DefaultSessionCrypto("test"))
+	proxy, mux, err := NewMCPProxy(l, stubMetrics{}, noopTracer, DefaultSessionCrypto("test", ""))
 
 	require.NoError(t, err)
 	require.NotNil(t, proxy)
@@ -72,7 +72,7 @@ func TestNewMCPProxy(t *testing.T) {
 
 func TestMCPProxy_HTTPMethods(t *testing.T) {
 	l := slog.Default()
-	_, mux, err := NewMCPProxy(l, stubMetrics{}, noopTracer, DefaultSessionCrypto("test"))
+	_, mux, err := NewMCPProxy(l, stubMetrics{}, noopTracer, DefaultSessionCrypto("test", ""))
 	require.NoError(t, err)
 
 	// Test unsupported method.

manifests/charts/ai-gateway-helm/values.yaml

@@ -165,8 +165,16 @@ controller:
   nodeSelector: {}
   tolerations: []
   affinity: {}
+
   # maxRecvMsgSize is the maximum message size in bytes that the gRPC extension server can receive
   # from xDS (envoy-gateway).
   # This value should be increased in setups where count/complexity of xDS
   # resources (configuration) is big. Defaults to 4MB.
   # maxRecvMsgSize: "4194304"
+
+  # mcpSessionEncryptionSeed is an arbitrary string seed used to derive the MCP session encryption key.
+  #	Do not include commas as they are used as separators. You can optionally pass "fallback" seed after the first one to allow for key rotation.
+  #	For example: "new-seed,old-seed-for-fallback". The fallback seed is only used for decryption.
+  #
+  # This value should be set to a secure random string in production environments instead of the default below.
+  mcpSessionEncryptionSeed: "default-insecure-seed"