update

Signed-off-by: bitliu <[email protected]>

Author: Xunzhuo

.github/workflows/integration-test-k8s.yml

@@ -65,13 +65,10 @@ jobs:
         id: e2e-test
         run: |
           set +e  # Don't exit on error, we want to capture the result
-          make e2e-test PROFILE=ai-gateway
+          make e2e-test E2E_PROFILE=ai-gateway E2E_VERBOSE=true E2E_KEEP_CLUSTER=false
           TEST_EXIT_CODE=$?
           echo "test_exit_code=${TEST_EXIT_CODE}" >> $GITHUB_OUTPUT
           exit ${TEST_EXIT_CODE}
-        env:
-          E2E_VERBOSE: "true"
-          KEEP_CLUSTER: "true"
 
       - name: Upload test reports
         if: always()

e2e/README.md

@@ -84,30 +84,85 @@ make e2e-test
 ### Run specific profile
 
 ```bash
-make e2e-test PROFILE=ai-gateway
+make e2e-test E2E_PROFILE=ai-gateway
 ```
 
 ### Run specific test cases
 
 ```bash
-# Run only the progressive stress test
-./e2e/cmd/e2e/e2e --profile ai-gateway --test-cases chat-completions-progressive-stress --verbose
+# Run only specific test cases using make
+make e2e-test-specific E2E_TESTS="chat-completions-progressive-stress"
 
-# Run multiple specific test cases
-./e2e/cmd/e2e/e2e --profile ai-gateway --test-cases chat-completions-request,chat-completions-progressive-stress
+# Run multiple specific test cases using make
+make e2e-test-specific E2E_TESTS="chat-completions-request,chat-completions-progressive-stress"
+
+# Or run directly with the binary
+./bin/e2e -profile ai-gateway -tests chat-completions-progressive-stress -verbose
+
+# Run multiple specific test cases with the binary
+./bin/e2e -profile ai-gateway -tests "chat-completions-request,chat-completions-progressive-stress"
 ```
 
 ### Run with custom options
 
 ```bash
 # Keep cluster after test
-make e2e-test KEEP_CLUSTER=true
+make e2e-test E2E_KEEP_CLUSTER=true
 
 # Use existing cluster
-make e2e-test USE_EXISTING_CLUSTER=true
+make e2e-test E2E_USE_EXISTING_CLUSTER=true
+
+# Disable verbose output
+make e2e-test E2E_VERBOSE=false
+
+# Run tests in parallel
+make e2e-test E2E_PARALLEL=true
+
+# Combine multiple options
+make e2e-test E2E_PROFILE=ai-gateway E2E_KEEP_CLUSTER=true E2E_VERBOSE=true
+```
+
+### Debug mode
+
+```bash
+# Run tests with debug mode (keeps cluster and enables verbose logging)
+make e2e-test-debug
+```
+
+### Advanced Workflows
+
+#### Setup environment once, run tests multiple times
+
+This is useful for development and debugging:
+
+```bash
+# Step 1: Setup environment only (no tests)
+make e2e-setup
+
+# Step 2: Run all tests (skip setup)
+make e2e-test-only
+
+# Step 3: Run specific tests (skip setup)
+make e2e-test-only E2E_TESTS="chat-completions-request"
+
+# Step 4: Make code changes and re-run tests
+make e2e-test-only E2E_TESTS="domain-classify"
+
+# Step 5: Clean up when done
+make e2e-cleanup
+```
+
+#### Using binary directly
 
-# Verbose output
-make e2e-test VERBOSE=true
+```bash
+# Setup only
+./bin/e2e -profile ai-gateway -setup-only -keep-cluster -verbose
+
+# Run tests only (assumes environment is already deployed)
+./bin/e2e -profile ai-gateway -skip-setup -use-existing-cluster -verbose
+
+# Run specific tests only
+./bin/e2e -profile ai-gateway -skip-setup -use-existing-cluster -tests "chat-completions-request"
 ```
 
 ### Test Reports
@@ -116,9 +171,46 @@ After running tests, reports are generated:
 
 - `test-report.json`: Structured test results
 - `test-report.md`: Human-readable Markdown report
+- `semantic-router-logs.txt`: Complete semantic-router pod logs
 
 Each test case also prints detailed statistics to the console.
 
+## Environment Variables
+
+The following environment variables can be used to customize test execution:
+
+| Variable | Description | Default | Example |
+|----------|-------------|---------|---------|
+| `E2E_PROFILE` | Test profile to run | `ai-gateway` | `make e2e-test E2E_PROFILE=ai-gateway` |
+| `E2E_CLUSTER_NAME` | Kind cluster name | `semantic-router-e2e` | `make e2e-test E2E_CLUSTER_NAME=my-cluster` |
+| `E2E_IMAGE_TAG` | Docker image tag | `e2e-test` | `make e2e-test E2E_IMAGE_TAG=v1.0.0` |
+| `E2E_KEEP_CLUSTER` | Keep cluster after tests | `false` | `make e2e-test E2E_KEEP_CLUSTER=true` |
+| `E2E_USE_EXISTING_CLUSTER` | Use existing cluster | `false` | `make e2e-test E2E_USE_EXISTING_CLUSTER=true` |
+| `E2E_VERBOSE` | Enable verbose logging | `true` | `make e2e-test E2E_VERBOSE=false` |
+| `E2E_PARALLEL` | Run tests in parallel | `false` | `make e2e-test E2E_PARALLEL=true` |
+| `E2E_TESTS` | Specific test cases to run | (all tests) | `make e2e-test-specific E2E_TESTS="test1,test2"` |
+| `E2E_SETUP_ONLY` | Only setup profile without running tests | `false` | `make e2e-test E2E_SETUP_ONLY=true` |
+| `E2E_SKIP_SETUP` | Skip setup and only run tests | `false` | `make e2e-test E2E_SKIP_SETUP=true` |
+
+**Note**: When using the binary directly (`./bin/e2e`), use command-line flags instead:
+
+- `-profile` instead of `E2E_PROFILE`
+- `-cluster` instead of `E2E_CLUSTER_NAME`
+- `-image-tag` instead of `E2E_IMAGE_TAG`
+- `-keep-cluster` instead of `E2E_KEEP_CLUSTER`
+- `-use-existing-cluster` instead of `E2E_USE_EXISTING_CLUSTER`
+- `-verbose` instead of `E2E_VERBOSE`
+- `-parallel` instead of `E2E_PARALLEL`
+- `-tests` instead of `E2E_TESTS`
+- `-setup-only` instead of `E2E_SETUP_ONLY`
+- `-skip-setup` instead of `E2E_SKIP_SETUP`
+
+Example:
+
+```bash
+./bin/e2e -profile ai-gateway -keep-cluster -verbose -tests "chat-completions-request"
+```
+
 ## Adding New Test Profiles
 
 1. Create a new directory under `profiles/`

e2e/cmd/e2e/main.go

@@ -25,10 +25,18 @@ func main() {
 		verbose            = flag.Bool("verbose", false, "Enable verbose logging")
 		parallel           = flag.Bool("parallel", false, "Run tests in parallel")
 		testCases          = flag.String("tests", "", "Comma-separated list of test cases to run (empty means all)")
+		setupOnly          = flag.Bool("setup-only", false, "Only setup the profile without running tests")
+		skipSetup          = flag.Bool("skip-setup", false, "Skip profile setup and only run tests (assumes environment is already deployed)")
 	)
 
 	flag.Parse()
 
+	// Validate flags
+	if *setupOnly && *skipSetup {
+		fmt.Fprintf(os.Stderr, "Error: --setup-only and --skip-setup cannot be used together\n")
+		os.Exit(1)
+	}
+
 	// Parse test cases
 	var testCasesList []string
 	if *testCases != "" {
@@ -48,6 +56,8 @@ func main() {
 		Verbose:            *verbose,
 		Parallel:           *parallel,
 		TestCases:          testCasesList,
+		SetupOnly:          *setupOnly,
+		SkipSetup:          *skipSetup,
 	}
 
 	// Get the profile implementation

e2e/pkg/framework/runner.go

@@ -115,28 +115,42 @@ func (r *Runner) Run(ctx context.Context) error {
 	r.reporter.SetKubeClient(kubeClient)
 
 	// Step 4: Setup profile (deploy Helm charts, etc.)
-	setupOpts := &SetupOptions{
-		KubeClient:  kubeClient,
-		KubeConfig:  kubeConfig,
-		ClusterName: r.opts.ClusterName,
-		ImageTag:    r.opts.ImageTag,
-		Verbose:     r.opts.Verbose,
-	}
-
-	if err := r.profile.Setup(ctx, setupOpts); err != nil {
-		exitCode = 1
-		return fmt.Errorf("failed to setup profile: %w", err)
-	}
-
-	defer func() {
-		teardownOpts := &TeardownOptions{
+	if !r.opts.SkipSetup {
+		setupOpts := &SetupOptions{
 			KubeClient:  kubeClient,
 			KubeConfig:  kubeConfig,
 			ClusterName: r.opts.ClusterName,
+			ImageTag:    r.opts.ImageTag,
 			Verbose:     r.opts.Verbose,
 		}
-		r.profile.Teardown(context.Background(), teardownOpts)
-	}()
+
+		if err := r.profile.Setup(ctx, setupOpts); err != nil {
+			exitCode = 1
+			return fmt.Errorf("failed to setup profile: %w", err)
+		}
+
+		defer func() {
+			teardownOpts := &TeardownOptions{
+				KubeClient:  kubeClient,
+				KubeConfig:  kubeConfig,
+				ClusterName: r.opts.ClusterName,
+				Verbose:     r.opts.Verbose,
+			}
+			r.profile.Teardown(context.Background(), teardownOpts)
+		}()
+	} else {
+		r.log("⏭️  Skipping profile setup (--skip-setup enabled)")
+	}
+
+	// If setup-only mode, stop here
+	if r.opts.SetupOnly {
+		r.log("✅ Profile setup complete (--setup-only mode)")
+		r.log("💡 Cluster is ready. You can now:")
+		r.log("   - Run tests manually: ./bin/e2e -profile %s -skip-setup -use-existing-cluster", r.opts.Profile)
+		r.log("   - Inspect the cluster: kubectl --context kind-%s get pods -A", r.opts.ClusterName)
+		r.log("   - Clean up when done: make e2e-cleanup")
+		return nil
+	}
 
 	// Step 5: Run tests
 	results, err := r.runTests(ctx, kubeClient)

e2e/pkg/framework/types.go

@@ -96,6 +96,12 @@ type TestOptions struct {
 
 	// TestCases is a list of specific test cases to run (empty means all)
 	TestCases []string
+
+	// SetupOnly only sets up the profile without running tests
+	SetupOnly bool
+
+	// SkipSetup skips profile setup and only runs tests (assumes environment is already deployed)
+	SkipSetup bool
 }
 
 // TestResult represents the result of a test case

e2e/profiles/ai-gateway/profile.go

@@ -109,7 +109,6 @@ func (p *Profile) GetTestCases() []string {
 	return []string{
 		"chat-completions-request",
 		"chat-completions-stress-request",
-		"chat-completions-progressive-stress",
 		"domain-classify",
 		"semantic-cache",
 		"pii-detection",

e2e/testcases/chat_completions_stress_request.go

@@ -16,7 +16,7 @@ import (
 
 func init() {
 	pkgtestcases.Register("chat-completions-stress-request", pkgtestcases.TestCase{
-		Description: "Send 1000 sequential requests and measure success rate",
+		Description: "Send 200 sequential requests and measure success rate",
 		Tags:        []string{"llm", "stress", "reliability"},
 		Fn:          testStressTest,
 	})

src/semantic-router/pkg/utils/pii/policy.go

@@ -14,8 +14,18 @@ type PolicyChecker struct {
 }
 
 // IsPIIEnabled checks if PII detection is enabled and properly configured
+// For LoRA adapters, it falls back to the base model's PII policy if not found
 func (c *PolicyChecker) IsPIIEnabled(model string) bool {
 	modelConfig, exists := c.ModelConfigs[model]
+	if !exists {
+		// Try to find base model for LoRA adapters
+		baseModel := c.findBaseModelForLoRA(model)
+		if baseModel != "" {
+			logging.Infof("LoRA adapter '%s' not found in model configs, falling back to base model '%s'", model, baseModel)
+			modelConfig, exists = c.ModelConfigs[baseModel]
+		}
+	}
+
 	if !exists {
 		logging.Infof("No PII policy found for model %s, allowing request", model)
 		return false
@@ -33,13 +43,23 @@ func NewPolicyChecker(cfg *config.RouterConfig, modelConfigs map[string]config.M
 }
 
 // CheckPolicy checks if the detected PII types are allowed for the given model
+// For LoRA adapters, it falls back to the base model's PII policy if not found
 func (pc *PolicyChecker) CheckPolicy(model string, detectedPII []string) (bool, []string, error) {
 	if !pc.IsPIIEnabled(model) {
 		logging.Infof("PII detection is disabled, allowing request")
 		return true, nil, nil
 	}
 
 	modelConfig, exists := pc.ModelConfigs[model]
+	if !exists {
+		// Try to find base model for LoRA adapters
+		baseModel := pc.findBaseModelForLoRA(model)
+		if baseModel != "" {
+			logging.Infof("LoRA adapter '%s' not found in model configs, falling back to base model '%s' for PII policy", model, baseModel)
+			modelConfig, exists = pc.ModelConfigs[baseModel]
+		}
+	}
+
 	if !exists {
 		// If no specific config, allow by default
 		logging.Infof("No PII policy found for model %s, allowing request", model)
@@ -102,3 +122,17 @@ func ExtractAllContent(userContent string, nonUserMessages []string) []string {
 	allContent = append(allContent, nonUserMessages...)
 	return allContent
 }
+
+// findBaseModelForLoRA finds the base model for a given LoRA adapter name
+// Returns empty string if the LoRA adapter is not found in any model's LoRA list
+func (pc *PolicyChecker) findBaseModelForLoRA(loraName string) string {
+	for modelName, modelConfig := range pc.ModelConfigs {
+		for _, lora := range modelConfig.LoRAs {
+			if lora.Name == loraName {
+				logging.Debugf("Found base model '%s' for LoRA adapter '%s'", modelName, loraName)
+				return modelName
+			}
+		}
+	}
+	return ""
+}