Merge branch 'wso2:main' into fix-json-schema-guardrail-scenarios

Author: IsuruGunarathne

.github/workflows/gateway-integration-test-postgres.yml

@@ -0,0 +1,106 @@
+name: Gateway Integration Test (Postgres)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'gateway/**'
+      - '.github/workflows/gateway-integration-test-postgres.yml'
+
+jobs:
+  integration-test:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.25'
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build coverage-instrumented images
+        run: |
+          cd gateway
+          make build-coverage VERSION=test
+
+      - name: Build mock server images
+        run: |
+          for mock in mock-jwks mock-azure-content-safety mock-aws-bedrock-guardrail mock-embedding-provider mock-analytics-collector; do
+            echo "Building $mock..."
+            docker build -t ghcr.io/wso2/api-platform/$mock:latest tests/mock-servers/$mock
+          done
+
+      - name: Verify gateway-controller uses PostgreSQL
+        run: |
+          set -euo pipefail
+          cd gateway/it
+
+          PROJECT="gateway-it-postgres-verify"
+          cleanup() {
+            docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml down -v --remove-orphans || true
+          }
+          trap cleanup EXIT
+
+          docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml up -d postgres gateway-controller
+
+          timeout 90 bash -c 'until curl -fsS http://localhost:9090/health >/dev/null; do sleep 2; done'
+
+          docker logs it-gateway-controller > /tmp/gateway-controller.log 2>&1 || true
+          grep -Eq "Initializing PostgreSQL storage|PostgreSQL schema up to date" /tmp/gateway-controller.log
+
+          schema_count="$(docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml exec -T postgres \
+            psql -U gateway -d gateway_test -tAc "SELECT COUNT(*) FROM schema_migrations WHERE id = 1 AND version >= 1;")"
+          [ "$schema_count" = "1" ]
+
+          conn_count="$(docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml exec -T postgres \
+            psql -U gateway -d gateway_test -tAc "SELECT COUNT(*) FROM pg_stat_activity WHERE application_name = 'gateway-controller';")"
+          [ "${conn_count:-0}" -ge 1 ]
+
+      - name: Run integration tests
+        run: |
+          cd gateway
+          COMPOSE_FILE=docker-compose.test.postgres.yaml make test-integration
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: coverage-report-postgres
+          path: gateway/it/coverage/output
+          retention-days: 7
+
+      - name: Upload test reports
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-reports-postgres
+          path: gateway/it/reports/
+          retention-days: 7
+
+      - name: Debug on failure - Dump logs
+        if: failure()
+        run: |
+          echo "=== Docker Containers ==="
+          docker ps -a
+
+          echo ""
+          echo "=== gateway/it/logs Directory Contents ==="
+          if [ -d gateway/it/logs ]; then
+            if [ "$(ls -A gateway/it/logs)" ]; then
+              for f in gateway/it/logs/*; do
+                echo ""
+                echo "--- Contents of $f ---"
+                cat "$f"
+              done
+            else
+              echo "No log files found in gateway/it/logs."
+            fi
+          else
+            echo "Directory gateway/it/logs does not exist."
+          fi

.github/workflows/gateway-integration-test.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Build coverage-instrumented images
         run: |
           cd gateway
-          make build-coverage
+          make build-coverage VERSION=test
 
       - name: Build mock server images
         run: |

.gitignore

@@ -126,3 +126,4 @@ cli/src/tests/target/
 # Component specific patterns
 # Platform API
 platform-api/src/data
+gateway/gateway-runtime/target

.vscode/launch.json

@@ -41,7 +41,7 @@
                 "-manifest", "${workspaceFolder}/gateway/build.yaml",
                 // "-manifest", "${workspaceFolder}/gateway/sample-policies/build.yaml",
                 "-system-manifest-lock", "${workspaceFolder}/gateway/system-policies/system-policy-manifest-lock.yaml",
-                "-policy-engine-src", "${workspaceFolder}/gateway/policy-engine",
+                "-policy-engine-src", "${workspaceFolder}/gateway/gateway-runtime/policy-engine",
                 "-out-dir", "${workspaceFolder}/gateway/gateway-builder/target/output",
                 "-log-level", "debug",
             ],
@@ -51,7 +51,7 @@
             "type": "go",
             "request": "launch",
             "mode": "auto",
-            "program": "${workspaceFolder}/gateway/policy-engine/cmd/policy-engine",
+            "program": "${workspaceFolder}/gateway/gateway-runtime/policy-engine/cmd/policy-engine",
             "args": [
                 "-config", "${workspaceFolder}/gateway/configs/config.toml" ,
                 "-xds-server", "localhost:18001",
@@ -63,10 +63,10 @@
             "type": "go",
             "request": "launch",
             "mode": "auto",
-            "program": "${workspaceFolder}/gateway/policy-engine/cmd/policy-engine",
+            "program": "${workspaceFolder}/gateway/gateway-runtime/policy-engine/cmd/policy-engine",
             "args": [
                 "-config", "${workspaceFolder}/gateway/configs/config.toml",
-                "-policy-chains-file", "${workspaceFolder}/gateway/policy-engine/configs/policy-chains.yaml",
+                "-policy-chains-file", "${workspaceFolder}/gateway/gateway-runtime/policy-engine/configs/policy-chains.yaml",
             ],
         },
     ]

gateway/build.yaml

@@ -54,6 +54,8 @@ policies:
     gomodule: github.com/wso2/gateway-controllers/policies/regex-guardrail@v0
   - name: remove-headers
     gomodule: github.com/wso2/gateway-controllers/policies/remove-headers@v0
+  - name: request-rewrite
+    gomodule: github.com/wso2/gateway-controllers/policies/request-rewrite@v0
   - name: respond
     gomodule: github.com/wso2/gateway-controllers/policies/respond@v0
   - name: semantic-cache

gateway/configs/config-template.toml

@@ -27,12 +27,28 @@ cert_file = "./certs/server.crt"
 key_file = "./certs/server.key"
 
 [gateway_controller.storage]
-# Storage type: "sqlite", "postgres" (future), or "memory"
+# Storage type: "sqlite", "postgres", or "memory"
 type = "sqlite"
 
 [gateway_controller.storage.sqlite]
 path = "./data/gateway.db"
 
+[gateway_controller.storage.postgres]
+# Optional DSN. If set, host/port/database/user/password are ignored.
+# dsn = "postgres://user:password@localhost:5432/gateway?sslmode=require"
+host = "localhost"
+port = 5432
+database = "gateway"
+user = "gateway"
+password = ""
+sslmode = "require" # disable, require, verify-ca, verify-full
+connect_timeout = "5s"
+max_open_conns = 25
+max_idle_conns = 5
+conn_max_lifetime = "30m"
+conn_max_idle_time = "5m"
+application_name = "gateway-controller"
+
 [gateway_controller.policies]
 # Directory containing policy definitions
 definitions_path = "./default-policies"

gateway/gateway-controller/README.md

@@ -92,9 +92,22 @@ server:
 
 # Storage configuration
 storage:
-  type: sqlite            # "sqlite", "postgres" (future), or "memory"
+  type: sqlite            # "sqlite", "postgres", or "memory"
   sqlite:
     path: ./data/gateway.db  # SQLite database file path
+  postgres:               # Used when type=postgres
+    host: localhost
+    port: 5432
+    database: gateway
+    user: gateway
+    password: ""
+    sslmode: require      # disable, require, verify-ca, verify-full
+    connect_timeout: 5s
+    max_open_conns: 25
+    max_idle_conns: 5
+    conn_max_lifetime: 30m
+    conn_max_idle_time: 5m
+    application_name: gateway-controller
 
 # Router (Envoy) configuration
 router:
@@ -130,6 +143,16 @@ export APIP_GW_GATEWAY__CONTROLLER_STORAGE_TYPE=memory
 # Override SQLite database path
 export APIP_GW_GATEWAY__CONTROLLER_STORAGE_SQLITE_PATH=/custom/path/gateway.db
 
+# Configure PostgreSQL storage
+export APIP_GW_GATEWAY__CONTROLLER_STORAGE_TYPE=postgres
+export APIP_GW_GATEWAY__CONTROLLER_STORAGE_POSTGRES_HOST=postgres.example.internal
+export APIP_GW_GATEWAY__CONTROLLER_STORAGE_POSTGRES_PORT=5432
+export APIP_GW_GATEWAY__CONTROLLER_STORAGE_POSTGRES_DATABASE=gateway
+export APIP_GW_GATEWAY__CONTROLLER_STORAGE_POSTGRES_USER=gateway
+export APIP_GW_GATEWAY__CONTROLLER_STORAGE_POSTGRES_PASSWORD=secret
+export APIP_GW_GATEWAY__CONTROLLER_STORAGE_POSTGRES_SSLMODE=require
+export APIP_GW_GATEWAY__CONTROLLER_STORAGE_POSTGRES_MAX__OPEN__CONNS=25
+
 # Disable access logs
 export APIP_GW_GATEWAY__CONTROLLER_ROUTER_ACCESS__LOGS_ENABLED=false
 
@@ -153,6 +176,27 @@ storage:
     path: ./data/gateway.db
 ```
 
+#### Persistent Mode with PostgreSQL
+Use an external PostgreSQL server for persistence:
+
+```yaml
+storage:
+  type: postgres
+  postgres:
+    host: postgres.example.internal
+    port: 5432
+    database: gateway
+    user: gateway
+    password: ${DB_PASSWORD}
+    sslmode: require
+    connect_timeout: 5s
+    max_open_conns: 25
+    max_idle_conns: 5
+    conn_max_lifetime: 30m
+    conn_max_idle_time: 5m
+    application_name: gateway-controller
+```
+
 #### Memory-Only Mode
 No persistent storage (useful for testing):
 

gateway/gateway-controller/cmd/controller/main.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"log/slog"
 	"net/http"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
@@ -38,6 +40,29 @@ var (
 	BuildDate = "unknown"
 )
 
+func toBackendConfig(cfg *config.Config) storage.BackendConfig {
+	pg := cfg.GatewayController.Storage.Postgres
+	return storage.BackendConfig{
+		Type:       cfg.GatewayController.Storage.Type,
+		SQLitePath: cfg.GatewayController.Storage.SQLite.Path,
+		Postgres: storage.PostgresConnectionConfig{
+			DSN:             pg.DSN,
+			Host:            pg.Host,
+			Port:            pg.Port,
+			Database:        pg.Database,
+			User:            pg.User,
+			Password:        pg.Password,
+			SSLMode:         pg.SSLMode,
+			ConnectTimeout:  pg.ConnectTimeout,
+			MaxOpenConns:    pg.MaxOpenConns,
+			MaxIdleConns:    pg.MaxIdleConns,
+			ConnMaxLifetime: pg.ConnMaxLifetime,
+			ConnMaxIdleTime: pg.ConnMaxIdleTime,
+			ApplicationName: pg.ApplicationName,
+		},
+	}
+}
+
 func main() {
 	// Parse command-line flags
 	configPath := flag.String("config", "", "Path to configuration file (required)")
@@ -86,29 +111,20 @@ func main() {
 	// Initialize storage based on type
 	var db storage.Storage
 	if cfg.IsPersistentMode() {
-		switch cfg.GatewayController.Storage.Type {
-		case "sqlite":
-			log.Info("Initializing SQLite storage", slog.String("path", cfg.GatewayController.Storage.SQLite.Path))
-			db, err = storage.NewSQLiteStorage(cfg.GatewayController.Storage.SQLite.Path, log)
-			if err != nil {
-				// Check for database locked error and provide clear guidance
-				if err.Error() == "database is locked" || err.Error() == "failed to open database: database is locked" {
-					log.Error("Database is locked by another process",
-						slog.String("database_path", cfg.GatewayController.Storage.SQLite.Path),
-						slog.String("troubleshooting", "Check if another gateway-controller instance is running or remove stale WAL files"))
-					os.Exit(1)
-				}
-				log.Error("Failed to initialize SQLite database", slog.Any("error", err))
+		db, err = storage.NewStorage(toBackendConfig(cfg), log)
+		if err != nil {
+			if strings.EqualFold(cfg.GatewayController.Storage.Type, "sqlite") && errors.Is(err, storage.ErrDatabaseLocked) {
+				log.Error("Database is locked by another process",
+					slog.String("database_path", cfg.GatewayController.Storage.SQLite.Path),
+					slog.String("troubleshooting", "Check if another gateway-controller instance is running or remove stale WAL files"))
 				os.Exit(1)
 			}
-			defer db.Close()
-		case "postgres":
-			log.Error("PostgreSQL storage not yet implemented")
-			os.Exit(1)
-		default:
-			log.Error("Unknown storage type", slog.String("type", cfg.GatewayController.Storage.Type))
+			log.Error("Failed to initialize database storage",
+				slog.String("type", cfg.GatewayController.Storage.Type),
+				slog.Any("error", err))
 			os.Exit(1)
 		}
+		defer db.Close()
 	} else {
 		log.Info("Running in memory-only mode (no persistent storage)")
 	}