CLI Inegration Test

cli/it/Makefile

@@ -0,0 +1,78 @@
+# --------------------------------------------------------------------
+# Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com).
+#
+# WSO2 LLC. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# --------------------------------------------------------------------
+
+.PHONY: test clean deps check-docker build-images help
+
+# Gateway IT compose file location
+GATEWAY_COMPOSE_FILE=../../gateway/it/docker-compose.test.yaml
+GATEWAY_DIR=../../gateway
+
+# Default target
+help:
+	@echo "CLI Integration Tests"
+	@echo ""
+	@echo "Usage:"
+	@echo "  make test           - Run integration tests (requires images to be built first)"
+	@echo "  make test-all       - Build gateway images and run integration tests"
+	@echo "  make build-images   - Build required gateway Docker images"
+	@echo "  make clean          - Clean up containers, volumes, and logs"
+	@echo "  make deps           - Install Go dependencies"
+	@echo ""
+	@echo "Prerequisites:"
+	@echo "  - Docker must be running"
+	@echo "  - Gateway images must be built (run 'make build-images' first)"
+
+# Run integration tests (assumes images are already built)
+test: check-docker
+	@mkdir -p logs
+	@pkgs="$$(go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./... | xargs)" ; \
+	if [ -z "$$pkgs" ]; then \
+		echo "No packages with tests found under $(PWD)" ; \
+	else \
+		go test -v -timeout 30m $$pkgs 2>&1 | awk '/^Logs directory:/{print;exit}1' ; \
+	fi
+
+# Build images and run tests (full flow)
+test-all: build-images test
+
+# Build required gateway Docker images locally
+build-images: check-docker
+	@echo "==============================================="
+	@echo "Building Gateway Docker Images (with coverage)"
+	@echo "==============================================="
+	@echo "This may take several minutes..."
+	@$(MAKE) -C $(GATEWAY_DIR) build-coverage
+	@echo "✓ Gateway images built successfully"
+
+# Clean up containers, volumes, and logs
+clean:
+	@echo "Cleaning up..."
+	@docker compose -f $(GATEWAY_COMPOSE_FILE) -p cli-it down -v --remove-orphans 2>/dev/null || true
+	@docker rm -f $$(docker ps -aq --filter "name=cli-it-") 2>/dev/null || true
+	@rm -rf logs/*.log
+	@echo "Clean complete"
+
+# Install dependencies
+deps:
+	go mod download
+	go mod tidy
+
+# Verify Docker is available
+check-docker:
+	@docker info > /dev/null 2>&1 || (echo "\033[31m✗ Error: Docker is not running. Please start Docker and try again.\033[0m" && exit 1)
+	@echo "\033[32m✓ Docker is available\033[0m"

cli/it/README.md

@@ -0,0 +1,83 @@
+# CLI Integration Tests
+
+End-to-end integration tests for the `ap` CLI, validating commands against real gateway and MCP server infrastructure.
+
+## Prerequisites
+
+- Go 1.25.1 or later
+- Docker and Docker Compose
+- Network access to pull Docker images
+
+Important notes:
+
+- Docker Compose: the test runner uses the `docker compose up --wait` flag
+  which requires Docker Compose v2 (the `docker compose` plugin). Ensure
+  your Docker installation provides Compose v2+; older `docker-compose`
+  binaries without the `--wait` flag will not work.
+
+- Configuration backup: the integration suite will temporarily back up your
+  real CLI config file (`~/.wso2ap/config.yaml`) to
+  `~/.wso2ap/config.yaml.backup` and write a clean config during test
+  execution. The original config will be restored after the suite finishes
+  (whether tests pass or fail). If you rely on a custom local config,
+  please be aware of this behavior.
+
+## Quick Start
+
+```bash
+# Run all integration tests
+make test
+
+# Clean up containers and logs
+make clean
+
+# Install dependencies
+make deps
+```
+
+## Configuration
+
+Tests can be enabled/disabled by editing `test-config.yaml`:
+
+```yaml
+tests:
+  gateway:
+    manage:
+      - id: GW-MANAGE-001
+        name: gateway add with valid parameters
+        enabled: true  # Set to false to skip
+        requires: [CLI, GATEWAY]
+```
+
+## Test Structure
+
+```
+cli/it/
+├── test-config.yaml      # Enable/disable tests
+├── features/             # Gherkin feature files
+│   └── gateway/
+├── steps/                # Step definitions
+├── resources/            # Test resources
+│   └── gateway/
+└── logs/                 # Test logs (git-ignored)
+```
+
+## Infrastructure Dependencies
+
+| ID | Component | Description |
+|----|-----------|-------------|
+| `CLI` | CLI Binary | The `ap` binary built from `cli/src/` |
+| `GATEWAY` | Gateway Stack | Docker Compose services: controller, router, policy-engine |
+| `MCP_SERVER` | MCP Server | MCP backend for generate command tests |
+
+## Logs
+
+Each test writes to a separate log file in `logs/`:
+- `logs/GW-MANAGE-001-gateway-add.log`
+- `logs/GW-API-001-api-list.log`
+- `logs/PHASE-1.log` (Phase 1 infrastructure setup)
+- etc.
+
+## Documentation
+
+See [INTEGRATION-TESTS.md](INTEGRATION-TESTS.md) for detailed documentation.

cli/it/config.go

@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com).
+ *
+ * WSO2 LLC. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package it
+
+import (
+	"os"
+	"sort"
+
+	"gopkg.in/yaml.v3"
+)
+
+// TestConfig represents the test configuration file structure
+type TestConfig struct {
+	Infrastructure InfrastructureConfig `yaml:"infrastructure"`
+	Tests          TestsConfig          `yaml:"tests"`
+}
+
+// InfrastructureConfig holds infrastructure-related configuration
+type InfrastructureConfig struct {
+	ComposeFile         string `yaml:"compose_file"`
+	StartupTimeout      string `yaml:"startup_timeout"`
+	HealthCheckInterval string `yaml:"health_check_interval"`
+}
+
+// TestsConfig holds all test group configurations
+type TestsConfig struct {
+	Gateway GatewayTestsConfig `yaml:"gateway"`
+}
+
+// GatewayTestsConfig holds gateway-related test configurations
+type GatewayTestsConfig struct {
+	Manage []TestDefinition `yaml:"manage"`
+	Apply  []TestDefinition `yaml:"apply"`
+	API    []TestDefinition `yaml:"api"`
+	MCP    []TestDefinition `yaml:"mcp"`
+	Build  []TestDefinition `yaml:"build"`
+}
+
+// TestDefinition represents a single test definition
+type TestDefinition struct {
+	ID       string   `yaml:"id"`
+	Name     string   `yaml:"name"`
+	Enabled  bool     `yaml:"enabled"`
+	Requires []string `yaml:"requires"`
+}
+
+// InfrastructureID represents infrastructure component identifiers
+type InfrastructureID string
+
+const (
+	// InfraCLI represents the CLI binary
+	InfraCLI InfrastructureID = "CLI"
+	// InfraGatewayImages represents the gateway Docker images (build step)
+	InfraGatewayImages InfrastructureID = "GATEWAY_IMAGES"
+	// InfraGateway represents the gateway stack (running containers)
+	InfraGateway InfrastructureID = "GATEWAY"
+	// InfraMCPServer represents the MCP server
+	InfraMCPServer InfrastructureID = "MCP_SERVER"
+)
+
+// LoadTestConfig loads the test configuration from a YAML file
+func LoadTestConfig(path string) (*TestConfig, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var config TestConfig
+	if err := yaml.Unmarshal(data, &config); err != nil {
+		return nil, err
+	}
+
+	return &config, nil
+}
+
+// GetEnabledTests returns all enabled test definitions
+func (c *TestConfig) GetEnabledTests() []TestDefinition {
+	var enabled []TestDefinition
+
+	// Collect from all test groups
+	for _, t := range c.Tests.Gateway.Manage {
+		if t.Enabled {
+			enabled = append(enabled, t)
+		}
+	}
+	for _, t := range c.Tests.Gateway.Apply {
+		if t.Enabled {
+			enabled = append(enabled, t)
+		}
+	}
+	for _, t := range c.Tests.Gateway.API {
+		if t.Enabled {
+			enabled = append(enabled, t)
+		}
+	}
+	for _, t := range c.Tests.Gateway.MCP {
+		if t.Enabled {
+			enabled = append(enabled, t)
+		}
+	}
+	for _, t := range c.Tests.Gateway.Build {
+		if t.Enabled {
+			enabled = append(enabled, t)
+		}
+	}
+
+	return enabled
+}
+
+// GetAllTests returns all test definitions regardless of enabled status
+func (c *TestConfig) GetAllTests() []TestDefinition {
+	var all []TestDefinition
+
+	all = append(all, c.Tests.Gateway.Manage...)
+	all = append(all, c.Tests.Gateway.Apply...)
+	all = append(all, c.Tests.Gateway.API...)
+	all = append(all, c.Tests.Gateway.MCP...)
+	all = append(all, c.Tests.Gateway.Build...)
+
+	return all
+}
+
+// GetRequiredInfrastructure returns unique infrastructure IDs required by enabled tests
+func (c *TestConfig) GetRequiredInfrastructure() []InfrastructureID {
+	required := make(map[InfrastructureID]bool)
+
+	for _, test := range c.GetEnabledTests() {
+		for _, req := range test.Requires {
+			required[InfrastructureID(req)] = true
+		}
+	}
+
+	order := []InfrastructureID{
+		InfraCLI,
+		InfraGatewayImages,
+		InfraGateway,
+		InfraMCPServer,
+	}
+
+	var result []InfrastructureID
+	for _, id := range order {
+		if required[id] {
+			result = append(result, id)
+			delete(required, id)
+		}
+	}
+
+	// Append any remaining IDs deterministically (sorted by string).
+	if len(required) > 0 {
+		var others []InfrastructureID
+		for id := range required {
+			others = append(others, id)
+		}
+		sort.Slice(others, func(i, j int) bool { return string(others[i]) < string(others[j]) })
+		result = append(result, others...)
+	}
+
+	return result
+}
+
+// IsTestEnabled checks if a test with the given ID is enabled
+func (c *TestConfig) IsTestEnabled(testID string) bool {
+	for _, test := range c.GetAllTests() {
+		if test.ID == testID {
+			return test.Enabled
+		}
+	}
+	return false
+}
+
+// GetTestByID returns the test definition for a given ID
+func (c *TestConfig) GetTestByID(testID string) *TestDefinition {
+	for _, test := range c.GetAllTests() {
+		if test.ID == testID {
+			return &test
+		}
+	}
+	return nil
+}