From 33cbf09a4d83cb0d5821beb781ce2fe287db19c0 Mon Sep 17 00:00:00 2001 From: Qindeel Ishtiaq Date: Mon, 3 Nov 2025 15:41:10 -0500 Subject: [PATCH 1/2] console testing for different versions --- .../materialize/console_version_matrix.py | 66 +++++ test/console/QUICKSTART.md | 256 ++++++++++++++++++ test/console/README.md | 99 ++++++- test/console/ci-examples/README.md | 236 ++++++++++++++++ test/console/mzcompose.py | 99 ++++++- test/console/resolve_version.py | 68 +++++ 6 files changed, 820 insertions(+), 4 deletions(-) create mode 100644 misc/python/materialize/console_version_matrix.py create mode 100644 test/console/QUICKSTART.md create mode 100644 test/console/ci-examples/README.md create mode 100755 test/console/resolve_version.py diff --git a/misc/python/materialize/console_version_matrix.py b/misc/python/materialize/console_version_matrix.py new file mode 100644 index 0000000000000..84d585dc1d570 --- /dev/null +++ b/misc/python/materialize/console_version_matrix.py @@ -0,0 +1,66 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +""" +Version matrix for console testing - uses existing version_list infrastructure. +""" + +from materialize.mz_version import MzVersion +from materialize.version_list import ( + get_all_mz_versions, + get_self_managed_versions, +) + + +def get_console_test_versions() -> dict[str, MzVersion | None]: + """ + Get all versions that should be tested for console. + + Uses existing version_list.py infrastructure to determine: + - cloud-backward: Last released minor version (from git tags) + - cloud-current: Current source (None) + - cloud-forward: Current source (None) + - sm-lts: Latest self-managed LTS minor version + + Returns: + Dictionary mapping test name to version (None = current source) + + Note: This function does NOT verify Docker images exist, for speed. + The assumption is that released versions have published images. + """ + # Get all versions from git tags (fast, no Docker check) + all_versions = get_all_mz_versions(newest_first=True) + + # Debug: Print last 5 versions from git tags + print(f"DEBUG: Last 5 released versions: {[str(v) for v in all_versions[:5]]}") + + # Map versions to deployment stages based on array position + # cloud-forward: newest version (index 0) (version in staging) + # cloud-current: the version released before the latest one (version in prod) + # cloud-backward: two versions back before the latest released version (version in staging) + + cloud_forward = all_versions[0] if len(all_versions) > 0 else None + cloud_current = all_versions[1] if len(all_versions) > 2 else None + cloud_backward = all_versions[2] if len(all_versions) > 4 else None + + # Self-managed LTS: Get from Helm chart (fast, already cached) + sm_versions = get_self_managed_versions() + # Currently targeting v25.2.x series as LTS + sm_lts = max( + (v for v in sm_versions if v.major == 25 and v.minor == 2), + default=max(sm_versions) if sm_versions else cloud_backward, + ) + + return { + "cloud-backward": cloud_backward, + "cloud-current": cloud_current, + "cloud-forward": cloud_forward, + "sm-lts": sm_lts, + } + diff --git a/test/console/QUICKSTART.md b/test/console/QUICKSTART.md new file mode 100644 index 0000000000000..f4d9814f74a41 --- /dev/null +++ b/test/console/QUICKSTART.md @@ -0,0 +1,256 @@ +# Quick Start: What You Can Run Now + +## In Materialize Repo + +All commands should be run from: `/Users/qindeel/dev/materialize/test/console` + +### 1. List Available Workflows + +```bash +./mzcompose list-workflows +``` + +**Output:** +``` +default +list-versions +start-version +``` + +### 2. Get Version Matrix (JSON) + +```bash +./mzcompose run list-versions +``` + +**Expected Output:** +```json +{ + "cloud-backward": "v0.162.2", + "cloud-current": "v0.162.3", + "cloud-forward": "v0.163.0", + "sm-lts": "v0.147.18" +} +``` + +**Use Case:** Console repo can parse this JSON to know which versions to test. + +### 3. Start Services with Default (Current Source) + +```bash +./mzcompose down -v # Clean up first +./mzcompose run default +``` + +**What It Does:** +- Starts Materialize with current source build +- Starts Redpanda, Postgres, MySQL, Testdrive +- Services keep running until you stop them + +**Expected Output:** +``` +Starting redpanda... +Starting postgres... +Starting mysql... +Starting materialized... +[Services running] +``` + +**After This:** You can run console tests: +```bash +cd ../../../console +yarn test:sql +``` + +**Clean Up:** +```bash +cd ../materialize/test/console +./mzcompose down -v +``` + +### 4. Start Services with Specific Version + +```bash +./mzcompose down -v # Clean up first +./mzcompose run start-version cloud-backward +# Or +./mzcompose run start-version sm-lts +# Or with direct version +./mzcompose run start-version v0.147.18 +``` + +**What It Does:** +- Starts Materialize with the specified version (from Docker Hub) +- Starts supporting services +- Services keep running + +**Expected Output:** +``` +Starting services for version: cloud-backward +Docker image: materialize/materialized:v0.162.2 +[Docker pulling image if needed] +[Services starting] + +✅ Services started successfully +You can now run console SQL tests from the console repo: + cd ../../../console && yarn test:sql +``` + +**Note:** Older versions might have issues. If a version fails to start, try a different one. + +**After This:** Run console tests then clean up: +```bash +cd ../../../console +yarn test:sql +cd - +./mzcompose down -v +``` + +### 5. Check Running Services + +```bash +docker ps +``` + +**Expected Output:** +``` +CONTAINER ID IMAGE PORTS +abc123... materialize/materialized 0.0.0.0:6875->6875/tcp +def456... redpanda 0.0.0.0:9092->9092/tcp +... +``` + +### 6. Check Logs if Services Fail + +```bash +docker logs console-materialized-1 +``` + +### 7. Stop All Services + +```bash +./mzcompose down -v +``` + +**What It Does:** +- Stops all containers +- Removes containers +- Removes volumes (-v flag) + +## Complete Testing Flow (Manual) + +Here's the complete flow to test console against a specific version: + +```bash +# 1. Start services with desired version +cd /Users/qindeel/dev/materialize/test/console +./mzcompose down -v +./mzcompose run start-version cloud-backward + +# 2. Run console tests +cd /Users/qindeel/dev/console +yarn test:sql + +# 3. Clean up +cd /Users/qindeel/dev/materialize/test/console +./mzcompose down -v +``` + +## What You Need to Build in Console Repo + +The console repo needs a script that automates the above flow for multiple versions: + +```bash +# Pseudocode for what console repo should do +for version in (cloud-backward, cloud-current, cloud-forward, sm-lts): + # Start services + cd ../materialize/test/console + ./mzcompose run start-version $version + + # Run tests + cd - + yarn test:sql + + # Clean up + cd ../materialize/test/console + ./mzcompose down -v +``` + +See **CONSOLE_REPO_INTEGRATION.md** for complete implementation examples in: +- Shell script +- Node.js/TypeScript +- GitHub Actions + +## Troubleshooting + +### Services Won't Start + +**Problem:** Container exits immediately + +**Check logs:** +```bash +docker logs console-materialized-1 +``` + +**Common causes:** +- Old version doesn't exist on Docker Hub +- Version incompatible with your system +- Port already in use + +**Solution:** +- Try a different version +- Use `cloud-current` (source build) which always works +- Check ports: `lsof -i :6875` + +### "Module not found" Errors + +Some helper scripts need to be run in the right context. Use the mzcompose workflows instead: + +**Don't:** +```bash +./resolve_version.py cloud-backward # Might fail +``` + +**Do:** +```bash +./mzcompose run list-versions # Always works +``` + +### Console Can't Connect + +**Problem:** yarn test:sql fails with connection errors + +**Check services are running:** +```bash +docker ps | grep console +``` + +**Test connection:** +```bash +docker exec console-materialized-1 psql -h localhost -p 6875 -U materialize -c "SELECT 1" +``` + +**Solution:** +- Ensure services fully started (wait a few seconds) +- Check Materialize is healthy in docker ps +- Verify port 6875 is accessible + +## Summary + +**You Can Run Now:** +1. ✅ `./mzcompose run list-versions` - See available versions +2. ✅ `./mzcompose run default` - Start with current source +3. ✅ `./mzcompose run start-version ` - Start with specific version + +**You Need to Build in Console Repo:** +1. ❌ Script to iterate through versions +2. ❌ Script to run yarn test:sql for each version +3. ❌ Script to collect and report results + +**Next Steps:** +1. Test the workflows manually (run the commands above) +2. Read **CONSOLE_REPO_INTEGRATION.md** for implementation guide +3. Choose implementation approach (shell/Node.js/GitHub Actions) +4. Build the test runner in console repo +5. Test locally before deploying to CI + diff --git a/test/console/README.md b/test/console/README.md index 5765c405bd404..336798fd13818 100644 --- a/test/console/README.md +++ b/test/console/README.md @@ -1,11 +1,104 @@ -# Introduction +# Console Test Fixture -Fixture for console tests +This directory contains mzcompose workflows for testing the Materialize console. -# Running +## Basic Usage + +Start Materialize and dependencies for console testing: ```bash ./mzcompose down -v +./mzcompose run default +``` + +Then in the console repository, run: +```bash +cd ../../../console # or wherever your console repo is +yarn test:sql +``` + +When done: +```bash +./mzcompose down -v +``` + +## Available Workflows +### `default` - Start Services (Current Source) + +Starts Materialize (current source build) and supporting services. + +```bash ./mzcompose run default ``` + +### `start-version` - Start Services with Specific Version + +Starts services with a specific Materialize version for console testing. + +```bash +./mzcompose run start-version cloud-backward +./mzcompose run start-version sm-lts +./mzcompose run start-version v0.147.18 # Or direct version +``` + +**Arguments:** +- `cloud-backward`: Last released version +- `cloud-current`: Current development version (uses source) +- `cloud-forward`: Forward compatibility (uses source) +- `sm-lts`: Self-managed LTS version +- Or any direct version like `v0.147.18` + +After starting services, run console tests from the console repo, then clean up: +```bash +cd ../../../console +yarn test:sql +cd - +./mzcompose down -v +``` + +### `list-versions` - Show Version Matrix + +Displays the version matrix as JSON: + +```bash +./mzcompose run list-versions +``` + +**Example output:** +```json +{ + "cloud-backward": "v0.147.18", + "cloud-current": null, + "cloud-forward": null, + "sm-lts": "v25.2.3" +} +``` + +## Multi-Version Testing + +To test console SQL against multiple Materialize versions, you need to implement a script **in the console repository** that: + +1. Calls `./mzcompose run list-versions` to get available versions +2. For each version: + - Calls `./mzcompose run start-version ` + - Runs `yarn test:sql` from console repo + - Calls `./mzcompose down -v` to clean up +3. Reports results + +See `CONSOLE_REPO_INTEGRATION.md` for detailed implementation guide with example scripts. + +## Helper Scripts + +### `resolve_version.py` + +Resolves version aliases to actual version tags: + +```bash +./resolve_version.py cloud-backward +# Output: v0.147.18 + +./resolve_version.py sm-lts +# Output: v25.2.3 +``` + diff --git a/test/console/ci-examples/README.md b/test/console/ci-examples/README.md new file mode 100644 index 0000000000000..bf7a6623cde16 --- /dev/null +++ b/test/console/ci-examples/README.md @@ -0,0 +1,236 @@ +# CI Configuration Examples + +This directory contains example CI/CD configurations for running Console tests against multiple Materialize versions. + +## Files + +- **`github-actions-example.yml`** - Example GitHub Actions workflow using matrix strategy +- **`buildkite-example.yml`** - Example Buildkite pipeline steps + +## Choosing a CI System + +### GitHub Actions (Recommended for Console Repo) + +**Pros:** +- Native to GitHub +- Easy matrix builds +- Good caching support +- No additional infrastructure + +**Use this if:** +- Console has its own CI pipeline +- You want independent test runs per PR +- Simple setup is preferred + +### Buildkite (Recommended for Materialize Repo Integration) + +**Pros:** +- Integrates with existing Materialize CI +- Can share build artifacts +- Flexible agent pools +- Better for monorepo-style testing + +**Use this if:** +- Want to integrate with Materialize's existing pipeline +- Need access to pre-built Materialize binaries +- Running in Materialize's infrastructure + +## Setup Instructions + +### For GitHub Actions + +1. Copy `github-actions-example.yml` to your console repo: + ```bash + cp ci-examples/github-actions-example.yml \ + ../../../console/.github/workflows/console-multi-version-tests.yml + ``` + +2. Adjust the workflow: + - Update repository paths if needed + - Configure any required secrets + - Adjust version matrix as needed + +3. Commit and push: + ```bash + cd ../../../console + git add .github/workflows/console-multi-version-tests.yml + git commit -m "Add multi-version console SQL tests" + git push + ``` + +### For Buildkite + +1. Add the steps from `buildkite-example.yml` to Materialize's pipeline: + ```bash + # Edit ci/test/pipeline.template.yml + # Add the console multi-version steps from buildkite-example.yml + ``` + +2. Ensure console repo is available: + - Either checkout as sibling directory + - Or add checkout step in pipeline + +3. Test the pipeline: + ```bash + cd /Users/qindeel/dev/materialize + ci/mkpipeline.sh > /tmp/pipeline.yml + # Review /tmp/pipeline.yml + ``` + +## Version Matrix Configuration + +The default matrix tests: + +| Alias | Description | Docker Image | Purpose | +|-------|-------------|--------------|---------| +| `cloud-backward` | Last released version | Published image | Test backwards compatibility | +| `cloud-current` | Current development | Source build | Test current development | +| `cloud-forward` | Forward compatibility | Source build | Test forward compatibility | +| `sm-lts` | Self-Managed LTS | Published image | Test LTS support | + +### Customizing the Matrix + +To add or remove versions, edit the matrix in your CI config: + +**GitHub Actions:** +```yaml +strategy: + matrix: + include: + - version_name: "Custom Version" + version_alias: "v0.100.0" +``` + +**Buildkite:** +```yaml +- id: console-custom-version + label: "Console Tests - Custom" + env: + MZ_VERSION: "v0.100.0" +``` + +## Troubleshooting + +### Tests fail to start + +**Problem:** `docker: Error response from daemon: pull access denied` + +**Solution:** Ensure the specified version exists: +```bash +cd /Users/qindeel/dev/materialize/test/console +./resolve_version.py cloud-backward +# Check if this version exists on Docker Hub +``` + +### Console can't connect to Materialize + +**Problem:** Connection refused or timeout + +**Solution:** Ensure services are fully started: +```yaml +- name: Wait for Materialize + run: sleep 10 # Add appropriate wait time +``` + +Or use health checks: +```bash +until docker exec console-materialized-1 pg_isready; do + echo "Waiting for Materialize..." + sleep 2 +done +``` + +### Version resolution fails + +**Problem:** `Unknown version alias: xyz` + +**Solution:** Check available aliases: +```bash +./resolve_version.py --help +``` + +### Tests pass locally but fail in CI + +**Common causes:** +- Different Docker images (version mismatch) +- Missing environment variables +- Timing issues (services not fully started) +- Resource constraints (memory/CPU) + +**Debug:** +```yaml +- name: Debug Environment + run: | + docker ps + docker logs console-materialized-1 + env | grep MZ_ +``` + +## Performance Optimization + +### Parallel Execution + +Both CI systems support parallel execution: + +**GitHub Actions:** Matrix jobs run in parallel automatically + +**Buildkite:** Add `parallelism` key: +```yaml +- id: console-tests + parallelism: 4 +``` + +### Caching + +**Cache Docker images:** +```yaml +# GitHub Actions +- name: Cache Docker images + uses: actions/cache@v4 + with: + path: /tmp/docker-images + key: docker-${{ matrix.version_alias }} +``` + +**Reuse Materialize builds:** +- For `cloud-current` and `cloud-forward`, depend on build job +- For published versions, images are cached by Docker + +### Resource Allocation + +Recommended agent specs: +- **Minimum:** 4 CPU, 8GB RAM +- **Recommended:** 8 CPU, 16GB RAM +- **Storage:** 50GB for Docker images + +## Monitoring and Alerts + +### Success Metrics + +Track these metrics: +- Pass rate per version +- Test duration +- Failure patterns (which tests fail on which versions) + +### Alerting + +Set up notifications for: +- Any test failure on `cloud-current` (blocks PR) +- Multiple failures on `cloud-backward` (compatibility issue) +- Failures on `sm-lts` (support issue) + +### Reporting + +Consider adding: +- Test result badges to README +- Slack/email notifications on failure +- Trend reports (pass rate over time) + +## Next Steps + +1. Choose your CI system (GitHub Actions or Buildkite) +2. Copy and customize the example configuration +3. Test locally first: `./test-with-version.sh cloud-backward` +4. Deploy to CI and monitor first run +5. Iterate on timing, resources, and version matrix as needed + diff --git a/test/console/mzcompose.py b/test/console/mzcompose.py index 2d5287835e005..7809f62326f97 100644 --- a/test/console/mzcompose.py +++ b/test/console/mzcompose.py @@ -7,7 +7,10 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -from materialize.mzcompose.composition import Composition, Service +import os +import sys + +from materialize.mzcompose.composition import Composition, Service, WorkflowArgumentParser from materialize.mzcompose.services.materialized import Materialized from materialize.mzcompose.services.mysql import MySql from materialize.mzcompose.services.postgres import Postgres @@ -34,3 +37,97 @@ def workflow_default(c: Composition) -> None: "materialized", Service("testdrive", idle=True), ) + + +def workflow_list_versions(c: Composition) -> None: + """ + Print the version matrix for console testing. + + This outputs a JSON object that Console CI can consume to determine + which Materialize versions to test against. + + Usage: + ./mzcompose run list-versions + """ + import json + from materialize.console_version_matrix import get_console_test_versions + + versions = get_console_test_versions() + + # Convert to JSON-friendly format (None -> null, MzVersion -> string) + output = { + name: str(version) if version else None + for name, version in versions.items() + } + + print(json.dumps(output, indent=2)) + + +def workflow_start_version(c: Composition, parser: WorkflowArgumentParser) -> None: + """ + Start Materialize services with a specific version from the version matrix. + + This workflow is designed to be called by the console repo to start services + before running console SQL tests. The console repo should call this, then run + its own yarn test:sql, then call ./mzcompose down. + + Usage: + ./mzcompose run start-version cloud-backward + ./mzcompose run start-version sm-lts + ./mzcompose run start-version # defaults to cloud-current + + Arguments: + version_alias: One of: cloud-backward, cloud-current, cloud-forward, sm-lts + Or a direct version like: v0.147.18 + """ + from materialize.console_version_matrix import get_console_test_versions + + parser.add_argument( + "version_alias", + nargs="?", + default="cloud-current", + help="Version alias (cloud-backward, cloud-current, cloud-forward, sm-lts) or direct version (v0.147.18)", + ) + args = parser.parse_args() + version_alias = args.version_alias + + # Check if it's a direct version string (starts with 'v') + if version_alias.startswith("v"): + version_str = version_alias + print(f"Using direct version: {version_str}") + else: + # Resolve from version matrix + versions = get_console_test_versions() + + if version_alias not in versions: + print(f"❌ Unknown version alias: {version_alias}") + print(f"Available aliases: {', '.join(versions.keys())}") + print(f"Or provide a direct version like: v0.147.18") + sys.exit(1) + + version = versions[version_alias] + version_str = str(version) if version else None + + print(f"Starting services for version: {version_alias}") + + if version_str: + print(f"Docker image: materialize/materialized:{version_str}") + # Override the Materialized service config with specific version + # Note: We update the compose dict directly so it persists + # This is necessary for sanity restarts to work correctly + c.compose["services"]["materialized"]["image"] = f"materialize/materialized:{version_str}" + else: + print("Using current source build") + + # Start services + c.up( + "redpanda", + "postgres", + "mysql", + "materialized", + Service("testdrive", idle=True), + ) + + print("\n✅ Services started successfully") + print("You can now run console SQL tests from the console repo:") + print(" cd ../../../console && yarn test:sql") diff --git a/test/console/resolve_version.py b/test/console/resolve_version.py new file mode 100755 index 0000000000000..8d8f48a4b3aa3 --- /dev/null +++ b/test/console/resolve_version.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +""" +Helper script to resolve version aliases to actual version tags. + +Usage: + ./resolve_version.py cloud-backward # prints: v0.99.0 + ./resolve_version.py sm-lts # prints: v25.1.3 + + # Use in shell: + export MZ_VERSION=$(./resolve_version.py cloud-backward) +""" + +import sys +from pathlib import Path + +# Add parent directories to path to import materialize modules +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from materialize.console_version_matrix import get_console_test_versions +from materialize.mz_version import MzVersion + + +def main(): + if len(sys.argv) != 2: + print("Usage: resolve_version.py ", file=sys.stderr) + print("\nAvailable aliases:", file=sys.stderr) + print(" cloud-backward - Last released minor version", file=sys.stderr) + print(" cloud-current - Current development version (uses source)", file=sys.stderr) + print(" cloud-forward - Forward compatible version (uses source)", file=sys.stderr) + print(" sm-lts - Self-Managed LTS", file=sys.stderr) + print("\nOr provide a direct version like: v0.100.0", file=sys.stderr) + sys.exit(1) + + version_alias = sys.argv[1] + + # Check if it's a direct version string + if version_alias.startswith("v"): + print(version_alias) + return + + # Otherwise, resolve from version matrix + versions = get_console_test_versions() + + if version_alias not in versions: + print(f"Error: Unknown version alias: {version_alias}", file=sys.stderr) + print(f"Available aliases: {', '.join(versions.keys())}", file=sys.stderr) + sys.exit(1) + + version = versions[version_alias] + if version: + print(str(version)) + else: + # Return empty for "use source" versions + print("") + + +if __name__ == "__main__": + main() + From cb7d6a36ce68b45cb476f15d799d95bca99dcccb Mon Sep 17 00:00:00 2001 From: Qindeel Ishtiaq Date: Wed, 3 Dec 2025 10:18:39 -0500 Subject: [PATCH 2/2] Materialize version control testing for console --- .../materialize/console_version_matrix.py | 38 ++- test/console/QUICKSTART.md | 256 ------------------ test/console/README.md | 137 ++++++---- test/console/ci-examples/README.md | 236 ---------------- test/console/mzcompose.py | 14 +- test/console/resolve_version.py | 68 ----- 6 files changed, 100 insertions(+), 649 deletions(-) delete mode 100644 test/console/QUICKSTART.md delete mode 100644 test/console/ci-examples/README.md delete mode 100755 test/console/resolve_version.py diff --git a/misc/python/materialize/console_version_matrix.py b/misc/python/materialize/console_version_matrix.py index 84d585dc1d570..774e463ba3dde 100644 --- a/misc/python/materialize/console_version_matrix.py +++ b/misc/python/materialize/console_version_matrix.py @@ -13,7 +13,7 @@ from materialize.mz_version import MzVersion from materialize.version_list import ( - get_all_mz_versions, + get_published_minor_mz_versions, get_self_managed_versions, ) @@ -21,41 +21,37 @@ def get_console_test_versions() -> dict[str, MzVersion | None]: """ Get all versions that should be tested for console. - + Uses existing version_list.py infrastructure to determine: - cloud-backward: Last released minor version (from git tags) - cloud-current: Current source (None) - - cloud-forward: Current source (None) + - cloud-forward: Current source (None) - sm-lts: Latest self-managed LTS minor version - + Returns: Dictionary mapping test name to version (None = current source) - - Note: This function does NOT verify Docker images exist, for speed. - The assumption is that released versions have published images. + + Note: Uses get_published_minor_mz_versions which verifies Docker images exist. """ - # Get all versions from git tags (fast, no Docker check) - all_versions = get_all_mz_versions(newest_first=True) - - # Debug: Print last 5 versions from git tags - print(f"DEBUG: Last 5 released versions: {[str(v) for v in all_versions[:5]]}") + # Get the latest patch version for each minor version (with Docker image verification) + # Limit to 5 versions for speed + minor_versions = get_published_minor_mz_versions(newest_first=True, limit=5) + + # Debug: Print minor versions + print(f"DEBUG: Minor versions (newest first): {[str(v) for v in minor_versions]}") # Map versions to deployment stages based on array position # cloud-forward: newest version (index 0) (version in staging) # cloud-current: the version released before the latest one (version in prod) # cloud-backward: two versions back before the latest released version (version in staging) - cloud_forward = all_versions[0] if len(all_versions) > 0 else None - cloud_current = all_versions[1] if len(all_versions) > 2 else None - cloud_backward = all_versions[2] if len(all_versions) > 4 else None + cloud_forward = minor_versions[0] if len(minor_versions) > 0 else None + cloud_current = minor_versions[1] if len(minor_versions) > 1 else None + cloud_backward = minor_versions[2] if len(minor_versions) > 2 else None - # Self-managed LTS: Get from Helm chart (fast, already cached) + # Self-managed LTS: Get the latest self-managed version from Helm chart sm_versions = get_self_managed_versions() - # Currently targeting v25.2.x series as LTS - sm_lts = max( - (v for v in sm_versions if v.major == 25 and v.minor == 2), - default=max(sm_versions) if sm_versions else cloud_backward, - ) + sm_lts = max(sm_versions) if sm_versions else cloud_backward return { "cloud-backward": cloud_backward, diff --git a/test/console/QUICKSTART.md b/test/console/QUICKSTART.md deleted file mode 100644 index f4d9814f74a41..0000000000000 --- a/test/console/QUICKSTART.md +++ /dev/null @@ -1,256 +0,0 @@ -# Quick Start: What You Can Run Now - -## In Materialize Repo - -All commands should be run from: `/Users/qindeel/dev/materialize/test/console` - -### 1. List Available Workflows - -```bash -./mzcompose list-workflows -``` - -**Output:** -``` -default -list-versions -start-version -``` - -### 2. Get Version Matrix (JSON) - -```bash -./mzcompose run list-versions -``` - -**Expected Output:** -```json -{ - "cloud-backward": "v0.162.2", - "cloud-current": "v0.162.3", - "cloud-forward": "v0.163.0", - "sm-lts": "v0.147.18" -} -``` - -**Use Case:** Console repo can parse this JSON to know which versions to test. - -### 3. Start Services with Default (Current Source) - -```bash -./mzcompose down -v # Clean up first -./mzcompose run default -``` - -**What It Does:** -- Starts Materialize with current source build -- Starts Redpanda, Postgres, MySQL, Testdrive -- Services keep running until you stop them - -**Expected Output:** -``` -Starting redpanda... -Starting postgres... -Starting mysql... -Starting materialized... -[Services running] -``` - -**After This:** You can run console tests: -```bash -cd ../../../console -yarn test:sql -``` - -**Clean Up:** -```bash -cd ../materialize/test/console -./mzcompose down -v -``` - -### 4. Start Services with Specific Version - -```bash -./mzcompose down -v # Clean up first -./mzcompose run start-version cloud-backward -# Or -./mzcompose run start-version sm-lts -# Or with direct version -./mzcompose run start-version v0.147.18 -``` - -**What It Does:** -- Starts Materialize with the specified version (from Docker Hub) -- Starts supporting services -- Services keep running - -**Expected Output:** -``` -Starting services for version: cloud-backward -Docker image: materialize/materialized:v0.162.2 -[Docker pulling image if needed] -[Services starting] - -✅ Services started successfully -You can now run console SQL tests from the console repo: - cd ../../../console && yarn test:sql -``` - -**Note:** Older versions might have issues. If a version fails to start, try a different one. - -**After This:** Run console tests then clean up: -```bash -cd ../../../console -yarn test:sql -cd - -./mzcompose down -v -``` - -### 5. Check Running Services - -```bash -docker ps -``` - -**Expected Output:** -``` -CONTAINER ID IMAGE PORTS -abc123... materialize/materialized 0.0.0.0:6875->6875/tcp -def456... redpanda 0.0.0.0:9092->9092/tcp -... -``` - -### 6. Check Logs if Services Fail - -```bash -docker logs console-materialized-1 -``` - -### 7. Stop All Services - -```bash -./mzcompose down -v -``` - -**What It Does:** -- Stops all containers -- Removes containers -- Removes volumes (-v flag) - -## Complete Testing Flow (Manual) - -Here's the complete flow to test console against a specific version: - -```bash -# 1. Start services with desired version -cd /Users/qindeel/dev/materialize/test/console -./mzcompose down -v -./mzcompose run start-version cloud-backward - -# 2. Run console tests -cd /Users/qindeel/dev/console -yarn test:sql - -# 3. Clean up -cd /Users/qindeel/dev/materialize/test/console -./mzcompose down -v -``` - -## What You Need to Build in Console Repo - -The console repo needs a script that automates the above flow for multiple versions: - -```bash -# Pseudocode for what console repo should do -for version in (cloud-backward, cloud-current, cloud-forward, sm-lts): - # Start services - cd ../materialize/test/console - ./mzcompose run start-version $version - - # Run tests - cd - - yarn test:sql - - # Clean up - cd ../materialize/test/console - ./mzcompose down -v -``` - -See **CONSOLE_REPO_INTEGRATION.md** for complete implementation examples in: -- Shell script -- Node.js/TypeScript -- GitHub Actions - -## Troubleshooting - -### Services Won't Start - -**Problem:** Container exits immediately - -**Check logs:** -```bash -docker logs console-materialized-1 -``` - -**Common causes:** -- Old version doesn't exist on Docker Hub -- Version incompatible with your system -- Port already in use - -**Solution:** -- Try a different version -- Use `cloud-current` (source build) which always works -- Check ports: `lsof -i :6875` - -### "Module not found" Errors - -Some helper scripts need to be run in the right context. Use the mzcompose workflows instead: - -**Don't:** -```bash -./resolve_version.py cloud-backward # Might fail -``` - -**Do:** -```bash -./mzcompose run list-versions # Always works -``` - -### Console Can't Connect - -**Problem:** yarn test:sql fails with connection errors - -**Check services are running:** -```bash -docker ps | grep console -``` - -**Test connection:** -```bash -docker exec console-materialized-1 psql -h localhost -p 6875 -U materialize -c "SELECT 1" -``` - -**Solution:** -- Ensure services fully started (wait a few seconds) -- Check Materialize is healthy in docker ps -- Verify port 6875 is accessible - -## Summary - -**You Can Run Now:** -1. ✅ `./mzcompose run list-versions` - See available versions -2. ✅ `./mzcompose run default` - Start with current source -3. ✅ `./mzcompose run start-version ` - Start with specific version - -**You Need to Build in Console Repo:** -1. ❌ Script to iterate through versions -2. ❌ Script to run yarn test:sql for each version -3. ❌ Script to collect and report results - -**Next Steps:** -1. Test the workflows manually (run the commands above) -2. Read **CONSOLE_REPO_INTEGRATION.md** for implementation guide -3. Choose implementation approach (shell/Node.js/GitHub Actions) -4. Build the test runner in console repo -5. Test locally before deploying to CI - diff --git a/test/console/README.md b/test/console/README.md index 336798fd13818..44f3b4a1c3d50 100644 --- a/test/console/README.md +++ b/test/console/README.md @@ -1,104 +1,123 @@ -# Console Test Fixture +# Introduction -This directory contains mzcompose workflows for testing the Materialize console. +This directory contains mzcompose workflows for testing the Materialize console +against different Materialize versions. -## Basic Usage - -Start Materialize and dependencies for console testing: +## Quick Start ```bash -./mzcompose down -v +# Start services with the default (current source) build ./mzcompose run default -``` -Then in the console repository, run: -```bash -cd ../../../console # or wherever your console repo is -yarn test:sql -``` +# Run console SQL tests (from console repo) +cd ~/dev/console && yarn test:sql -When done: -```bash -./mzcompose down -v +# Stop services when done +cd ~/dev/materialize/test/console && ./mzcompose down -v ``` ## Available Workflows ### `default` - Start Services (Current Source) -Starts Materialize (current source build) and supporting services. +Starts Materialize (current source build) with Redpanda, Postgres, MySQL, and SQL Server. ```bash ./mzcompose run default ``` +### `list-versions` - Show Version Matrix + +Outputs a JSON object with the versions to test against. This is useful for CI/CD +pipelines that need to determine which versions to test. + +```bash +./mzcompose run list-versions +``` + +**Example output:** +```json +{ + "cloud-backward": "v0.162.4", + "cloud-current": "v0.163.1", + "cloud-forward": "v0.164.2", + "sm-lts": "v26.1.1" +} +``` + +**Version meanings:** +- `cloud-forward`: Newest minor version (staging) +- `cloud-current`: Second newest minor version (production) +- `cloud-backward`: Third newest minor version (older staging) +- `sm-lts`: Latest self-managed version from Helm chart + ### `start-version` - Start Services with Specific Version -Starts services with a specific Materialize version for console testing. +Starts services with a specific Materialize version. Use this for testing console +compatibility across different Materialize releases. ```bash +# Using version aliases +./mzcompose run start-version cloud-forward +./mzcompose run start-version cloud-current ./mzcompose run start-version cloud-backward ./mzcompose run start-version sm-lts -./mzcompose run start-version v0.147.18 # Or direct version + +# Using a direct version tag +./mzcompose run start-version v0.164.2 ``` -**Arguments:** -- `cloud-backward`: Last released version -- `cloud-current`: Current development version (uses source) -- `cloud-forward`: Forward compatibility (uses source) -- `sm-lts`: Self-managed LTS version -- Or any direct version like `v0.147.18` +After starting, run tests from the console repo: +```bash +cd ~/dev/console && yarn test:sql +``` -After starting services, run console tests from the console repo, then clean up: +Then stop services: ```bash -cd ../../../console -yarn test:sql -cd - ./mzcompose down -v ``` -### `list-versions` - Show Version Matrix +## Multi-Version Testing (Local) -Displays the version matrix as JSON: +To test against all versions locally: ```bash +# 1. See available versions ./mzcompose run list-versions -``` - -**Example output:** -```json -{ - "cloud-backward": "v0.147.18", - "cloud-current": null, - "cloud-forward": null, - "sm-lts": "v25.2.3" -} -``` -## Multi-Version Testing +# 2. Test each version +for version in cloud-forward cloud-current cloud-backward sm-lts; do + echo "Testing $version..." + ./mzcompose down -v + ./mzcompose run start-version $version + cd ~/dev/console && yarn test:sql + cd ~/dev/materialize/test/console +done -To test console SQL against multiple Materialize versions, you need to implement a script **in the console repository** that: +./mzcompose down -v +``` -1. Calls `./mzcompose run list-versions` to get available versions -2. For each version: - - Calls `./mzcompose run start-version ` - - Runs `yarn test:sql` from console repo - - Calls `./mzcompose down -v` to clean up -3. Reports results +## GitHub Actions Integration -See `CONSOLE_REPO_INTEGRATION.md` for detailed implementation guide with example scripts. +For CI/CD in the console repository, the typical flow is: -## Helper Scripts +1. Checkout the materialize repo +2. Run `./mzcompose run list-versions` to get the version matrix as JSON +3. Parse the JSON to create a GitHub Actions matrix strategy +4. For each version in the matrix: + - Run `./mzcompose run start-version ` + - Run `yarn test:sql` + - Run `./mzcompose down -v` -### `resolve_version.py` +## How Version Selection Works -Resolves version aliases to actual version tags: +The version matrix is generated by `get_published_minor_mz_versions()` which: -```bash -./resolve_version.py cloud-backward -# Output: v0.147.18 +1. Gets all Materialize versions from git tags +2. Filters to the latest patch for each minor version (e.g., v0.164.2, not v0.164.0) +3. Verifies Docker images exist for each version +4. Returns the 5 most recent minor versions -./resolve_version.py sm-lts -# Output: v25.2.3 -``` +For self-managed (`sm-lts`), it fetches from the Helm chart at +`https://materializeinc.github.io/materialize/index.yaml` and returns the latest version. diff --git a/test/console/ci-examples/README.md b/test/console/ci-examples/README.md deleted file mode 100644 index bf7a6623cde16..0000000000000 --- a/test/console/ci-examples/README.md +++ /dev/null @@ -1,236 +0,0 @@ -# CI Configuration Examples - -This directory contains example CI/CD configurations for running Console tests against multiple Materialize versions. - -## Files - -- **`github-actions-example.yml`** - Example GitHub Actions workflow using matrix strategy -- **`buildkite-example.yml`** - Example Buildkite pipeline steps - -## Choosing a CI System - -### GitHub Actions (Recommended for Console Repo) - -**Pros:** -- Native to GitHub -- Easy matrix builds -- Good caching support -- No additional infrastructure - -**Use this if:** -- Console has its own CI pipeline -- You want independent test runs per PR -- Simple setup is preferred - -### Buildkite (Recommended for Materialize Repo Integration) - -**Pros:** -- Integrates with existing Materialize CI -- Can share build artifacts -- Flexible agent pools -- Better for monorepo-style testing - -**Use this if:** -- Want to integrate with Materialize's existing pipeline -- Need access to pre-built Materialize binaries -- Running in Materialize's infrastructure - -## Setup Instructions - -### For GitHub Actions - -1. Copy `github-actions-example.yml` to your console repo: - ```bash - cp ci-examples/github-actions-example.yml \ - ../../../console/.github/workflows/console-multi-version-tests.yml - ``` - -2. Adjust the workflow: - - Update repository paths if needed - - Configure any required secrets - - Adjust version matrix as needed - -3. Commit and push: - ```bash - cd ../../../console - git add .github/workflows/console-multi-version-tests.yml - git commit -m "Add multi-version console SQL tests" - git push - ``` - -### For Buildkite - -1. Add the steps from `buildkite-example.yml` to Materialize's pipeline: - ```bash - # Edit ci/test/pipeline.template.yml - # Add the console multi-version steps from buildkite-example.yml - ``` - -2. Ensure console repo is available: - - Either checkout as sibling directory - - Or add checkout step in pipeline - -3. Test the pipeline: - ```bash - cd /Users/qindeel/dev/materialize - ci/mkpipeline.sh > /tmp/pipeline.yml - # Review /tmp/pipeline.yml - ``` - -## Version Matrix Configuration - -The default matrix tests: - -| Alias | Description | Docker Image | Purpose | -|-------|-------------|--------------|---------| -| `cloud-backward` | Last released version | Published image | Test backwards compatibility | -| `cloud-current` | Current development | Source build | Test current development | -| `cloud-forward` | Forward compatibility | Source build | Test forward compatibility | -| `sm-lts` | Self-Managed LTS | Published image | Test LTS support | - -### Customizing the Matrix - -To add or remove versions, edit the matrix in your CI config: - -**GitHub Actions:** -```yaml -strategy: - matrix: - include: - - version_name: "Custom Version" - version_alias: "v0.100.0" -``` - -**Buildkite:** -```yaml -- id: console-custom-version - label: "Console Tests - Custom" - env: - MZ_VERSION: "v0.100.0" -``` - -## Troubleshooting - -### Tests fail to start - -**Problem:** `docker: Error response from daemon: pull access denied` - -**Solution:** Ensure the specified version exists: -```bash -cd /Users/qindeel/dev/materialize/test/console -./resolve_version.py cloud-backward -# Check if this version exists on Docker Hub -``` - -### Console can't connect to Materialize - -**Problem:** Connection refused or timeout - -**Solution:** Ensure services are fully started: -```yaml -- name: Wait for Materialize - run: sleep 10 # Add appropriate wait time -``` - -Or use health checks: -```bash -until docker exec console-materialized-1 pg_isready; do - echo "Waiting for Materialize..." - sleep 2 -done -``` - -### Version resolution fails - -**Problem:** `Unknown version alias: xyz` - -**Solution:** Check available aliases: -```bash -./resolve_version.py --help -``` - -### Tests pass locally but fail in CI - -**Common causes:** -- Different Docker images (version mismatch) -- Missing environment variables -- Timing issues (services not fully started) -- Resource constraints (memory/CPU) - -**Debug:** -```yaml -- name: Debug Environment - run: | - docker ps - docker logs console-materialized-1 - env | grep MZ_ -``` - -## Performance Optimization - -### Parallel Execution - -Both CI systems support parallel execution: - -**GitHub Actions:** Matrix jobs run in parallel automatically - -**Buildkite:** Add `parallelism` key: -```yaml -- id: console-tests - parallelism: 4 -``` - -### Caching - -**Cache Docker images:** -```yaml -# GitHub Actions -- name: Cache Docker images - uses: actions/cache@v4 - with: - path: /tmp/docker-images - key: docker-${{ matrix.version_alias }} -``` - -**Reuse Materialize builds:** -- For `cloud-current` and `cloud-forward`, depend on build job -- For published versions, images are cached by Docker - -### Resource Allocation - -Recommended agent specs: -- **Minimum:** 4 CPU, 8GB RAM -- **Recommended:** 8 CPU, 16GB RAM -- **Storage:** 50GB for Docker images - -## Monitoring and Alerts - -### Success Metrics - -Track these metrics: -- Pass rate per version -- Test duration -- Failure patterns (which tests fail on which versions) - -### Alerting - -Set up notifications for: -- Any test failure on `cloud-current` (blocks PR) -- Multiple failures on `cloud-backward` (compatibility issue) -- Failures on `sm-lts` (support issue) - -### Reporting - -Consider adding: -- Test result badges to README -- Slack/email notifications on failure -- Trend reports (pass rate over time) - -## Next Steps - -1. Choose your CI system (GitHub Actions or Buildkite) -2. Copy and customize the example configuration -3. Test locally first: `./test-with-version.sh cloud-backward` -4. Deploy to CI and monitor first run -5. Iterate on timing, resources, and version matrix as needed - diff --git a/test/console/mzcompose.py b/test/console/mzcompose.py index 7809f62326f97..5437a2685bb3e 100644 --- a/test/console/mzcompose.py +++ b/test/console/mzcompose.py @@ -7,7 +7,6 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -import os import sys from materialize.mzcompose.composition import Composition, Service, WorkflowArgumentParser @@ -78,7 +77,7 @@ def workflow_start_version(c: Composition, parser: WorkflowArgumentParser) -> No Arguments: version_alias: One of: cloud-backward, cloud-current, cloud-forward, sm-lts - Or a direct version like: v0.147.18 + Or a direct version like: v0.26.1.1 """ from materialize.console_version_matrix import get_console_test_versions @@ -102,27 +101,24 @@ def workflow_start_version(c: Composition, parser: WorkflowArgumentParser) -> No if version_alias not in versions: print(f"❌ Unknown version alias: {version_alias}") print(f"Available aliases: {', '.join(versions.keys())}") - print(f"Or provide a direct version like: v0.147.18") + print(f"Or provide a direct version like: v0.26.1.1") sys.exit(1) version = versions[version_alias] version_str = str(version) if version else None - print(f"Starting services for version: {version_alias}") - + if version_str: print(f"Docker image: materialize/materialized:{version_str}") - # Override the Materialized service config with specific version - # Note: We update the compose dict directly so it persists - # This is necessary for sanity restarts to work correctly c.compose["services"]["materialized"]["image"] = f"materialize/materialized:{version_str}" else: print("Using current source build") - + # Start services c.up( "redpanda", "postgres", + "sql-server", "mysql", "materialized", Service("testdrive", idle=True), diff --git a/test/console/resolve_version.py b/test/console/resolve_version.py deleted file mode 100755 index 8d8f48a4b3aa3..0000000000000 --- a/test/console/resolve_version.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -""" -Helper script to resolve version aliases to actual version tags. - -Usage: - ./resolve_version.py cloud-backward # prints: v0.99.0 - ./resolve_version.py sm-lts # prints: v25.1.3 - - # Use in shell: - export MZ_VERSION=$(./resolve_version.py cloud-backward) -""" - -import sys -from pathlib import Path - -# Add parent directories to path to import materialize modules -sys.path.insert(0, str(Path(__file__).parent.parent.parent)) - -from materialize.console_version_matrix import get_console_test_versions -from materialize.mz_version import MzVersion - - -def main(): - if len(sys.argv) != 2: - print("Usage: resolve_version.py ", file=sys.stderr) - print("\nAvailable aliases:", file=sys.stderr) - print(" cloud-backward - Last released minor version", file=sys.stderr) - print(" cloud-current - Current development version (uses source)", file=sys.stderr) - print(" cloud-forward - Forward compatible version (uses source)", file=sys.stderr) - print(" sm-lts - Self-Managed LTS", file=sys.stderr) - print("\nOr provide a direct version like: v0.100.0", file=sys.stderr) - sys.exit(1) - - version_alias = sys.argv[1] - - # Check if it's a direct version string - if version_alias.startswith("v"): - print(version_alias) - return - - # Otherwise, resolve from version matrix - versions = get_console_test_versions() - - if version_alias not in versions: - print(f"Error: Unknown version alias: {version_alias}", file=sys.stderr) - print(f"Available aliases: {', '.join(versions.keys())}", file=sys.stderr) - sys.exit(1) - - version = versions[version_alias] - if version: - print(str(version)) - else: - # Return empty for "use source" versions - print("") - - -if __name__ == "__main__": - main() -