diff --git a/misc/python/materialize/console_version_matrix.py b/misc/python/materialize/console_version_matrix.py new file mode 100644 index 0000000000000..774e463ba3dde --- /dev/null +++ b/misc/python/materialize/console_version_matrix.py @@ -0,0 +1,62 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +""" +Version matrix for console testing - uses existing version_list infrastructure. +""" + +from materialize.mz_version import MzVersion +from materialize.version_list import ( + get_published_minor_mz_versions, + get_self_managed_versions, +) + + +def get_console_test_versions() -> dict[str, MzVersion | None]: + """ + Get all versions that should be tested for console. + + Uses existing version_list.py infrastructure to determine: + - cloud-backward: Last released minor version (from git tags) + - cloud-current: Current source (None) + - cloud-forward: Current source (None) + - sm-lts: Latest self-managed LTS minor version + + Returns: + Dictionary mapping test name to version (None = current source) + + Note: Uses get_published_minor_mz_versions which verifies Docker images exist. + """ + # Get the latest patch version for each minor version (with Docker image verification) + # Limit to 5 versions for speed + minor_versions = get_published_minor_mz_versions(newest_first=True, limit=5) + + # Debug: Print minor versions + print(f"DEBUG: Minor versions (newest first): {[str(v) for v in minor_versions]}") + + # Map versions to deployment stages based on array position + # cloud-forward: newest version (index 0) (version in staging) + # cloud-current: the version released before the latest one (version in prod) + # cloud-backward: two versions back before the latest released version (version in staging) + + cloud_forward = minor_versions[0] if len(minor_versions) > 0 else None + cloud_current = minor_versions[1] if len(minor_versions) > 1 else None + cloud_backward = minor_versions[2] if len(minor_versions) > 2 else None + + # Self-managed LTS: Get the latest self-managed version from Helm chart + sm_versions = get_self_managed_versions() + sm_lts = max(sm_versions) if sm_versions else cloud_backward + + return { + "cloud-backward": cloud_backward, + "cloud-current": cloud_current, + "cloud-forward": cloud_forward, + "sm-lts": sm_lts, + } + diff --git a/test/console/README.md b/test/console/README.md index 5765c405bd404..44f3b4a1c3d50 100644 --- a/test/console/README.md +++ b/test/console/README.md @@ -1,11 +1,123 @@ # Introduction -Fixture for console tests +This directory contains mzcompose workflows for testing the Materialize console +against different Materialize versions. -# Running +## Quick Start ```bash -./mzcompose down -v +# Start services with the default (current source) build +./mzcompose run default + +# Run console SQL tests (from console repo) +cd ~/dev/console && yarn test:sql + +# Stop services when done +cd ~/dev/materialize/test/console && ./mzcompose down -v +``` + +## Available Workflows +### `default` - Start Services (Current Source) + +Starts Materialize (current source build) with Redpanda, Postgres, MySQL, and SQL Server. + +```bash ./mzcompose run default ``` + +### `list-versions` - Show Version Matrix + +Outputs a JSON object with the versions to test against. This is useful for CI/CD +pipelines that need to determine which versions to test. + +```bash +./mzcompose run list-versions +``` + +**Example output:** +```json +{ + "cloud-backward": "v0.162.4", + "cloud-current": "v0.163.1", + "cloud-forward": "v0.164.2", + "sm-lts": "v26.1.1" +} +``` + +**Version meanings:** +- `cloud-forward`: Newest minor version (staging) +- `cloud-current`: Second newest minor version (production) +- `cloud-backward`: Third newest minor version (older staging) +- `sm-lts`: Latest self-managed version from Helm chart + +### `start-version` - Start Services with Specific Version + +Starts services with a specific Materialize version. Use this for testing console +compatibility across different Materialize releases. + +```bash +# Using version aliases +./mzcompose run start-version cloud-forward +./mzcompose run start-version cloud-current +./mzcompose run start-version cloud-backward +./mzcompose run start-version sm-lts + +# Using a direct version tag +./mzcompose run start-version v0.164.2 +``` + +After starting, run tests from the console repo: +```bash +cd ~/dev/console && yarn test:sql +``` + +Then stop services: +```bash +./mzcompose down -v +``` + +## Multi-Version Testing (Local) + +To test against all versions locally: + +```bash +# 1. See available versions +./mzcompose run list-versions + +# 2. Test each version +for version in cloud-forward cloud-current cloud-backward sm-lts; do + echo "Testing $version..." + ./mzcompose down -v + ./mzcompose run start-version $version + cd ~/dev/console && yarn test:sql + cd ~/dev/materialize/test/console +done + +./mzcompose down -v +``` + +## GitHub Actions Integration + +For CI/CD in the console repository, the typical flow is: + +1. Checkout the materialize repo +2. Run `./mzcompose run list-versions` to get the version matrix as JSON +3. Parse the JSON to create a GitHub Actions matrix strategy +4. For each version in the matrix: + - Run `./mzcompose run start-version ` + - Run `yarn test:sql` + - Run `./mzcompose down -v` + +## How Version Selection Works + +The version matrix is generated by `get_published_minor_mz_versions()` which: + +1. Gets all Materialize versions from git tags +2. Filters to the latest patch for each minor version (e.g., v0.164.2, not v0.164.0) +3. Verifies Docker images exist for each version +4. Returns the 5 most recent minor versions + +For self-managed (`sm-lts`), it fetches from the Helm chart at +`https://materializeinc.github.io/materialize/index.yaml` and returns the latest version. + diff --git a/test/console/mzcompose.py b/test/console/mzcompose.py index 2d5287835e005..5437a2685bb3e 100644 --- a/test/console/mzcompose.py +++ b/test/console/mzcompose.py @@ -7,7 +7,9 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -from materialize.mzcompose.composition import Composition, Service +import sys + +from materialize.mzcompose.composition import Composition, Service, WorkflowArgumentParser from materialize.mzcompose.services.materialized import Materialized from materialize.mzcompose.services.mysql import MySql from materialize.mzcompose.services.postgres import Postgres @@ -34,3 +36,94 @@ def workflow_default(c: Composition) -> None: "materialized", Service("testdrive", idle=True), ) + + +def workflow_list_versions(c: Composition) -> None: + """ + Print the version matrix for console testing. + + This outputs a JSON object that Console CI can consume to determine + which Materialize versions to test against. + + Usage: + ./mzcompose run list-versions + """ + import json + from materialize.console_version_matrix import get_console_test_versions + + versions = get_console_test_versions() + + # Convert to JSON-friendly format (None -> null, MzVersion -> string) + output = { + name: str(version) if version else None + for name, version in versions.items() + } + + print(json.dumps(output, indent=2)) + + +def workflow_start_version(c: Composition, parser: WorkflowArgumentParser) -> None: + """ + Start Materialize services with a specific version from the version matrix. + + This workflow is designed to be called by the console repo to start services + before running console SQL tests. The console repo should call this, then run + its own yarn test:sql, then call ./mzcompose down. + + Usage: + ./mzcompose run start-version cloud-backward + ./mzcompose run start-version sm-lts + ./mzcompose run start-version # defaults to cloud-current + + Arguments: + version_alias: One of: cloud-backward, cloud-current, cloud-forward, sm-lts + Or a direct version like: v0.26.1.1 + """ + from materialize.console_version_matrix import get_console_test_versions + + parser.add_argument( + "version_alias", + nargs="?", + default="cloud-current", + help="Version alias (cloud-backward, cloud-current, cloud-forward, sm-lts) or direct version (v0.147.18)", + ) + args = parser.parse_args() + version_alias = args.version_alias + + # Check if it's a direct version string (starts with 'v') + if version_alias.startswith("v"): + version_str = version_alias + print(f"Using direct version: {version_str}") + else: + # Resolve from version matrix + versions = get_console_test_versions() + + if version_alias not in versions: + print(f"āŒ Unknown version alias: {version_alias}") + print(f"Available aliases: {', '.join(versions.keys())}") + print(f"Or provide a direct version like: v0.26.1.1") + sys.exit(1) + + version = versions[version_alias] + version_str = str(version) if version else None + print(f"Starting services for version: {version_alias}") + + if version_str: + print(f"Docker image: materialize/materialized:{version_str}") + c.compose["services"]["materialized"]["image"] = f"materialize/materialized:{version_str}" + else: + print("Using current source build") + + # Start services + c.up( + "redpanda", + "postgres", + "sql-server", + "mysql", + "materialized", + Service("testdrive", idle=True), + ) + + print("\nāœ… Services started successfully") + print("You can now run console SQL tests from the console repo:") + print(" cd ../../../console && yarn test:sql")