Testing different console versions

misc/python/materialize/console_version_matrix.py

@@ -0,0 +1,62 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+"""
+Version matrix for console testing - uses existing version_list infrastructure.
+"""
+
+from materialize.mz_version import MzVersion
+from materialize.version_list import (
+    get_published_minor_mz_versions,
+    get_self_managed_versions,
+)
+
+
+def get_console_test_versions() -> dict[str, MzVersion | None]:
+    """
+    Get all versions that should be tested for console.
+
+    Uses existing version_list.py infrastructure to determine:
+    - cloud-backward: Last released minor version (from git tags)
+    - cloud-current: Current source (None)
+    - cloud-forward: Current source (None)
+    - sm-lts: Latest self-managed LTS minor version
+
+    Returns:
+        Dictionary mapping test name to version (None = current source)
+
+    Note: Uses get_published_minor_mz_versions which verifies Docker images exist.
+    """
+    # Get the latest patch version for each minor version (with Docker image verification)
+    # Limit to 5 versions for speed
+    minor_versions = get_published_minor_mz_versions(newest_first=True, limit=5)
+
+    # Debug: Print minor versions
+    print(f"DEBUG: Minor versions (newest first): {[str(v) for v in minor_versions]}")
+
+    # Map versions to deployment stages based on array position
+    # cloud-forward: newest version (index 0) (version in staging)
+    # cloud-current: the version released before the latest one (version in prod)
+    # cloud-backward: two versions back before the latest released version (version in staging)
+
+    cloud_forward = minor_versions[0] if len(minor_versions) > 0 else None
+    cloud_current = minor_versions[1] if len(minor_versions) > 1 else None
+    cloud_backward = minor_versions[2] if len(minor_versions) > 2 else None
+
+    # Self-managed LTS: Get the latest self-managed version from Helm chart
+    sm_versions = get_self_managed_versions()
+    sm_lts = max(sm_versions) if sm_versions else cloud_backward
+
+    return {
+        "cloud-backward": cloud_backward,
+        "cloud-current": cloud_current,
+        "cloud-forward": cloud_forward,
+        "sm-lts": sm_lts,
+    }
+

test/console/README.md

@@ -1,11 +1,123 @@
 # Introduction
 
-Fixture for console tests
+This directory contains mzcompose workflows for testing the Materialize console
+against different Materialize versions.
 
-# Running
+## Quick Start
 
 ```bash
-./mzcompose down -v
+# Start services with the default (current source) build
+./mzcompose run default
+
+# Run console SQL tests (from console repo)
+cd ~/dev/console && yarn test:sql
+
+# Stop services when done
+cd ~/dev/materialize/test/console && ./mzcompose down -v
+```
+
+## Available Workflows
 
+### `default` - Start Services (Current Source)
+
+Starts Materialize (current source build) with Redpanda, Postgres, MySQL, and SQL Server.
+
+```bash
 ./mzcompose run default
 ```
+
+### `list-versions` - Show Version Matrix
+
+Outputs a JSON object with the versions to test against. This is useful for CI/CD
+pipelines that need to determine which versions to test.
+
+```bash
+./mzcompose run list-versions
+```
+
+**Example output:**
+```json
+{
+  "cloud-backward": "v0.162.4",
+  "cloud-current": "v0.163.1",
+  "cloud-forward": "v0.164.2",
+  "sm-lts": "v26.1.1"
+}
+```
+
+**Version meanings:**
+- `cloud-forward`: Newest minor version (staging)
+- `cloud-current`: Second newest minor version (production)
+- `cloud-backward`: Third newest minor version (older staging)
+- `sm-lts`: Latest self-managed version from Helm chart
+
+### `start-version` - Start Services with Specific Version
+
+Starts services with a specific Materialize version. Use this for testing console
+compatibility across different Materialize releases.
+
+```bash
+# Using version aliases
+./mzcompose run start-version cloud-forward
+./mzcompose run start-version cloud-current
+./mzcompose run start-version cloud-backward
+./mzcompose run start-version sm-lts
+
+# Using a direct version tag
+./mzcompose run start-version v0.164.2
+```
+
+After starting, run tests from the console repo:
+```bash
+cd ~/dev/console && yarn test:sql
+```
+
+Then stop services:
+```bash
+./mzcompose down -v
+```
+
+## Multi-Version Testing (Local)
+
+To test against all versions locally:
+
+```bash
+# 1. See available versions
+./mzcompose run list-versions
+
+# 2. Test each version
+for version in cloud-forward cloud-current cloud-backward sm-lts; do
+  echo "Testing $version..."
+  ./mzcompose down -v
+  ./mzcompose run start-version $version
+  cd ~/dev/console && yarn test:sql
+  cd ~/dev/materialize/test/console
+done
+
+./mzcompose down -v
+```
+
+## GitHub Actions Integration
+
+For CI/CD in the console repository, the typical flow is:
+
+1. Checkout the materialize repo
+2. Run `./mzcompose run list-versions` to get the version matrix as JSON
+3. Parse the JSON to create a GitHub Actions matrix strategy
+4. For each version in the matrix:
+   - Run `./mzcompose run start-version <version-alias>`
+   - Run `yarn test:sql`
+   - Run `./mzcompose down -v`
+
+## How Version Selection Works
+
+The version matrix is generated by `get_published_minor_mz_versions()` which:
+
+1. Gets all Materialize versions from git tags
+2. Filters to the latest patch for each minor version (e.g., v0.164.2, not v0.164.0)
+3. Verifies Docker images exist for each version
+4. Returns the 5 most recent minor versions
+
+For self-managed (`sm-lts`), it fetches from the Helm chart at
+`https://materializeinc.github.io/materialize/index.yaml` and returns the latest version.
+

test/console/mzcompose.py

@@ -7,7 +7,9 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-from materialize.mzcompose.composition import Composition, Service
+import sys
+
+from materialize.mzcompose.composition import Composition, Service, WorkflowArgumentParser
 from materialize.mzcompose.services.materialized import Materialized
 from materialize.mzcompose.services.mysql import MySql
 from materialize.mzcompose.services.postgres import Postgres
@@ -34,3 +36,94 @@ def workflow_default(c: Composition) -> None:
         "materialized",
         Service("testdrive", idle=True),
     )
+
+
+def workflow_list_versions(c: Composition) -> None:
+    """
+    Print the version matrix for console testing.
+
+    This outputs a JSON object that Console CI can consume to determine
+    which Materialize versions to test against.
+
+    Usage:
+        ./mzcompose run list-versions
+    """
+    import json
+    from materialize.console_version_matrix import get_console_test_versions
+
+    versions = get_console_test_versions()
+
+    # Convert to JSON-friendly format (None -> null, MzVersion -> string)
+    output = {
+        name: str(version) if version else None
+        for name, version in versions.items()
+    }
+
+    print(json.dumps(output, indent=2))
+
+
+def workflow_start_version(c: Composition, parser: WorkflowArgumentParser) -> None:
+    """
+    Start Materialize services with a specific version from the version matrix.
+
+    This workflow is designed to be called by the console repo to start services
+    before running console SQL tests. The console repo should call this, then run
+    its own yarn test:sql, then call ./mzcompose down.
+
+    Usage:
+        ./mzcompose run start-version cloud-backward
+        ./mzcompose run start-version sm-lts
+        ./mzcompose run start-version  # defaults to cloud-current
+
+    Arguments:
+        version_alias: One of: cloud-backward, cloud-current, cloud-forward, sm-lts
+                      Or a direct version like: v0.26.1.1
+    """
+    from materialize.console_version_matrix import get_console_test_versions
+
+    parser.add_argument(
+        "version_alias",
+        nargs="?",
+        default="cloud-current",
+        help="Version alias (cloud-backward, cloud-current, cloud-forward, sm-lts) or direct version (v0.147.18)",
+    )
+    args = parser.parse_args()
+    version_alias = args.version_alias
+
+    # Check if it's a direct version string (starts with 'v')
+    if version_alias.startswith("v"):
+        version_str = version_alias
+        print(f"Using direct version: {version_str}")
+    else:
+        # Resolve from version matrix
+        versions = get_console_test_versions()
+
+        if version_alias not in versions:
+            print(f"❌ Unknown version alias: {version_alias}")
+            print(f"Available aliases: {', '.join(versions.keys())}")
+            print(f"Or provide a direct version like: v0.26.1.1")
+            sys.exit(1)
+
+        version = versions[version_alias]
+        version_str = str(version) if version else None
+    print(f"Starting services for version: {version_alias}")
+
+    if version_str:
+        print(f"Docker image: materialize/materialized:{version_str}")
+        c.compose["services"]["materialized"]["image"] = f"materialize/materialized:{version_str}"
+    else:
+        print("Using current source build")
+
+    # Start services
+    c.up(
+        "redpanda",
+        "postgres",
+        "sql-server",
+        "mysql",
+        "materialized",
+        Service("testdrive", idle=True),
+    )
+
+    print("\n✅ Services started successfully")
+    print("You can now run console SQL tests from the console repo:")
+    print("  cd ../../../console && yarn test:sql")