Skip to content

Commit 91a0e4b

Browse files
leedqinleedqin
authored
Merge pull request #34345 from MaterializeInc/qindeel/consoleTestingIteration
Testing different versions of Materialize against console sql queries
2 parents 4ba5ad3 + 968d731 commit 91a0e4b

File tree

3 files changed

+278
-11
lines changed

3 files changed

+278
-11
lines changed

‎misc/python/materialize/console_version_matrix.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Copyright Materialize, Inc. and contributors. All rights reserved.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the LICENSE file at the root of this repository.
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0.
9+
10+
"""
11+
Version matrix for console testing - uses existing version_list infrastructure.
12+
"""
13+
14+
from materialize.mz_version import MzVersion
15+
from materialize.version_list import (
16+
get_published_minor_mz_versions,
17+
get_self_managed_versions,
18+
)
19+
20+
21+
def get_console_test_versions() -> dict[str, MzVersion]:
22+
"""
23+
Get all versions that should be tested for console.
24+
25+
Uses existing version_list.py infrastructure to determine:
26+
- latest: Most recent minor version
27+
- previous: One minor version back
28+
- older: Two minor versions back
29+
- self-managed: Latest self-managed version from Helm chart
30+
31+
Returns:
32+
Dictionary mapping test name to version
33+
34+
Note: Uses get_published_minor_mz_versions which verifies Docker images exist.
35+
"""
36+
# Get the latest patch version for each minor version (with Docker image verification)
37+
# Limit to 5 versions for speed
38+
minor_versions = get_published_minor_mz_versions(newest_first=True, limit=5)
39+
40+
assert (
41+
len(minor_versions) >= 3
42+
), f"Expected at least 3 minor versions, got {len(minor_versions)}"
43+
44+
latest = minor_versions[0]
45+
previous = minor_versions[1]
46+
older = minor_versions[2]
47+
48+
# Self-managed: Get the latest self-managed version from Helm chart
49+
sm_versions = get_self_managed_versions()
50+
assert sm_versions, "Expected at least one self-managed version"
51+
self_managed = max(sm_versions)
52+
53+
return {
54+
"latest": latest,
55+
"previous": previous,
56+
"older": older,
57+
"self-managed": self_managed,
58+
}

‎test/console/README.md‎

Lines changed: 114 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,122 @@
11
# Introduction
22

3-
Fixture for console tests
3+
This directory contains mzcompose workflows for testing the Materialize console
4+
against different Materialize versions.
45

5-
# Running
6+
## Quick Start
67

78
```bash
8-
./mzcompose down -v
9+
# Start services with the default (current source) build
10+
./mzcompose run default
11+
12+
# Run console SQL tests (from console repo)
13+
cd ~/dev/console && yarn test:sql
14+
15+
# Stop services when done
16+
cd ~/dev/materialize/test/console && ./mzcompose down -v
17+
```
18+
19+
## Available Workflows
920

21+
### `default` - Start Services (Current Source)
22+
23+
Starts Materialize (current source build) with Redpanda, Postgres, MySQL, and SQL Server.
24+
25+
```bash
1026
./mzcompose run default
1127
```
28+
29+
### `list-versions` - Show Version Matrix
30+
31+
Outputs a JSON object with the versions to test against. This is useful for CI/CD
32+
pipelines that need to determine which versions to test.
33+
34+
```bash
35+
./mzcompose run list-versions
36+
```
37+
38+
**Example output:**
39+
```json
40+
{
41+
"latest": "v26.1.1",
42+
"previous": "v26.0.0",
43+
"older": "v0.164.2",
44+
"self-managed": "v26.1.1"
45+
}
46+
```
47+
48+
**Version meanings:**
49+
- `latest`: Most recent minor version
50+
- `previous`: One minor version back
51+
- `older`: Two minor versions back
52+
- `self-managed`: Latest self-managed version from Helm chart
53+
54+
### `start-version` - Start Services with Specific Version
55+
56+
Starts services with a specific Materialize version. Use this for testing console
57+
compatibility across different Materialize releases.
58+
59+
```bash
60+
# Using version aliases
61+
./mzcompose run start-version latest
62+
./mzcompose run start-version previous
63+
./mzcompose run start-version older
64+
./mzcompose run start-version self-managed
65+
66+
# Using a direct version tag
67+
./mzcompose run start-version v26.1.1
68+
```
69+
70+
After starting, run tests from the console repo:
71+
```bash
72+
cd ~/dev/console && yarn test:sql
73+
```
74+
75+
Then stop services:
76+
```bash
77+
./mzcompose down -v
78+
```
79+
80+
## Multi-Version Testing (Local)
81+
82+
To test against all versions locally:
83+
84+
```bash
85+
# 1. See available versions
86+
./mzcompose run list-versions
87+
88+
# 2. Test each version
89+
for version in latest previous older self-managed; do
90+
echo "Testing $version..."
91+
./mzcompose down -v
92+
./mzcompose run start-version $version
93+
cd ~/dev/console && yarn test:sql
94+
cd ~/dev/materialize/test/console
95+
done
96+
97+
./mzcompose down -v
98+
```
99+
100+
## GitHub Actions Integration
101+
102+
For CI/CD in the console repository, the typical flow is:
103+
104+
1. Checkout the materialize repo
105+
2. Run `./mzcompose run list-versions` to get the version matrix as JSON
106+
3. Parse the JSON to create a GitHub Actions matrix strategy
107+
4. For each version in the matrix:
108+
- Run `./mzcompose run start-version <version-alias>`
109+
- Run `yarn test:sql`
110+
- Run `./mzcompose down -v`
111+
112+
## How Version Selection Works
113+
114+
The version matrix is generated by `get_published_minor_mz_versions()` which:
115+
116+
1. Gets all Materialize versions from git tags
117+
2. Filters to the latest patch for each minor version (e.g., v26.1.1, not v26.1.0)
118+
3. Verifies Docker images exist for each version
119+
4. Returns the 5 most recent minor versions
120+
121+
For self-managed, it fetches from the Helm chart at
122+
`https://materializeinc.github.io/materialize/index.yaml` and returns the latest version.

‎test/console/mzcompose.py‎

Lines changed: 106 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,13 @@
77
# the Business Source License, use of this software will be governed
88
# by the Apache License, Version 2.0.
99

10-
from materialize.mzcompose.composition import Composition, Service
10+
import sys
11+
12+
from materialize.mzcompose.composition import (
13+
Composition,
14+
Service,
15+
WorkflowArgumentParser,
16+
)
1117
from materialize.mzcompose.services.materialized import Materialized
1218
from materialize.mzcompose.services.mysql import MySql
1319
from materialize.mzcompose.services.postgres import Postgres
@@ -26,11 +32,103 @@
2632

2733

2834
def workflow_default(c: Composition) -> None:
29-
c.up(
30-
"redpanda",
31-
"postgres",
32-
"mysql",
33-
"sql-server",
34-
"materialized",
35-
Service("testdrive", idle=True),
35+
"""
36+
Run all console test workflows.
37+
"""
38+
for name in c.workflows:
39+
if name in ("default", "list-versions", "start-version"):
40+
continue
41+
42+
with c.test_case(name):
43+
c.workflow(name)
44+
45+
46+
def workflow_list_versions(c: Composition) -> None:
47+
"""
48+
Print the version matrix for console testing.
49+
50+
This outputs a JSON object that Console CI can consume to determine
51+
which Materialize versions to test against.
52+
53+
Usage:
54+
./mzcompose run list-versions
55+
"""
56+
import json
57+
58+
from materialize.console_version_matrix import get_console_test_versions
59+
60+
versions = get_console_test_versions()
61+
62+
# Convert to JSON-friendly format (None -> null, MzVersion -> string)
63+
output = {
64+
name: str(version) if version else None for name, version in versions.items()
65+
}
66+
67+
print(json.dumps(output, indent=2))
68+
69+
70+
def workflow_start_version(c: Composition, parser: WorkflowArgumentParser) -> None:
71+
"""
72+
Start Materialize services with a specific version from the version matrix.
73+
74+
This workflow is designed to be called by the console repo to start services
75+
before running console SQL tests. The console repo should call this, then run
76+
its own yarn test:sql, then call ./mzcompose down.
77+
78+
Usage:
79+
./mzcompose run start-version latest
80+
./mzcompose run start-version self-managed
81+
./mzcompose run start-version # defaults to latest
82+
83+
Arguments:
84+
version_alias: One of: latest, previous, older, self-managed
85+
Or a direct version like: v26.1.1
86+
"""
87+
from materialize.console_version_matrix import get_console_test_versions
88+
89+
parser.add_argument(
90+
"version_alias",
91+
nargs="?",
92+
default="latest",
93+
help="Version alias (latest, previous, older, self-managed) or direct version (v26.1.1)",
3694
)
95+
args = parser.parse_args()
96+
version_alias = args.version_alias
97+
98+
# Check if it's a direct version string (starts with 'v')
99+
if version_alias.startswith("v"):
100+
version_str = version_alias
101+
print(f"Using direct version: {version_str}")
102+
else:
103+
# Resolve from version matrix
104+
versions = get_console_test_versions()
105+
106+
if version_alias not in versions:
107+
print(f"❌ Unknown version alias: {version_alias}")
108+
print(f"Available aliases: {', '.join(versions.keys())}")
109+
print("Or provide a direct version like: v26.1.1")
110+
sys.exit(1)
111+
112+
version = versions[version_alias]
113+
version_str = str(version) if version else None
114+
print(f"Starting services for version: {version_alias}")
115+
116+
print(f"Docker image: materialize/materialized:{version_str}")
117+
mz_service = Materialized(
118+
image=f"materialize/materialized:{version_str}" if version_str else None,
119+
system_parameter_defaults={"enable_rbac_checks": "false"},
120+
)
121+
122+
with c.override(mz_service):
123+
c.up(
124+
"redpanda",
125+
"postgres",
126+
"sql-server",
127+
"mysql",
128+
"materialized",
129+
Service("testdrive", idle=True),
130+
)
131+
132+
print("\n✅ Services started successfully")
133+
print("You can now run console SQL tests from the console repo:")
134+
print(" cd ../../../console && yarn test:sql")

0 commit comments

Comments
 (0)