Skip to content

Commit ebb5d9a

Browse files
def-def-
authored
Merge pull request #34139 from def-/pr-terraform-operator-version
terraform tests: Use correct operator version & ci: Make terraform and orchestratord tests depend on devel tags
2 parents ee6d0aa + 9dec108 commit ebb5d9a

File tree

10 files changed

+68
-34
lines changed

10 files changed

+68
-34
lines changed

‎ci/mkpipeline.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -624,6 +624,7 @@ def trim_test_selection_id(pipeline: Any, step_ids_to_run: set[int]) -> None:
624624
"build-aarch64",
625625
"build-x86_64-lto",
626626
"build-aarch64-lto",
627+
"devel-docker-tags",
627628
)
628629
and not step.get("async")
629630
):
@@ -646,6 +647,7 @@ def trim_test_selection_name(pipeline: Any, steps_to_run: set[str]) -> None:
646647
"build-aarch64",
647648
"build-x86_64-lto",
648649
"build-aarch64-lto",
650+
"devel-docker-tags",
649651
)
650652
and not step.get("async")
651653
):
@@ -921,6 +923,11 @@ def trim_builds(
921923
elif step.get("id") == "upload-debug-symbols-aarch64":
922924
if hash_check[Arch.AARCH64][1]:
923925
step["skip"] = True
926+
elif step.get("id") == "devel-docker-tags":
927+
step["concurrency"] = 1
928+
step["concurrency_group"] = (
929+
f"devel-docker-tags/{hash_check[Arch.X86_64][0]}"
930+
)
924931

925932

926933
_github_changed_files: set[str] | None = None

‎ci/nightly/pipeline.template.yml‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,24 @@ steps:
7474
queue: hetzner-aarch64-16cpu-32gb
7575
branches: "main"
7676

77+
- id: devel-docker-tags
78+
label: Tag development docker images
79+
command: bin/ci-builder run min bin/pyactivate -m ci.test.dev_tag
80+
inputs:
81+
- "*"
82+
depends_on:
83+
- build-x86_64
84+
- build-aarch64
85+
timeout_in_minutes: 20
86+
agents:
87+
queue: linux-aarch64-small
88+
coverage: skip
89+
# Fortify against intermittent DockerHub/GHCR issues
90+
retry:
91+
automatic:
92+
- exit_status: 1
93+
limit: 2
94+
7795
- group: Linters
7896
key: linters
7997
steps:
@@ -1442,7 +1460,7 @@ steps:
14421460
- id: terraform-aws
14431461
label: "Terraform + Helm Chart E2E on AWS"
14441462
artifact_paths: [test/terraform/aws-temporary/terraform.tfstate, "mz_debug_*.zip"]
1445-
depends_on: build-aarch64
1463+
depends_on: devel-docker-tags
14461464
timeout_in_minutes: 120
14471465
concurrency: 1
14481466
concurrency_group: 'terraform-aws'
@@ -1461,7 +1479,7 @@ steps:
14611479
- id: terraform-aws-upgrade
14621480
label: "Terraform + Helm Chart Upgrade on AWS"
14631481
artifact_paths: [test/terraform/aws-upgrade/terraform.tfstate, "mz_debug_*.zip"]
1464-
depends_on: build-aarch64
1482+
depends_on: devel-docker-tags
14651483
timeout_in_minutes: 120
14661484
concurrency: 1
14671485
concurrency_group: 'terraform-upgrade-aws'
@@ -1485,7 +1503,7 @@ steps:
14851503
- id: terraform-gcp
14861504
label: "Terraform + Helm Chart E2E on GCP"
14871505
artifact_paths: [test/terraform/gcp-temporary/terraform.tfstate, "mz_debug_*.zip"]
1488-
depends_on: build-aarch64
1506+
depends_on: devel-docker-tags
14891507
timeout_in_minutes: 120
14901508
concurrency: 1
14911509
concurrency_group: 'terraform-gcp'
@@ -1504,7 +1522,7 @@ steps:
15041522
- id: terraform-azure
15051523
label: "Terraform + Helm Chart E2E on Azure"
15061524
artifact_paths: [test/terraform/azure-temporary/terraform.tfstate, "mz_debug_*.zip"]
1507-
depends_on: build-aarch64
1525+
depends_on: devel-docker-tags
15081526
timeout_in_minutes: 120
15091527
concurrency: 1
15101528
concurrency_group: 'terraform-azure'
@@ -2346,7 +2364,7 @@ steps:
23462364
steps:
23472365
- id: orchestratord-defaults
23482366
label: "Orchestratord test (defaults from documentation)"
2349-
depends_on: build-aarch64
2367+
depends_on: devel-docker-tags
23502368
timeout_in_minutes: 120
23512369
plugins:
23522370
- ./ci/plugins/mzcompose:
@@ -2358,7 +2376,7 @@ steps:
23582376

23592377
- id: orchestratord-default-properties
23602378
label: "Orchestratord test (defaults for properties)"
2361-
depends_on: build-aarch64
2379+
depends_on: devel-docker-tags
23622380
timeout_in_minutes: 120
23632381
plugins:
23642382
- ./ci/plugins/mzcompose:
@@ -2370,7 +2388,7 @@ steps:
23702388

23712389
- id: orchestratord-individual
23722390
label: "Orchestratord test (individual properties)"
2373-
depends_on: build-aarch64
2391+
depends_on: devel-docker-tags
23742392
timeout_in_minutes: 120
23752393
plugins:
23762394
- ./ci/plugins/mzcompose:
@@ -2394,7 +2412,7 @@ steps:
23942412

23952413
- id: orchestratord-upgrade-individual
23962414
label: "Orchestratord test (upgrade, individual props)"
2397-
depends_on: build-aarch64
2415+
depends_on: devel-docker-tags
23982416
timeout_in_minutes: 120
23992417
plugins:
24002418
- ./ci/plugins/mzcompose:
@@ -2409,7 +2427,7 @@ steps:
24092427

24102428
- id: orchestratord-upgrade-combine
24112429
label: "Orchestratord test (upgrade, combine props)"
2412-
depends_on: build-aarch64
2430+
depends_on: devel-docker-tags
24132431
timeout_in_minutes: 120
24142432
plugins:
24152433
- ./ci/plugins/mzcompose:
@@ -2424,7 +2442,7 @@ steps:
24242442

24252443
- id: orchestratord-upgrade-chain-individual
24262444
label: "Orchestratord test (upgrade chain, individual props)"
2427-
depends_on: build-aarch64
2445+
depends_on: devel-docker-tags
24282446
timeout_in_minutes: 120
24292447
plugins:
24302448
- ./ci/plugins/mzcompose:
@@ -2439,7 +2457,7 @@ steps:
24392457

24402458
- id: orchestratord-upgrade-chain-combine
24412459
label: "Orchestratord test (upgrade chain, combine props)"
2442-
depends_on: build-aarch64
2460+
depends_on: devel-docker-tags
24432461
timeout_in_minutes: 120
24442462
plugins:
24452463
- ./ci/plugins/mzcompose:

‎ci/test/pipeline.template.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ steps:
140140
agents:
141141
queue: linux-aarch64-small
142142
coverage: skip
143-
# Fortify against intermittent DockerHub issues
143+
# Fortify against intermittent DockerHub/GHCR issues
144144
retry:
145145
automatic:
146146
- exit_status: 1

‎misc/python/materialize/mzbuild.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1451,10 +1451,17 @@ def publish_multiarch_images(
14511451
names = set(image.image.name for image in images)
14521452
assert len(names) == 1, "dependency sets did not contain identical images"
14531453
name = images[0].image.docker_name(tag)
1454-
spawn.runv(
1455-
["docker", "manifest", "create", name, *(image.spec() for image in images)]
1456-
)
1457-
spawn.runv(["docker", "manifest", "push", name])
1454+
if not is_docker_image_pushed(name):
1455+
spawn.runv(
1456+
[
1457+
"docker",
1458+
"manifest",
1459+
"create",
1460+
name,
1461+
*(image.spec() for image in images),
1462+
]
1463+
)
1464+
spawn.runv(["docker", "manifest", "push", name])
14581465
print(f"--- Nofifying for tag {tag}")
14591466
markdown = f"""Pushed images with Docker tag `{tag}`"""
14601467
spawn.runv(

‎test/terraform/aws-persistent/main.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ resource "random_password" "db_password" {
1818

1919
variable "operator_version" {
2020
type = string
21-
default = "v26.0.0-beta.1.tgz"
21+
default = "v26.0.0-beta.1"
2222
}
2323

2424
variable "orchestratord_version" {
@@ -34,7 +34,7 @@ module "materialize_infrastructure" {
3434
environment = "dev"
3535
install_materialize_operator = true
3636
use_local_chart = true
37-
helm_chart = "materialize-operator-v26.0.0-beta.1.tgz"
37+
helm_chart = "materialize-operator-${var.operator_version}.tgz"
3838
operator_version = var.operator_version
3939
orchestratord_version = var.orchestratord_version
4040

‎test/terraform/aws-temporary/main.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ resource "random_password" "db_password" {
4242

4343
variable "operator_version" {
4444
type = string
45-
default = "v26.0.0-beta.1.tgz"
45+
default = "v26.0.0-beta.1"
4646
}
4747

4848
variable "orchestratord_version" {
@@ -67,7 +67,7 @@ module "materialize_infrastructure" {
6767

6868
install_materialize_operator = true
6969
use_local_chart = true
70-
helm_chart = "materialize-operator-v26.0.0-beta.1.tgz"
70+
helm_chart = "materialize-operator-${var.operator_version}.tgz"
7171
operator_version = var.operator_version
7272
orchestratord_version = var.orchestratord_version
7373

‎test/terraform/aws-upgrade/main.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ resource "random_password" "db_password" {
4242

4343
variable "operator_version" {
4444
type = string
45-
default = "v26.0.0-beta.1.tgz"
45+
default = "v26.0.0-beta.1"
4646
}
4747

4848
variable "orchestratord_version" {
@@ -67,7 +67,7 @@ module "materialize_infrastructure" {
6767

6868
install_materialize_operator = true
6969
use_local_chart = true
70-
helm_chart = "materialize-operator-v26.0.0-beta.1.tgz"
70+
helm_chart = "materialize-operator-${var.operator_version}.tgz"
7171
operator_version = var.operator_version
7272
orchestratord_version = var.orchestratord_version
7373

‎test/terraform/azure-temporary/main.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ resource "random_password" "pass" {
4545

4646
variable "operator_version" {
4747
type = string
48-
default = "v26.0.0-beta.1.tgz"
48+
default = "v26.0.0-beta.1"
4949
}
5050

5151
variable "orchestratord_version" {
@@ -67,7 +67,7 @@ module "materialize" {
6767

6868
install_materialize_operator = true
6969
use_local_chart = true
70-
helm_chart = "materialize-operator-v26.0.0-beta.1.tgz"
70+
helm_chart = "materialize-operator-${var.operator_version}.tgz"
7171
operator_version = var.operator_version
7272
orchestratord_version = var.orchestratord_version
7373

‎test/terraform/gcp-temporary/main.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ module "materialize" {
5454

5555
install_materialize_operator = true
5656
use_local_chart = true
57-
helm_chart = "materialize-operator-v26.0.0-beta.1.tgz"
57+
helm_chart = "materialize-operator-${var.operator_version}.tgz"
5858
operator_version = var.operator_version
5959
orchestratord_version = var.orchestratord_version
6060

@@ -97,7 +97,7 @@ variable "database_password" {
9797

9898
variable "operator_version" {
9999
type = string
100-
default = "v26.0.0-beta.1.tgz"
100+
default = "v26.0.0-beta.1"
101101
}
102102

103103
variable "orchestratord_version" {

‎test/terraform/mzcompose.py‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,12 @@ def get_tag(tag: str | None) -> str:
196196
return tag or f"v{ci_util.get_mz_version()}--pr.g{git.rev_parse('HEAD')}"
197197

198198

199+
def get_operator_version() -> str:
200+
with open(MZ_ROOT / "misc" / "helm-charts" / "operator" / "Chart.yaml") as f:
201+
content = yaml.load(f, Loader=yaml.Loader)
202+
return content["version"]
203+
204+
199205
def build_mz_debug_async(env: dict[str, str] | None = None) -> threading.Thread:
200206
def run():
201207
spawn.capture(
@@ -566,11 +572,7 @@ def test(
566572
cur.execute("SELECT mz_version()")
567573
version = cur.fetchall()[0][0]
568574
assert version.startswith(tag.split("--")[0] + " ")
569-
with open(
570-
MZ_ROOT / "misc" / "helm-charts" / "operator" / "Chart.yaml"
571-
) as f:
572-
content = yaml.load(f, Loader=yaml.Loader)
573-
helm_chart_version = content["version"]
575+
helm_chart_version = get_operator_version()
574576
assert version.endswith(
575577
f", helm chart: {helm_chart_version})"
576578
), f"Actual version: {version}, expected to contain {helm_chart_version}"
@@ -685,7 +687,7 @@ def setup(
685687

686688
vars = [
687689
"-var",
688-
"operator_version=v26.0.0-beta.1",
690+
f"operator_version={get_operator_version()}",
689691
]
690692
if orchestratord_override:
691693
vars += [
@@ -1168,7 +1170,7 @@ def workflow_gcp_temporary(c: Composition, parser: WorkflowArgumentParser) -> No
11681170

11691171
vars = [
11701172
"-var",
1171-
"operator_version=v26.0.0-beta.1",
1173+
f"operator_version={get_operator_version()}",
11721174
]
11731175
if args.orchestratord_override:
11741176
vars += [
@@ -1280,7 +1282,7 @@ def workflow_azure_temporary(c: Composition, parser: WorkflowArgumentParser) ->
12801282

12811283
vars = [
12821284
"-var",
1283-
"operator_version=v26.0.0-beta.1",
1285+
f"operator_version={get_operator_version()}",
12841286
]
12851287
if args.orchestratord_override:
12861288
vars += [

0 commit comments

Comments
 (0)