terraform test: New version with local helm chart support

ci/nightly/pipeline.template.yml

@@ -1268,7 +1268,8 @@ steps:
         - ./ci/plugins/mzcompose:
             composition: terraform
             run: aws-temporary
-      branches: "main v*.* lts-v*"
+      # same test runs in test pipeline for changes that touch relevant files
+      branches: "main v*.* self-managed/v*"
 
     - id: terraform-gcp
       label: "Terraform + Helm Chart E2E on GCP"
@@ -1284,7 +1285,8 @@ steps:
         - ./ci/plugins/mzcompose:
             composition: terraform
             run: gcp-temporary
-      branches: "main v*.* lts-v*"
+      # same test runs in test pipeline for changes that touch relevant files
+      branches: "main v*.* self-managed/v*"
 
   - group: "Output consistency"
     key: output-consistency

ci/plugins/mzcompose/hooks/pre-exit

@@ -148,6 +148,6 @@ run down --volumes
 
 if [[ "$BUILDKITE_LABEL" =~ Terraform\ .* ]]; then
   ci_unimportant_heading "terraform: Destroying leftover state in case job was cancelled or timed out..."
-  bin/ci-builder run stable terraform -chdir=test/terraform/aws destroy || true
-  bin/ci-builder run stable terraform -chdir=test/terraform/gcp destroy || true
+  bin/ci-builder run stable terraform -chdir=test/terraform/aws-temporary destroy || true
+  bin/ci-builder run stable terraform -chdir=test/terraform/gcp-temporary destroy || true
 fi

ci/test/pipeline.template.yml

@@ -207,7 +207,7 @@ steps:
           queue: mac
         coverage: skip
         sanitizer: skip
-        branches: "main v*.* lts-v*"
+        branches: "main v*.* self-managed/v*"
 
       - id: lint-deps
         label: Lint dependencies
@@ -742,6 +742,51 @@ steps:
       queue: hetzner-aarch64-4cpu-8gb
     skip: "Version upgrade skips are allowed for LTS releases now"
 
+  - group: E2E
+    key: e2e
+    steps:
+    - id: terraform-aws
+      label: "Terraform + Helm Chart E2E on AWS"
+      artifact_paths: [test/terraform/aws/terraform.tfstate]
+      depends_on: build-aarch64
+      timeout_in_minutes: 1200
+      concurrency: 1
+      concurrency_group: 'terraform-aws'
+      agents:
+        queue: linux-aarch64-small
+      plugins:
+        - ./ci/plugins/scratch-aws-access: ~
+        - ./ci/plugins/mzcompose:
+            composition: terraform
+            run: aws-temporary
+      # same test runs in nightly on main
+      inputs:
+        - misc/helm-charts/
+        - src/orchestratord/
+        - test/terraform/
+      branches: "!main !v*.* !self-managed/v*"
+
+    - id: terraform-gcp
+      label: "Terraform + Helm Chart E2E on GCP"
+      artifact_paths: [test/terraform/aws/terraform.tfstate]
+      depends_on: build-aarch64
+      timeout_in_minutes: 1200
+      concurrency: 1
+      concurrency_group: 'terraform-gcp'
+      agents:
+        queue: linux-aarch64-small
+      plugins:
+        - ./ci/plugins/scratch-aws-access: ~
+        - ./ci/plugins/mzcompose:
+            composition: terraform
+            run: gcp-temporary
+      # same test runs in nightly on main
+      inputs:
+        - misc/helm-charts/
+        - src/orchestratord/
+        - test/terraform/
+      branches: "!main !v*.* !self-managed/v*"
+
   - id: deploy-website
     label: Deploy website
     depends_on: lint-docs

test/terraform/aws-persistent/main.tf

@@ -16,22 +16,41 @@ resource "random_password" "db_password" {
   special = false
 }
 
+variable "operator_version" {
+  type    = string
+  default = "v25.2.0-beta.1"
+}
+
+variable "orchestratord_version" {
+  type    = string
+  default = "v0.130.3"
+}
+
 module "materialize_infrastructure" {
-  source = "git::https://github.com/MaterializeInc/terraform-aws-materialize.git?ref=v0.2.0"
+  source = "git::https://github.com/MaterializeInc/terraform-aws-materialize.git?ref=v0.2.5"
 
   # Basic settings
   namespace    = "aws-persistent"
   environment  = "dev"
   install_materialize_operator = true
-
-  helm_values = {
-      defaultReplicationFactor = {
-          system = 1
-          probe = 1
-          support = 1
-          analytics = 1
-      }
-  }
+  use_local_chart = true
+  helm_chart = "materialize-operator-v25.2.0-beta.1.tgz"
+  operator_version = var.operator_version
+  orchestratord_version = var.orchestratord_version
+
+  # TODO: Doesn't seem to work yet
+  # helm_values = {
+  #   operator = {
+  #     clusters = {
+  #       defaultReplicationFactor = {
+  #           system = 1
+  #           probe = 1
+  #           support = 1
+  #           analytics = 1
+  #       }
+  #     }
+  #   }
+  # }
 
   # VPC Configuration
   vpc_cidr             = "10.0.0.0/16"
@@ -42,7 +61,7 @@ module "materialize_infrastructure" {
 
   # EKS Configuration
   cluster_version           = "1.31"
-  node_group_instance_types = ["c7a.2xlarge"]
+  node_group_instance_types = ["r8g.2xlarge"]
   node_group_desired_size   = 2
   node_group_min_size       = 1
   node_group_max_size       = 3

test/terraform/aws-temporary/main.tf

@@ -16,24 +16,43 @@ resource "random_password" "db_password" {
   special = false
 }
 
+variable "operator_version" {
+  type    = string
+  default = "v25.2.0-beta.1.tgz"
+}
+
+variable "orchestratord_version" {
+  type    = string
+  default = "v0.130.3"
+}
+
 module "materialize_infrastructure" {
-  source = "git::https://github.com/MaterializeInc/terraform-aws-materialize.git?ref=v0.2.0"
+  source = "git::https://github.com/MaterializeInc/terraform-aws-materialize.git?ref=v0.2.5"
 
   # Basic settings
   # The namespace and environment variables are used to construct the names of the resources
   # e.g. ${namespace}-${environment}-eks and etc.
   namespace    = "aws-test"
   environment  = "dev"
   install_materialize_operator = true
-
-  helm_values = {
-      defaultReplicationFactor = {
-          system = 1
-          probe = 1
-          support = 1
-          analytics = 1
-      }
-  }
+  use_local_chart = true
+  helm_chart = "materialize-operator-v25.2.0-beta.1.tgz"
+  operator_version = var.operator_version
+  orchestratord_version = var.orchestratord_version
+
+  # TODO: Doesn't seem to work yet
+  # helm_values = {
+  #   operator = {
+  #     clusters = {
+  #       defaultReplicationFactor = {
+  #           system = 1
+  #           probe = 1
+  #           support = 1
+  #           analytics = 1
+  #       }
+  #     }
+  #   }
+  # }
 
   # VPC Configuration
   vpc_cidr             = "10.0.0.0/16"
@@ -44,7 +63,7 @@ module "materialize_infrastructure" {
 
   # EKS Configuration
   cluster_version           = "1.31"
-  node_group_instance_types = ["c7a.2xlarge"]
+  node_group_instance_types = ["r8g.2xlarge"]
   node_group_desired_size   = 2
   node_group_min_size       = 1
   node_group_max_size       = 3

test/terraform/gcp-temporary/main.tf

@@ -44,11 +44,13 @@ module "materialize" {
   install_materialize_operator = true
 
   helm_values = {
-      defaultReplicationFactor = {
-          system = 1
-          probe = 1
-          support = 1
-          analytics = 1
+      clusters = {
+        defaultReplicationFactor = {
+            system = 1
+            probe = 1
+            support = 1
+            analytics = 1
+        }
       }
   }
 }

test/terraform/mzcompose.py

@@ -158,11 +158,21 @@ def setup(self, prefix: str, setup: bool, tag: str) -> None:
             )
             return
 
+        vars = [
+            "-var",
+            "operator_version=v25.2.0-beta.1",
+            "-var",
+            f"orchestratord_version={tag}",
+        ]
+
         print("--- Setup")
+        spawn.runv(
+            ["helm", "package", "../../../misc/helm-charts/operator/"], cwd=self.path
+        )
         spawn.runv(["terraform", "init"], cwd=self.path)
         spawn.runv(["terraform", "validate"], cwd=self.path)
-        spawn.runv(["terraform", "plan"], cwd=self.path)
-        spawn.runv(["terraform", "apply", "-auto-approve"], cwd=self.path)
+        spawn.runv(["terraform", "plan", *vars], cwd=self.path)
+        spawn.runv(["terraform", "apply", "-auto-approve", *vars], cwd=self.path)
 
         metadata_backend_url = spawn.capture(
             ["terraform", "output", "-raw", "metadata_backend_url"], cwd=self.path
@@ -420,6 +430,14 @@ def connect(self, c: Composition) -> None:
         )
         time.sleep(10)
 
+        with psycopg.connect(
+            "postgres://mz_system:materialize@127.0.0.1:6877/materialize",
+            autocommit=True,
+        ) as conn:
+            with conn.cursor() as cur:
+                # Required for some testdrive tests
+                cur.execute("ALTER CLUSTER mz_system SET (REPLICATION FACTOR 1)")
+
         c.up("testdrive", persistent=True)
         c.testdrive(
             dedent(
@@ -573,11 +591,10 @@ def workflow_aws_temporary(c: Composition, parser: WorkflowArgumentParser) -> No
                         MZ_ROOT / "misc" / "helm-charts" / "operator" / "Chart.yaml"
                     ) as f:
                         content = yaml.load(f, Loader=yaml.Loader)
-                        content["version"]
-                    # TODO: Reenable when we can pass the helm-chart path in directly
-                    # assert version.endswith(
-                    #     f", helm chart: {helm_chart_version})"
-                    # ), f"Actual version: {version}, expected to contain {helm_chart_version}"
+                        helm_chart_version = content["version"]
+                    assert version.endswith(
+                        f", helm chart: {helm_chart_version})"
+                    ), f"Actual version: {version}, expected to contain {helm_chart_version}"
 
             c.run_testdrive_files(*args.files)
     finally: