Skip to content

Commit 19a17fd

Browse files
authored
Merge pull request #31214 from def-/pr-gcp-terraform
GCP terraform + helm-chart test
2 parents 15d12a2 + fcc0efd commit 19a17fd

File tree

7 files changed

+591
-67
lines changed

7 files changed

+591
-67
lines changed

bin/ci-builder

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ case "$cmd" in
188188
--env COMMON_ANCESTOR_OVERRIDE
189189
--env CONFLUENT_CLOUD_DEVEX_KAFKA_PASSWORD
190190
--env CONFLUENT_CLOUD_DEVEX_KAFKA_USERNAME
191+
--env GCP_SERVICE_ACCOUNT_JSON
191192
--env GITHUB_TOKEN
192193
--env GPG_KEY
193194
--env LAUNCHDARKLY_API_TOKEN

ci/builder/Dockerfile

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,17 @@ RUN curl -fsSL https://github.com/deb-s3/deb-s3/releases/download/0.11.3/deb-s3-
326326
RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-$ARCH_GCC-2.17.2.zip" > awscli.zip \
327327
&& unzip awscli.zip \
328328
&& ./aws/install \
329-
&& rm -rf aws
329+
&& rm -rf aws awscli.zip
330+
331+
# Install the gcloud CLI.
332+
333+
RUN arch_gcloud=$(echo "$ARCH_GCC" | sed -e "s/aarch64/arm/" -e "s/amd64/x86_64/") \
334+
&& curl -fsSL "https://storage.googleapis.com/cloud-sdk-release/google-cloud-cli-507.0.0-linux-$arch_gcloud.tar.gz" > gcloud.tar.gz \
335+
&& tar -xzf gcloud.tar.gz -C /opt \
336+
&& rm gcloud.tar.gz \
337+
&& CLOUDSDK_CORE_DISABLE_PROMPTS=1 /opt/google-cloud-sdk/install.sh --path-update false \
338+
&& /opt/google-cloud-sdk/bin/gcloud config set disable_usage_reporting false \
339+
&& /opt/google-cloud-sdk/bin/gcloud components install gke-gcloud-auth-plugin
330340

331341
# Install docs site dependencies. These are towards the end for the same reason
332342
# as the Python dependencies. These are only supported on x86_64 at the moment.
@@ -410,7 +420,7 @@ ENV TARGET_CC=$CC
410420
ENV TARGET_CXX=$CXX
411421
ENV TARGET_CXXSTDLIB=static=stdc++
412422
ENV TARGET_RANLIB=$RANLIB
413-
ENV PATH=/opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin:$PATH
423+
ENV PATH=/opt/google-cloud-sdk/bin:/opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin:$PATH
414424
ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-unknown-linux-gnu-cc
415425
ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-unknown-linux-gnu-cc
416426
ENV CARGO_TARGET_DIR=/mnt/build

ci/nightly/pipeline.template.yml

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1228,15 +1228,32 @@ steps:
12281228
depends_on: build-aarch64
12291229
timeout_in_minutes: 1200
12301230
concurrency: 1
1231-
concurrency_group: 'terraform'
1231+
concurrency_group: 'terraform-aws'
12321232
agents:
12331233
queue: linux-aarch64-small
12341234
plugins:
12351235
- ./ci/plugins/scratch-aws-access: ~
12361236
- ./ci/plugins/mzcompose:
12371237
composition: terraform
1238+
run: aws
12381239
branches: "main v*.* lts-v*"
12391240

1241+
- id: terraform-gcp
1242+
label: "Terraform + Helm Chart E2E on GCP"
1243+
artifact_paths: [test/terraform/aws/terraform.tfstate]
1244+
depends_on: build-aarch64
1245+
timeout_in_minutes: 1200
1246+
concurrency: 1
1247+
concurrency_group: 'terraform-gcp'
1248+
agents:
1249+
queue: linux-aarch64-small
1250+
plugins:
1251+
- ./ci/plugins/scratch-aws-access: ~
1252+
- ./ci/plugins/mzcompose:
1253+
composition: terraform
1254+
run: gcp
1255+
#branches: "main v*.* lts-v*"
1256+
12401257
- group: "Output consistency"
12411258
key: output-consistency
12421259
steps:

ci/plugins/mzcompose/hooks/pre-exit

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ timeout 300 buildkite-agent artifact upload "$artifacts_str" || true
8585
bin/ci-builder run stable bin/ci-annotate-errors --test-cmd="$(cat test_cmd)" --test-desc="$(cat test_desc)" "${artifacts[@]}" > ci-annotate-errors.log || CI_ANNOTATE_ERRORS_RESULT=$?
8686
buildkite-agent artifact upload "ci-annotate-errors.log"
8787

88-
if [ ! -s services.log ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] && [ "$BUILDKITE_LABEL" != "Mz E2E Test" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for DFR)" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for CTF)" ] && [ "$BUILDKITE_LABEL" != "QA Canary Environment Base Load" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on AWS" ]; then
88+
if [ ! -s services.log ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] && [ "$BUILDKITE_LABEL" != "Mz E2E Test" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for DFR)" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for CTF)" ] && [ "$BUILDKITE_LABEL" != "QA Canary Environment Base Load" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on AWS" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on GCP" ]; then
8989
echo "+++ services.log is empty, failing"
9090
exit 1
9191
fi

ci/test/lint-main/checks/check-mzcompose-files.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ check_default_workflow_references_others() {
4545
-not -wholename "./test/canary-environment/mzcompose.py" `# Only run manually` \
4646
-not -wholename "./test/ssh-connection/mzcompose.py" `# Handled differently` \
4747
-not -wholename "./test/scalability/mzcompose.py" `# Other workflows are for manual usage` \
48+
-not -wholename "./test/terraform/mzcompose.py" `# Handled differently` \
4849
)
4950

5051
for file in "${MZCOMPOSE_TEST_FILES[@]}"; do

test/terraform/gcp/main.tf

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Copyright Materialize, Inc. and contributors. All rights reserved.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the LICENSE file at the root of this repository.
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0.
9+
10+
terraform {
11+
required_version = ">= 1.0"
12+
13+
required_providers {
14+
google = {
15+
source = "hashicorp/google"
16+
version = ">= 6.0"
17+
}
18+
}
19+
}
20+
21+
provider "google" {
22+
project = var.project_id
23+
region = var.region
24+
}
25+
26+
module "materialize" {
27+
source = "github.com/MaterializeInc/terraform-google-materialize?ref=v0.1.1"
28+
29+
project_id = var.project_id
30+
region = var.region
31+
prefix = "tf-gcp-test"
32+
33+
database_config = {
34+
tier = "db-custom-2-4096"
35+
version = "POSTGRES_15"
36+
password = var.database_password
37+
}
38+
39+
labels = {
40+
environment = "simple"
41+
example = "true"
42+
}
43+
44+
install_materialize_operator = false
45+
}
46+
47+
variable "project_id" {
48+
description = "GCP Project ID"
49+
type = string
50+
default = "materialize-ci"
51+
}
52+
53+
variable "region" {
54+
description = "GCP Region"
55+
type = string
56+
default = "us-east1"
57+
}
58+
59+
variable "database_password" {
60+
description = "Password for Cloud SQL database user"
61+
default = "your-strong-password"
62+
type = string
63+
sensitive = true
64+
}
65+
66+
output "gke_cluster" {
67+
description = "GKE cluster details"
68+
value = module.materialize.gke_cluster
69+
sensitive = true
70+
}
71+
72+
output "service_accounts" {
73+
description = "Service account details"
74+
value = module.materialize.service_accounts
75+
}
76+
77+
output "connection_strings" {
78+
description = "Connection strings for metadata and persistence backends"
79+
value = module.materialize.connection_strings
80+
sensitive = true
81+
}

0 commit comments

Comments
 (0)