Skip to content

Commit 5caa29b

Browse files
salmanyglasnt
authored and
niharika-98
committed
docs: Add MKC Samples for Creating Connectors (terraform-google-modules#878)
* docs: Add Managed Kafka Connect terraform sample for Clusters * Add sample code for creating a Kafka Connect cluster * Update tags in main.tf Corrected tags. * Addressed PR comments. * Add comment to clarify memory_bytes value. * Fix incorrect Kafka cluster reference. * docs: Add Managed Kafka Connect terraform sample for Connectors Adds Terraform samples for Managed Kafka Connect to create connectors for BigQuery, Cloud Storage, MirrorMaker, Pub/Sub Sink, and Pub/Sub Source. * Fix typo in memory_bytes for connectors * Fixed formatting. * Fix formatting * Fix snippet tags. * Fix tags * Adding workaround for subnet resource deletion. * Fixed errors and formatting. * Disable tests and modify subnet resources. * Updated connector sample configs. * Fix formatting. * Add comment for MM connector target cluster. * Change MM2 example to include 2 GMK clusters. * Fix connector name to match id. * Fix connector bootstrap server addresses. * Fix bootstrap server addresses and add comment. * Fix whitespaces and formatting. * Add newline. * Add comments for access to MM2 clusters and other fixes. * Formatting fix. * Applying suggestions. --------- Co-authored-by: Katie McLaughlin <[email protected]>
1 parent 9496d6c commit 5caa29b

File tree

10 files changed

+648
-0
lines changed

10 files changed

+648
-0
lines changed

‎managedkafka/managedkafka_create_connector_bigquery_sink/main.tf‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
# [START managedkafka_create_connector_bigquery_sink_parent]
18+
19+
resource "google_managed_kafka_cluster" "default" {
20+
project = data.google_project.default.project_id
21+
cluster_id = "my-cluster-id"
22+
location = "us-central1"
23+
capacity_config {
24+
vcpu_count = 3
25+
memory_bytes = 3221225472 # 3 GiB
26+
}
27+
gcp_config {
28+
access_config {
29+
network_configs {
30+
subnet = google_compute_subnetwork.default.id
31+
}
32+
}
33+
}
34+
}
35+
36+
resource "google_managed_kafka_connect_cluster" "default" {
37+
provider = google-beta
38+
project = data.google_project.default.project_id
39+
connect_cluster_id = "my-connect-cluster-id"
40+
location = "us-central1"
41+
kafka_cluster = google_managed_kafka_cluster.default.id
42+
capacity_config {
43+
vcpu_count = 12
44+
memory_bytes = 12884901888 # 12 GiB
45+
}
46+
gcp_config {
47+
access_config {
48+
network_configs {
49+
primary_subnet = google_compute_subnetwork.default.id
50+
}
51+
}
52+
}
53+
}
54+
55+
# Note: Due to a known issue, network attachment resources may not be
56+
# properly deleted, which can cause 'terraform destroy' to hang. It is
57+
# recommended to destroy network resources separately from the Kafka
58+
# Connect resources.
59+
# The documentation elaborates further on the recommended approach.
60+
# [START managedkafka_subnetwork]
61+
resource "google_compute_subnetwork" "default" {
62+
name = "test-subnetwork"
63+
ip_cidr_range = "10.2.0.0/16"
64+
region = "us-central1"
65+
network = google_compute_network.default.id
66+
}
67+
68+
resource "google_compute_network" "default" {
69+
name = "test-network"
70+
auto_create_subnetworks = false
71+
}
72+
# [END managedkafka_subnetwork]
73+
74+
# [START managedkafka_create_connector_bigquery_sink]
75+
resource "google_managed_kafka_connector" "example-bigquery-sink-connector" {
76+
project = data.google_project.default.project_id
77+
connector_id = "my-bigquery-sink-connector"
78+
connect_cluster = google_managed_kafka_connect_cluster.default.connect_cluster_id
79+
location = "us-central1"
80+
81+
configs = {
82+
"name" = "my-bigquery-sink-connector"
83+
"project" = "GCP_PROJECT_ID"
84+
"topics" = "GMK_TOPIC_ID"
85+
"tasks.max" = "3"
86+
"connector.class" = "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector"
87+
"key.converter" = "org.apache.kafka.connect.storage.StringConverter"
88+
"value.converter" = "org.apache.kafka.connect.json.JsonConverter"
89+
"value.converter.schemas.enable" = "false"
90+
"defaultDataset" = "BQ_DATASET_ID"
91+
}
92+
93+
provider = google-beta
94+
}
95+
# [END managedkafka_create_connector_bigquery_sink]
96+
97+
data "google_project" "default" {
98+
}
99+
100+
# [END managedkafka_create_connector_bigquery_sink_parent]

‎managedkafka/managedkafka_create_connector_bigquery_sink/test.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
apiVersion: blueprints.cloud.google.com/v1alpha1
16+
kind: BlueprintTest
17+
metadata:
18+
name: managedkafka_create_connector_bigquery_sink
19+
spec:
20+
# This test is being skipped due to a known issue with leaking Network
21+
# Attachment resources which prevents `terraform destroy` from completing
22+
# successfully in the test environment.
23+
skip: true

‎managedkafka/managedkafka_create_connector_cloud_storage_sink/main.tf‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
# [START managedkafka_create_connector_pubsub_source_parent]
18+
19+
resource "google_managed_kafka_cluster" "default" {
20+
project = data.google_project.default.project_id
21+
cluster_id = "my-cluster-id"
22+
location = "us-central1"
23+
capacity_config {
24+
vcpu_count = 3
25+
memory_bytes = 3221225472 # 3 GiB
26+
}
27+
gcp_config {
28+
access_config {
29+
network_configs {
30+
subnet = google_compute_subnetwork.default.id
31+
}
32+
}
33+
}
34+
}
35+
36+
resource "google_managed_kafka_connect_cluster" "default" {
37+
provider = google-beta
38+
project = data.google_project.default.project_id
39+
connect_cluster_id = "my-connect-cluster-id"
40+
location = "us-central1"
41+
kafka_cluster = google_managed_kafka_cluster.default.id
42+
capacity_config {
43+
vcpu_count = 12
44+
memory_bytes = 12884901888 # 12 GiB
45+
}
46+
gcp_config {
47+
access_config {
48+
network_configs {
49+
primary_subnet = google_compute_subnetwork.default.id
50+
}
51+
}
52+
}
53+
}
54+
55+
# Note: Due to a known issue, network attachment resources may not be
56+
# properly deleted, which can cause 'terraform destroy' to hang. It is
57+
# recommended to destroy network resources separately from the Kafka
58+
# Connect resources.
59+
# The documentation elaborates further on the recommended approach.
60+
# [START managedkafka_subnetwork]
61+
resource "google_compute_subnetwork" "default" {
62+
name = "test-subnetwork"
63+
ip_cidr_range = "10.2.0.0/16"
64+
region = "us-central1"
65+
network = google_compute_network.default.id
66+
}
67+
68+
resource "google_compute_network" "default" {
69+
name = "test-network"
70+
auto_create_subnetworks = false
71+
}
72+
# [END managedkafka_subnetwork]
73+
74+
# [START managedkafka_create_connector_cloud_storage_sink]
75+
resource "google_managed_kafka_connector" "example-cloud-storage-sink-connector" {
76+
project = data.google_project.default.project_id
77+
connector_id = "my-gcs-sink-connector"
78+
connect_cluster = google_managed_kafka_connect_cluster.default.connect_cluster_id
79+
location = "us-central1"
80+
81+
configs = {
82+
"connector.class" = "io.aiven.kafka.connect.gcs.GcsSinkConnector"
83+
"tasks.max" = "3"
84+
"topics" = "GMK_TOPIC_ID"
85+
"gcs.bucket.name" = "GCS_BUCKET_NAME"
86+
"gcs.credentials.default" = "true"
87+
"format.output.type" = "json"
88+
"name" = "my-gcs-sink-connector"
89+
"value.converter" = "org.apache.kafka.connect.json.JsonConverter"
90+
"value.converter.schemas.enable" = "false"
91+
"key.converter" = "org.apache.kafka.connect.storage.StringConverter"
92+
}
93+
provider = google-beta
94+
}
95+
# [END managedkafka_create_connector_cloud_storage_sink]
96+
97+
data "google_project" "default" {
98+
}
99+
100+
# [END managedkafka_create_connector_pubsub_source_parent]

‎managedkafka/managedkafka_create_connector_cloud_storage_sink/test.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
apiVersion: blueprints.cloud.google.com/v1alpha1
16+
kind: BlueprintTest
17+
metadata:
18+
name: managedkafka_create_connector_cloud_storage_sink
19+
spec:
20+
# This test is being skipped due to a known issue with leaking Network
21+
# Attachment resources which prevents `terraform destroy` from completing
22+
# successfully in the test environment.
23+
skip: true

‎managedkafka/managedkafka_create_connector_mirrormaker2_source/main.tf‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
# [START managedkafka_create_connector_mirrormaker2_source_parent]
18+
19+
# Define the target Kafka cluster. This is where data will be replicated to.
20+
resource "google_managed_kafka_cluster" "target" {
21+
project = data.google_project.default.project_id
22+
cluster_id = "mm2-target-cluster-id"
23+
location = "us-central1"
24+
capacity_config {
25+
vcpu_count = 3
26+
memory_bytes = 3221225472 # 3 GiB
27+
}
28+
gcp_config {
29+
access_config {
30+
network_configs {
31+
subnet = google_compute_subnetwork.default.id
32+
}
33+
}
34+
}
35+
}
36+
37+
# Define the source Kafka cluster.
38+
resource "google_managed_kafka_cluster" "source" {
39+
project = data.google_project.default.project_id
40+
cluster_id = "mm2-source-cluster-id"
41+
location = "us-central1"
42+
capacity_config {
43+
vcpu_count = 3
44+
memory_bytes = 3221225472 # 3 GiB
45+
}
46+
gcp_config {
47+
access_config {
48+
network_configs {
49+
subnet = google_compute_subnetwork.default.id
50+
}
51+
}
52+
}
53+
}
54+
55+
resource "google_managed_kafka_connect_cluster" "default" {
56+
provider = google-beta
57+
project = data.google_project.default.project_id
58+
connect_cluster_id = "my-connect-cluster-id"
59+
location = "us-central1"
60+
# The Connect cluster is usually co-located with the target Kafka cluster.
61+
kafka_cluster = google_managed_kafka_cluster.target.id
62+
capacity_config {
63+
vcpu_count = 12
64+
memory_bytes = 12884901888 # 12 GiB
65+
}
66+
gcp_config {
67+
access_config {
68+
network_configs {
69+
primary_subnet = google_compute_subnetwork.default.id
70+
# As part of the configuration, ensure the Connect cluster has necessary access
71+
# to all MM2 source/target Kafka clusters. For more information on configuring access,
72+
# see the documentation:
73+
# (https://cloud.google.com/managed-service-for-apache-kafka/docs/connect-cluster/create-mirrormaker-connector)
74+
# As an example, we enable network access to Kafka Clusters below using DNS domain names.
75+
# The format for DNS name should be:
76+
# "GMK_CLUSTER_ID.REGION.managedkafka.PROJECT_ID.cloud.goog.*"
77+
# Please note that we do NOT need to add the DNS name of the primary Kafka cluster to the
78+
# `dns_domain_names` list, as our Connect cluster configures that automatically.
79+
dns_domain_names = ["DNS_DOMAIN_NAME"]
80+
}
81+
}
82+
}
83+
}
84+
85+
# Note: Due to a known issue, network attachment resources may not be
86+
# properly deleted, which can cause 'terraform destroy' to hang. It is
87+
# recommended to destroy network resources separately from the Kafka
88+
# Connect resources.
89+
# The documentation elaborates further on the recommended approach.
90+
# [START managedkafka_subnetwork]
91+
resource "google_compute_subnetwork" "default" {
92+
name = "test-subnetwork"
93+
ip_cidr_range = "10.2.0.0/16"
94+
region = "us-central1"
95+
network = google_compute_network.default.id
96+
}
97+
98+
resource "google_compute_network" "default" {
99+
name = "test-network"
100+
auto_create_subnetworks = false
101+
}
102+
# [END managedkafka_subnetwork]
103+
104+
# [START managedkafka_create_connector_mirrormaker2_source]
105+
# A single MirrorMaker 2 Source Connector to replicate from one source to one target.
106+
resource "google_managed_kafka_connector" "default" {
107+
project = data.google_project.default.project_id
108+
connector_id = "mm2-source-to-target-connector-id"
109+
connect_cluster = google_managed_kafka_connect_cluster.default.connect_cluster_id
110+
location = "us-central1"
111+
112+
configs = {
113+
"connector.class" = "org.apache.kafka.connect.mirror.MirrorSourceConnector"
114+
"name" = "mm2-source-to-target-connector-id"
115+
"tasks.max" = "3"
116+
"source.cluster.alias" = "source"
117+
"target.cluster.alias" = "target"
118+
"topics" = ".*" # Replicate all topics from the source
119+
# The value for bootstrap.servers is a comma-separated list of hostname:port pairs
120+
# for one or more Kafka brokers in the source/target cluster.
121+
"source.cluster.bootstrap.servers" = "source_cluster_dns"
122+
"target.cluster.bootstrap.servers" = "target_cluster_dns"
123+
# You can define an exclusion policy for topics as follows:
124+
# To exclude internal MirrorMaker 2 topics, internal topics and replicated topics,.
125+
"topics.exclude" = "mm2.*\\.internal,.*\\.replica,__.*"
126+
}
127+
128+
provider = google-beta
129+
}
130+
# [END managedkafka_create_connector_mirrormaker2_source]
131+
132+
data "google_project" "default" {
133+
}
134+
135+
# [END managedkafka_create_connector_mirrormaker2_source_parent]

0 commit comments

Comments
 (0)