Skip to content

Commit be446e6

Browse files
salmanysalmany
committed
docs: Add Managed Kafka Connect terraform sample
for Connectors Adds Terraform samples for Managed Kafka Connect to create connectors for BigQuery, Cloud Storage, MirrorMaker, Pub/Sub Sink, and Pub/Sub Source.
1 parent 315b503 commit be446e6

File tree

5 files changed

+407
-0
lines changed
  • managedkafka
    • managedkafka_create_connector_bigquery_sink
    • managedkafka_create_connector_cloud_storage_sink
    • managedkafka_create_connector_mirrormaker
    • managedkafka_create_connector_pubsub_sink
    • managedkafka_create_connector_pubsub_source

5 files changed

+407
-0
lines changed

‎managedkafka/managedkafka_create_connector_bigquery_sink/main.tf‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
# [START managedkafkaconnect_create_connector_bigquery_sink_parent]
18+
data "google_project" "default" {
19+
provider = google-beta
20+
}
21+
22+
resource "google_managed_kafka_cluster" "default" {
23+
project = data.google_project.default.project_id
24+
cluster_id = "my-cluster-id"
25+
location = "us-central1"
26+
capacity_config {
27+
vcpu_count = 3
28+
memory_bytes = 3221225472 # 3 GiB
29+
}
30+
gcp_config {
31+
access_config {
32+
network_configs {
33+
subnet = "projects/${data.google_project.default.number}/regions/us-central1/subnetworks/default"
34+
}
35+
}
36+
}
37+
}
38+
39+
resource "google_managed_kafka_connect_cluster" "default" {
40+
provider = google-beta
41+
project = data.google_project.default.project_id
42+
connect_cluster_id = "my-connect-cluster-id"
43+
location = "us-central1"
44+
kafka_cluster = google_managed_kafka_cluster.default.id
45+
capacity_config {
46+
vcpu_count = 3
47+
memory_bytes = 2884901888 # 12 GiB
48+
}
49+
gcp_config {
50+
access_config {
51+
network_configs {
52+
primary_subnet = "projects/${data.google_project.default.number}/regions/us-central1/subnetworks/default"
53+
}
54+
}
55+
}
56+
}
57+
# [END managedkafkaconnect_create_connector_bigquery_sink_parent]
58+
59+
# [START managedkafkaconnect_create_connector_bigquery_sink]
60+
resource "google_managed_kafka_connector" "example-bigquery-sink-connector" {
61+
project = data.google_project.default.project_id
62+
connector_id = "my-bigquery-sink-connector"
63+
connect_cluster = google_managed_kafka_connect_cluster.default.connect_cluster_id
64+
location = "us-central1"
65+
66+
configs = {
67+
"name" = "my-bigquery-sink-connector"
68+
"project" = "GCP_PROJECT_ID"
69+
"topics" = "GMK_TOPIC_ID"
70+
"tasks.max" = "3"
71+
"connector.class" = "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector"
72+
"key.converter" = "org.apache.kafka.connect.storage.StringConverter"
73+
"value.converter" = "org.apache.kafka.connect.json.JsonConverter"
74+
"value.converter.schemas.enable"= "false"
75+
"defaultDataset" = "BQ_DATASET_ID"
76+
}
77+
78+
provider = google-beta
79+
}
80+
# [END managedkafkaconnect_create_connector_bigquery_sink]

‎managedkafka/managedkafka_create_connector_cloud_storage_sink/main.tf‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
# [START managedkafkaconnect_create_connector_cloud_storage_sink_parent]
18+
data "google_project" "default" {
19+
provider = google-beta
20+
}
21+
22+
resource "google_managed_kafka_cluster" "default" {
23+
project = data.google_project.default.project_id
24+
cluster_id = "my-cluster-id"
25+
location = "us-central1"
26+
capacity_config {
27+
vcpu_count = 3
28+
memory_bytes = 3221225472 # 3 GiB
29+
}
30+
gcp_config {
31+
access_config {
32+
network_configs {
33+
subnet = "projects/${data.google_project.default.number}/regions/us-central1/subnetworks/default"
34+
}
35+
}
36+
}
37+
}
38+
39+
resource "google_managed_kafka_connect_cluster" "default" {
40+
provider = google-beta
41+
project = data.google_project.default.project_id
42+
connect_cluster_id = "my-connect-cluster-id"
43+
location = "us-central1"
44+
kafka_cluster = google_managed_kafka_cluster.default.id
45+
capacity_config {
46+
vcpu_count = 12
47+
memory_bytes = 2884901888 # 12 GiB
48+
}
49+
gcp_config {
50+
access_config {
51+
network_configs {
52+
primary_subnet = "projects/${data.google_project.default.number}/regions/us-central1/subnetworks/default"
53+
}
54+
}
55+
}
56+
}
57+
# [END managedkafkaconnect_create_connector_cloud_storage_sink_parent]
58+
59+
# [START managedkafkaconnect_create_connector_cloud_storage_sink]
60+
resource "google_managed_kafka_connector" "example-cloud-storage-sink-connector" {
61+
project = data.google_project.default.project_id
62+
connector_id = "my-gcs-sink-connector"
63+
connect_cluster = google_managed_kafka_connect_cluster.default.connect_cluster_id
64+
location = "us-central1"
65+
66+
configs = {
67+
"connector.class" = "io.aiven.kafka.connect.gcs.GcsSinkConnector"
68+
"tasks.max" = "1"
69+
"topics" = "GMK_TOPIC_ID"
70+
"gcs.bucket.name" = "GCS_BUCKET_NAME"
71+
"gcs.credentials.default" = "true"
72+
"format.output.type" = "json"
73+
"name" = "my-gcs-sink-connector"
74+
"value.converter" = "org.apache.kafka.connect.json.JsonConverter"
75+
"value.converter.schemas.enable"= "false"
76+
"key.converter" = "org.apache.kafka.connect.storage.StringConverter"
77+
}
78+
provider = google-beta
79+
}
80+
# [END managedkafkaconnect_create_connector_cloud_storage_sink]

‎managedkafka/managedkafka_create_connector_mirrormaker/main.tf‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
# [START managedkafkaconnect_create_connector_mirrormaker_parent]
18+
data "google_project" "default" {
19+
provider = google-beta
20+
}
21+
22+
resource "google_managed_kafka_cluster" "default" {
23+
project = data.google_project.default.project_id
24+
cluster_id = "my-cluster-id"
25+
location = "us-central1"
26+
capacity_config {
27+
vcpu_count = 3
28+
memory_bytes = 3221225472 # 3 GiB
29+
}
30+
gcp_config {
31+
access_config {
32+
network_configs {
33+
subnet = "projects/${data.google_project.default.number}/regions/us-central1/subnetworks/default"
34+
}
35+
}
36+
}
37+
}
38+
39+
resource "google_managed_kafka_connect_cluster" "default" {
40+
provider = google-beta
41+
project = data.google_project.default.project_id
42+
connect_cluster_id = "my-connect-cluster-id"
43+
location = "us-central1"
44+
kafka_cluster = google_managed_kafka_cluster.default.id
45+
capacity_config {
46+
vcpu_count = 12
47+
memory_bytes = 2884901888 # 12 GiB
48+
}
49+
gcp_config {
50+
access_config {
51+
network_configs {
52+
primary_subnet = "projects/${data.google_project.default.number}/regions/us-central1/subnetworks/default"
53+
}
54+
}
55+
}
56+
}
57+
# [END managedkafkaconnect_create_connector_mirrormaker_parent]
58+
59+
# [START managedkafkaconnect_create_connector_mirrormaker]
60+
resource "google_managed_kafka_connector" "default" {
61+
project = data.google_project.default.project_id
62+
connector_id = "MM2_CONNECTOR_ID"
63+
connect_cluster = google_managed_kafka_connect_cluster.default.connect_cluster_id
64+
location = "us-central1"
65+
66+
configs = {
67+
"connector.class" = "org.apache.kafka.connect.mirror.MirrorSourceConnector"
68+
"name" = "MM2_CONNECTOR_ID"
69+
"source.cluster.alias" = "source"
70+
"target.cluster.alias" = "target"
71+
"topics" = "GMK_TOPIC_NAME"
72+
"source.cluster.bootstrap.servers" = "GMK_SOURCE_CLUSTER_DNS"
73+
"target.cluster.bootstrap.servers" = "GMK_TARGET_CLUSTER_DNS"
74+
"offset-syncs.topic.replication.factor" = "1"
75+
"source.cluster.security.protocol" = "SASL_SSL"
76+
"source.cluster.sasl.mechanism" = "OAUTHBEARER"
77+
"source.cluster.sasl.login.callback.handler.class" = "com.google.cloud.hosted.kafka.auth.GcpLoginCallbackHandler"
78+
"source.cluster.sasl.jaas.config" = "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required;"
79+
"target.cluster.security.protocol" = "SASL_SSL"
80+
"target.cluster.sasl.mechanism" = "OAUTHBEARER"
81+
"target.cluster.sasl.login.callback.handler.class" = "com.google.cloud.hosted.kafka.auth.GcpLoginCallbackHandler"
82+
"target.cluster.sasl.jaas.config" = "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required;"
83+
}
84+
85+
provider = google-beta
86+
}
87+
# [END managedkafkaconnect_create_connector_mirrormaker]

‎managedkafka/managedkafka_create_connector_pubsub_sink/main.tf‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
# [START managedkafkaconnect_create_connector_pubsub_sink_parent]
18+
data "google_project" "default" {
19+
provider = google-beta
20+
}
21+
22+
resource "google_managed_kafka_cluster" "default" {
23+
project = data.google_project.default.project_id
24+
cluster_id = "my-cluster-id"
25+
location = "us-central1"
26+
capacity_config {
27+
vcpu_count = 3
28+
memory_bytes = 3221225472 # 3 GiB
29+
}
30+
gcp_config {
31+
access_config {
32+
network_configs {
33+
subnet = "projects/${data.google_project.default.number}/regions/us-central1/subnetworks/default"
34+
}
35+
}
36+
}
37+
}
38+
39+
resource "google_managed_kafka_connect_cluster" "default" {
40+
provider = google-beta
41+
project = data.google_project.default.project_id
42+
connect_cluster_id = "my-connect-cluster-id"
43+
location = "us-central1"
44+
kafka_cluster = google_managed_kafka_cluster.default.id
45+
capacity_config {
46+
vcpu_count = 12
47+
memory_bytes = 2884901888 # 12 GiB
48+
}
49+
gcp_config {
50+
access_config {
51+
network_configs {
52+
primary_subnet = "projects/${data.google_project.default.number}/regions/us-central1/subnetworks/default"
53+
}
54+
}
55+
}
56+
}
57+
58+
# [END managedkafkaconnect_create_connector_pubsub_sink_parent]
59+
60+
# [START managedkafkaconnect_create_connector_pubsub_sink]
61+
resource "google_managed_kafka_connector" "example-pubsub-sink-connector" {
62+
project = data.google_project.default.project_id
63+
connector_id = "my-pubsub-sink-connector"
64+
connect_cluster = google_managed_kafka_connect_cluster.default.connect_cluster_id
65+
location = "us-central1"
66+
67+
configs = {
68+
"connector.class" = "com.google.pubsub.kafka.sink.CloudPubSubSinkConnector"
69+
"name" = "my-pubsub-sink-connector"
70+
"tasks.max" = "1"
71+
"topics" = "TOPIC_NAME"
72+
"cps.topic" = "CPS_TOPIC_NAME"
73+
"cps.project" = "CPS_PROJECT_NAME"
74+
"value.converter" = "org.apache.kafka.connect.storage.StringConverter"
75+
"key.converter" = "org.apache.kafka.connect.storage.StringConverter"
76+
}
77+
78+
provider = google-beta
79+
}
80+
# [END managedkafkaconnect_create_connector_pubsub_sink]

0 commit comments

Comments
 (0)