Skip to content

Commit 145270c

Browse files
mccullyamccullya
authored
Merge pull request #20 from osodevops/cluster-linking
added basic cluster liknking example
2 parents 2a1b162 + 7fc0a1b commit 145270c

24 files changed

+570
-0
lines changed

‎examples/cluster-linking/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Replicator
2+
3+
In this scenario example, you'll deploy two Confluent clusters. One is the source cluster, and one is the destination cluster. You'll deploy Confluent Replicator on the destination cluster, where it will copy topic messages from the source cluster and write to the destination cluster.
4+
5+
1.

‎examples/cluster-linking/destination/control-centre.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
apiVersion: platform.confluent.io/v1beta1
2+
kind: ControlCenter
3+
metadata:
4+
name: controlcenter
5+
spec:
6+
authorization:
7+
type: rbac
8+
tls:
9+
secretRef: tls-group1
10+
dependencies:
11+
kafka:
12+
bootstrapEndpoint: kafka.destination.svc.cluster.local:9071
13+
authentication:
14+
type: plain
15+
jaasConfig:
16+
secretRef: mds-client-connect
17+
tls:
18+
enabled: true
19+
mds:
20+
endpoint: https://kafka.destination.svc.cluster.local:8090
21+
tokenKeyPair:
22+
secretRef: mds-public
23+
authentication:
24+
type: bearer
25+
bearer:
26+
secretRef: mds-client-c3
27+
tls:
28+
enabled: true
29+
connect:
30+
- name: replicator
31+
url: https://replicator.destination.svc.cluster.local:8083
32+
tls:
33+
enabled: true
34+
schemaRegistry:
35+
url: https://schemaregistry.destination.svc.cluster.local:8081
36+
tls:
37+
enabled: true

‎examples/cluster-linking/destination/controlcenter-testadmin-rolebindings.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
---
2+
apiVersion: platform.confluent.io/v1beta1
3+
kind: ConfluentRolebinding
4+
metadata:
5+
name: testadmin-rb
6+
spec:
7+
principal:
8+
type: user
9+
name: testadmin
10+
role: ClusterAdmin
11+
---
12+
apiVersion: platform.confluent.io/v1beta1
13+
kind: ConfluentRolebinding
14+
metadata:
15+
name: testadmin-rb-sr
16+
spec:
17+
principal:
18+
type: user
19+
name: testadmin
20+
clustersScopeByIds:
21+
schemaRegistryClusterId: id_schemaregistry_destination
22+
role: SystemAdmin
23+
---
24+
apiVersion: platform.confluent.io/v1beta1
25+
kind: ConfluentRolebinding
26+
metadata:
27+
name: testadmin-rb-replicator
28+
spec:
29+
principal:
30+
type: user
31+
name: testadmin
32+
clustersScopeByIds:
33+
connectClusterId: destination.replicator
34+
role: SystemAdmin
35+
---

‎examples/cluster-linking/destination/controlcentre-c3-rb.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: platform.confluent.io/v1beta1
2+
kind: ConfluentRolebinding
3+
metadata:
4+
name: c3-rb-replicator
5+
spec:
6+
principal:
7+
type: user
8+
name: c3
9+
clustersScopeByIds:
10+
connectClusterId: destination.replicator
11+
role: SystemAdmin
12+
---

‎examples/cluster-linking/destination/kafka.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
---
2+
apiVersion: platform.confluent.io/v1beta1
3+
kind: Kafka
4+
metadata:
5+
name: kafka
6+
spec:
7+
configOverrides:
8+
server:
9+
- confluent.schema.registry.url=https://schemaregistry.destination.svc.cluster.local:8081
10+
- listener.name.internal.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
11+
- listener.name.external.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
12+
- listener.name.replication.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
13+
tls:
14+
secretRef: tls-group1
15+
metricReporter:
16+
enabled: true
17+
authentication:
18+
type: plain
19+
jaasConfigPassThrough:
20+
secretRef: broker-credential
21+
tls:
22+
enabled: true
23+
listeners:
24+
internal:
25+
authentication:
26+
type: plain
27+
jaasConfigPassThrough:
28+
secretRef: broker-credential
29+
tls:
30+
enabled: true
31+
external:
32+
authentication:
33+
type: plain
34+
jaasConfigPassThrough:
35+
secretRef: broker-credential
36+
tls:
37+
enabled: true
38+
authorization:
39+
type: rbac
40+
superUsers:
41+
- User:kafka
42+
services:
43+
mds:
44+
tls:
45+
enabled: true
46+
tokenKeyPair:
47+
secretRef: broker-credential
48+
provider:
49+
type: ldap
50+
ldap:
51+
address: ldap://ldap.sandbox.svc.cluster.local:389
52+
authentication:
53+
type: simple
54+
simple:
55+
secretRef: broker-credential
56+
tls:
57+
enabled: true
58+
configurations:
59+
groupNameAttribute: cn
60+
groupObjectClass: groupOfNames
61+
groupMemberAttribute: member
62+
groupMemberAttributePattern: cn=(.*),ou=users,dc=test,dc=com
63+
groupSearchBase: ou=groups,dc=test,dc=com
64+
userNameAttribute: cn
65+
userMemberOfAttributePattern: cn=(.*),ou=users,dc=test,dc=com
66+
userObjectClass: organizationalRole
67+
userSearchBase: ou=users,dc=test,dc=com
68+
dependencies:
69+
kafkaRest:
70+
authentication:
71+
type: bearer
72+
bearer:
73+
secretRef: broker-credential
74+
zookeeper:
75+
endpoint: zookeeper.destination.svc.cluster.local:2182
76+
authentication:
77+
type: digest
78+
jaasConfig:
79+
secretRef: broker-credential
80+
tls:
81+
enabled: true

‎examples/cluster-linking/destination/kustomization.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
namespace: destination
4+
resources:
5+
- ./namespace.yaml
6+
- ../../../kustomize/base/confluent
7+
- ../../../kustomize/base/secrets-tls
8+
- ../../../kustomize/base/secrets-user
9+
- ./replicator.yaml
10+
- ./controlcentre-c3-rb.yaml
11+
- ./replicator-rb.yaml
12+
- ./schemaregistry-base.yaml
13+
- ./controlcenter-testadmin-rolebindings.yaml
14+
patchesStrategicMerge:
15+
- ./rest-class.yaml
16+
- zookeeper.yaml
17+
- kafka.yaml
18+
- schema-registry.yaml
19+
- control-centre.yaml

‎examples/cluster-linking/destination/namespace.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: destination

‎examples/cluster-linking/destination/replicator-rb.yaml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
2+
apiVersion: platform.confluent.io/v1beta1
3+
kind: ConfluentRolebinding
4+
metadata:
5+
name: connect-rb-cluster-admin
6+
spec:
7+
principal:
8+
type: user
9+
name: connect
10+
role: ClusterAdmin
11+
---
12+
apiVersion: platform.confluent.io/v1beta1
13+
kind: ConfluentRolebinding
14+
metadata:
15+
name: replicator-topics
16+
namespace: confluent
17+
spec:
18+
principal:
19+
name: connect
20+
type: user
21+
resourcePatterns:
22+
- name: topic-in-source_replica
23+
patternType: LITERAL
24+
resourceType: Topic
25+
- name: _confluent-license
26+
patternType: LITERAL
27+
resourceType: Topic
28+
- name: _confluent-gcsconnect-secrets
29+
patternType: LITERAL
30+
resourceType: Topic
31+
- name: _confluent-connect-offsets
32+
patternType: LITERAL
33+
resourceType: Topic
34+
- name: _confluent-connect-configs
35+
patternType: LITERAL
36+
resourceType: Topic
37+
- name: _confluent-connect-status
38+
patternType: LITERAL
39+
resourceType: Topic
40+
- name: _confluent-gcsconnect-status
41+
patternType: LITERAL
42+
resourceType: Topic
43+
- name: _confluent-gcsconnect-offsets
44+
patternType: LITERAL
45+
resourceType: Topic
46+
- name: _confluent-gcsconnect-configs
47+
patternType: LITERAL
48+
resourceType: Topic
49+
- name: confluent.gcsconnect-
50+
patternType: PREFIXED
51+
resourceType: Topic
52+
role: ResourceOwner
53+
---
54+
apiVersion: platform.confluent.io/v1beta1
55+
kind: ConfluentRolebinding
56+
metadata:
57+
name: gcsconnect-idempotent-write
58+
namespace: confluent
59+
spec:
60+
principal:
61+
name: connect
62+
type: user
63+
resourcePatterns:
64+
- name: kafka-cluster
65+
resourceType: Cluster
66+
role: DeveloperWrite

‎examples/cluster-linking/destination/replicator.yaml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
apiVersion: platform.confluent.io/v1beta1
2+
# Confluent Replicator is built as a connector, and so will use the `Connect` CRD.
3+
kind: Connect
4+
metadata:
5+
name: replicator
6+
spec:
7+
replicas: 1
8+
# Configure to have TLS encryption, and use auto-generated server certs
9+
tls:
10+
secretRef: tls-group1
11+
authorization:
12+
type: rbac
13+
image:
14+
# Use the `cp-enterprise-replicator-operator` Docker image, that contains the Replicator jars
15+
application: confluentinc/cp-enterprise-replicator-operator:6.1.1.0
16+
init: confluentinc/cp-init-container-operator:6.1.1.0
17+
podTemplate:
18+
resources:
19+
requests:
20+
cpu: 1
21+
memory: 2Gi
22+
envVars:
23+
# The Confluent Replicator Monitoring Extension allows for detailed metrics from Replicator tasks to be
24+
# collected using an exposed REST API.
25+
# You'll need to update the version string in replicate-test-extension-<version>.jar based on what
26+
# CP version you are using
27+
- name: CLASSPATH
28+
value: /usr/share/java/kafka-connect-replicator/replicator-rest-extension-6.1.1.jar
29+
configOverrides:
30+
# When the Connect distributed cluster hosting Replicator has a REST endpoint with SSL encryption
31+
# enabled, you must configure security properties for the SSL keystore and truststore used by the
32+
# Replicator monitoring extension to communicate with other Connect nodes in the cluster.
33+
# `/mnt/sslcerts/truststore.jks` is the truststore location when auto-genarated certs are used.
34+
jvm:
35+
# - -Djavax.net.debug=all
36+
- -Djavax.net.ssl.trustStore=/mnt/sslcerts/truststore.p12
37+
- -Djavax.net.ssl.trustStorePassword=mystorepassword
38+
server:
39+
- connector.class=io.confluent.connect.replicator.ReplicatorSourceConnector
40+
# To activate the monitoring extension, configure this property
41+
- rest.extension.classes=io.confluent.connect.replicator.monitoring.ReplicatorMonitoringExtension,io.confluent.connect.security.ConnectSecurityExtension
42+
- admin.sasl.jaas.config=org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required metadataServerUrls="https://kafka.destination.svc.cluster.local:8090" username="connect" password="connect-secret";
43+
- consumer.sasl.jaas.config=org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required metadataServerUrls="https://kafka.destination.svc.cluster.local:8090" username="connect" password="connect-secret";
44+
- producer.sasl.jaas.config=org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required metadataServerUrls="https://kafka.destination.svc.cluster.local:8090" username="connect" password="connect-secret";
45+
- sasl.jaas.config=org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required metadataServerUrls="https://kafka.destination.svc.cluster.local:8090" username="connect" password="connect-secret";
46+
- confluent.metadata.basic.auth.user.info=connect:connect-secret
47+
dependencies:
48+
kafka:
49+
bootstrapEndpoint: kafka.destination.svc.cluster.local:9071
50+
authentication:
51+
type: plain
52+
jaasConfig:
53+
secretRef: mds-client-connect
54+
tls:
55+
enabled: true
56+
mds:
57+
endpoint: https://kafka.destination.svc.cluster.local:8090
58+
tokenKeyPair:
59+
secretRef: mds-public
60+
authentication:
61+
type: bearer
62+
bearer:
63+
secretRef: mds-client-connect
64+
tls:
65+
enabled: true
66+
interceptor:
67+
enabled: true

‎examples/cluster-linking/destination/rest-class.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: platform.confluent.io/v1beta1
2+
kind: KafkaRestClass
3+
metadata:
4+
name: default
5+
spec:
6+
kafkaRest:
7+
authentication:
8+
type: bearer
9+
bearer:
10+
secretRef: rest-credential

0 commit comments

Comments
 (0)