connect kafka in both external network and internal network

[damuji8]

i deployed kafka in aws internal vpc. i need connect kafka in both external network and internal network. i connect kafka from nlb in extralnel environment like this https://strimzi.io/blog/2020/01/02/using-strimzi-with-amazon-nlb-loadbalancers/ . but i also need connect kafka in internal vpc and not in k8s node. how can i config this. i use strimzi 0.19.0 . this is my deploying yaml.
apiVersion: kafka.strimzi.io/v1beta1
kind: Kafka
metadata:
name: kafka-dc
namespace: kafka-release
spec:
kafka:
template:
pod:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: env
operator: In
values:
- release
version: 2.4.1
replicas: 3
logging:
type: inline
loggers:
kafka.root.logger.level: "INFO"
resources:
requests:
memory: 2Gi
cpu: 100m
limits:
memory: 8Gi
cpu: "1"
readinessProbe:
initialDelaySeconds: 15
timeoutSeconds: 5
livenessProbe:
initialDelaySeconds: 15
timeoutSeconds: 5
jvmOptions:
-Xms: 2048m
-Xmx: 2048m
listeners:
plain: { }
tls: { }
external:
type: nodeport
tls: false

authentication:

type: tls

    configuration:
      preferredAddressType: InternalIP
    overrides:
      bootstrap:
        nodePort: 32006
      brokers:
      - broker: 0
        nodePort: 32003
        advertisedHost: k8s-kafka-dc-staging-.elb.us-east-2.amazonaws.com
        advertisedPort: 19092
      - broker: 1
        nodePort: 32004
        advertisedHost: k8s-kafka-dc-staging-.elb.us-east-2.amazonaws.com
        advertisedPort: 29092
      - broker: 2
        nodePort: 32005
        advertisedHost: k8s-kafka-dc-staging-.elb.us-east-2.amazonaws.com
        advertisedPort: 39092
config:
  offsets.topic.replication.factor: 3
  transaction.state.log.replication.factor: 3
  transaction.state.log.min.isr: 2
  default.replication.factor: 3
  min.insync.replicas: 2
  inter.broker.protocol.version: "2.4"
storage:
  type: jbod
  volumes:
  - id: 0
    type: persistent-claim
    size: 50Gi
    deleteClaim: false
    class: gp2-wfc
metrics:
  # Inspired by config from Kafka 2.0.0 example rules:
  # https://github.com/prometheus/jmx_exporter/blob/master/example_configs/kafka-2_0_0.yml
  lowercaseOutputName: true
  rules:
  # Special cases and very specific rules
  - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
    name: kafka_server_$1_$2
    type: GAUGE
    labels:
      clientId: "$3"
      topic: "$4"
      partition: "$5"
  - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
    name: kafka_server_$1_$2
    type: GAUGE
    labels:
      clientId: "$3"
      broker: "$4:$5"
  # Some percent metrics use MeanRate attribute
  # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate
  - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate
    name: kafka_$1_$2_$3_percent
    type: GAUGE
  # Generic gauges for percents
  - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value
    name: kafka_$1_$2_$3_percent
    type: GAUGE
  - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value
    name: kafka_$1_$2_$3_percent
    type: GAUGE
    labels:
      "$4": "$5"
  # Generic per-second counters with 0-2 key/value pairs
  - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
    name: kafka_$1_$2_$3_total
    type: COUNTER
    labels:
      "$4": "$5"
      "$6": "$7"
  - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
    name: kafka_$1_$2_$3_total
    type: COUNTER
    labels:
      "$4": "$5"
  - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
    name: kafka_$1_$2_$3_total
    type: COUNTER
  # Generic gauges with 0-2 key/value pairs
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      "$4": "$5"
      "$6": "$7"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      "$4": "$5"
  - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
    name: kafka_$1_$2_$3
    type: GAUGE
  # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
  # Note that these are missing the '_sum' metric!
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
    name: kafka_$1_$2_$3_count
    type: COUNTER
    labels:
      "$4": "$5"
      "$6": "$7"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      "$4": "$5"
      "$6": "$7"
      quantile: "0.$8"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
    name: kafka_$1_$2_$3_count
    type: COUNTER
    labels:
      "$4": "$5"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      "$4": "$5"
      quantile: "0.$6"
  - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
    name: kafka_$1_$2_$3_count
    type: COUNTER
  - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      quantile: "0.$4"

zookeeper:
template:
pod:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: env
operator: In
values:
- release
replicas: 3
logging:
type: inline
loggers:
zookeeper.root.logger: "INFO"
resources:
requests:
memory: 1Gi
cpu: 100m
limits:
memory: 2Gi
cpu: "1"
jvmOptions:
-Xms: 1024m
-Xmx: 1024m
storage:
type: persistent-claim
size: 5Gi
deleteClaim: false
class: gp2-wfc
metrics:
# Inspired by Zookeeper rules
# https://github.com/prometheus/jmx_exporter/blob/master/example_configs/zookeeper.yaml
lowercaseOutputName: true
rules:
# replicated Zookeeper
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\d+)><>(\w+)"
name: "zookeeper_$2"
type: GAUGE
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\d+), name1=replica.(\d+)><>(\w+)"
name: "zookeeper_$3"
type: GAUGE
labels:
replicaId: "$2"
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\d+), name1=replica.(\d+), name2=(\w+)><>(Packets\w+)"
name: "zookeeper_$4"
type: COUNTER
labels:
replicaId: "$2"
memberType: "$3"
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\d+), name1=replica.(\d+), name2=(\w+)><>(\w+)"
name: "zookeeper_$4"
type: GAUGE
labels:
replicaId: "$2"
memberType: "$3"
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\d+), name1=replica.(\d+), name2=(\w+), name3=(\w+)><>(\w+)"
name: "zookeeper_$4_$5"
type: GAUGE
labels:
replicaId: "$2"
memberType: "$3"
# standalone Zookeeper
- pattern: "org.apache.ZooKeeperService<name0=StandaloneServer_port(\d+)><>(\w+)"
type: GAUGE
name: "zookeeper_$2"
- pattern: "org.apache.ZooKeeperService<name0=StandaloneServer_port(\d+), name1=InMemoryDataTree><>(\w+)"
type: GAUGE
name: "zookeeper_$2"

entityOperator:

topicOperator: {}

userOperator: {}

kafkaExporter:
template:
pod:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: env
operator: In
values:
- release
topicRegex: "."
groupRegex: "."

[damuji8]

when i use nodeip:nodeport in internal ec2 like 10.201.54.78:32006. it seems connect still from public nlb. and this will cost Public network traffic.

[scholzj]

You can create different external listeners. For example nodeports and loabalancers, two different nodeport listeners etc. That way you can use each for a different network. But 0.19 is super old. I'm not sure we support it in that version.

[damuji8]

unluckly,0.20.0 support this...
thanks a lot

[scholzj]

Ok, they you would need to upgrade I guess. The problem is that the bootstrap address you use is only used for the initial connection used to get the metadata. But the subsequent connections use the advertised address which is only once per-listener. So at the end it will always lead your clients to the same advertised address.

[damuji8]

yes, you are right. i try to upgrade strimzi.