Skip to content

ci: Update Kafka container. #787

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 3, 2025
Merged

ci: Update Kafka container. #787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e2a39a
ci: Update Kafka container.
pvital Aug 20, 2025
a02ff58
ci: Fix Kafka connection issues to ubuntu/kafka
CagriYonca Oct 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 31 additions & 12 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,18 +244,37 @@ jobs:
- store-pytest-results
- store-coverage-report

py312kafka:
py313kafka:
docker:
- image: public.ecr.aws/docker/library/python:3.12
- image: public.ecr.aws/bitnami/kafka:3.9.0
- image: public.ecr.aws/docker/library/python:3.13
- image: public.ecr.aws/ubuntu/zookeeper:3.1-22.04_edge
environment:
TZ: UTC
- image: public.ecr.aws/ubuntu/kafka:3.1-22.04_edge
environment:
KAFKA_CFG_NODE_ID: 0
KAFKA_CFG_PROCESS_ROLES: controller,broker
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
TZ: UTC
ZOOKEEPER_HOST: localhost
ZOOKEEPER_PORT: 2181
command:
- /opt/kafka/config/server.properties
- --override
- listeners=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9094
- --override
- advertised.listeners=INTERNAL://localhost:9093,EXTERNAL://localhost:9094
- --override
- listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- --override
- inter.broker.listener.name=INTERNAL
- --override
- broker.id=1
- --override
- offsets.topic.replication.factor=1
- --override
- transaction.state.log.replication.factor=1
- --override
- transaction.state.log.min.isr=1
- --override
- auto.create.topics.enable=true
working_directory: ~/repo
steps:
- checkout
Expand Down Expand Up @@ -309,7 +328,7 @@ workflows:
- py39cassandra
- py39gevent
- py312aws
- py312kafka
- py313kafka
- autowrapt:
matrix:
parameters:
Expand All @@ -322,5 +341,5 @@ workflows:
- py39cassandra
- py39gevent
- py312aws
- py312kafka
- py313kafka
- autowrapt
53 changes: 38 additions & 15 deletions .tekton/task.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,23 +166,46 @@ metadata:
name: python-tracer-unittest-kafka-task
spec:
sidecars:
- name: zookeeper
image: public.ecr.aws/ubuntu/zookeeper:3.1-22.04_edge
ports:
- containerPort: 9093
env:
- name: TZ
value: "UTC"
- name: kafka
image: public.ecr.aws/bitnami/kafka:3.9.0
image: public.ecr.aws/ubuntu/kafka:3.1-22.04_edge
env:
- name: KAFKA_CFG_NODE_ID
value: "0"
- name: KAFKA_CFG_PROCESS_ROLES
value: "controller,broker"
- name: KAFKA_CFG_LISTENERS
value: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094"
- name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
value: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS
value: "0@kafka:9093"
- name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES
value: "CONTROLLER"
- name: KAFKA_CFG_ADVERTISED_LISTENERS
value: "PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094"
- name: TZ
value: "UTC"
- name: ZOOKEEPER_HOST
value: localhost
- name: ZOOKEEPER_PORT
value: "2181"
ports:
- containerPort: 9093
- containerPort: 9094
command:
- /opt/kafka/bin/kafka-server-start.sh
- /opt/kafka/config/server.properties
- --override
- listeners=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9094
- --override
- advertised.listeners=INTERNAL://localhost:9093,EXTERNAL://localhost:9094
- --override
- listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- --override
- inter.broker.listener.name=INTERNAL
- --override
- broker.id=1
- --override
- offsets.topic.replication.factor=1
- --override
- transaction.state.log.replication.factor=1
- --override
- transaction.state.log.min.isr=1
- --override
- auto.create.topics.enable=true
params:
- name: imageDigest
type: string
Expand Down
45 changes: 34 additions & 11 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,16 +61,39 @@ services:
- "8681:8681"
- "8682:8682"

# Sidecar container for Kafka
zookeeper:
image: public.ecr.aws/ubuntu/zookeeper:3.1-22.04_edge
ports: ["2181:2181"]
environment: [ "TZ=UTC" ]

kafka:
image: public.ecr.aws/bitnami/kafka:latest
ports:
- '9092:9092'
- '9094:9094'
image: public.ecr.aws/ubuntu/kafka:3.1-22.04_edge
depends_on: [zookeeper]
ports:
- "9094:9094"
- "9093:9093"
environment:
- KAFKA_CFG_NODE_ID=0
- KAFKA_CFG_PROCESS_ROLES=controller,broker
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094
- TZ=UTC
- ZOOKEEPER_HOST=zookeeper
- ZOOKEEPER_PORT=2181
command:
- /opt/kafka/config/server.properties
- --override
- listeners=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9094
- --override
- advertised.listeners=INTERNAL://kafka:9093,EXTERNAL://127.0.0.1:9094
- --override
- listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- --override
- inter.broker.listener.name=INTERNAL
- --override
- broker.id=1
- --override
- offsets.topic.replication.factor=1
- --override
- transaction.state.log.replication.factor=1
- --override
- transaction.state.log.min.isr=1
- --override
- auto.create.topics.enable=true
Loading