diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fef25d9..1bba6674 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -244,18 +244,37 @@ jobs: - store-pytest-results - store-coverage-report - py312kafka: + py313kafka: docker: - - image: public.ecr.aws/docker/library/python:3.12 - - image: public.ecr.aws/bitnami/kafka:3.9.0 + - image: public.ecr.aws/docker/library/python:3.13 + - image: public.ecr.aws/ubuntu/zookeeper:3.1-22.04_edge + environment: + TZ: UTC + - image: public.ecr.aws/ubuntu/kafka:3.1-22.04_edge environment: - KAFKA_CFG_NODE_ID: 0 - KAFKA_CFG_PROCESS_ROLES: controller,broker - KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093 - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094 + TZ: UTC + ZOOKEEPER_HOST: localhost + ZOOKEEPER_PORT: 2181 + command: + - /opt/kafka/config/server.properties + - --override + - listeners=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9094 + - --override + - advertised.listeners=INTERNAL://localhost:9093,EXTERNAL://localhost:9094 + - --override + - listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + - --override + - inter.broker.listener.name=INTERNAL + - --override + - broker.id=1 + - --override + - offsets.topic.replication.factor=1 + - --override + - transaction.state.log.replication.factor=1 + - --override + - transaction.state.log.min.isr=1 + - --override + - auto.create.topics.enable=true working_directory: ~/repo steps: - checkout @@ -309,7 +328,7 @@ workflows: - py39cassandra - py39gevent - py312aws - - py312kafka + - py313kafka - autowrapt: matrix: parameters: @@ -322,5 +341,5 @@ workflows: - py39cassandra - py39gevent - py312aws - - py312kafka + - py313kafka - autowrapt diff --git a/.tekton/task.yaml b/.tekton/task.yaml index e5d79c92..0a9a6d05 100644 --- a/.tekton/task.yaml +++ b/.tekton/task.yaml @@ -166,23 +166,46 @@ metadata: name: python-tracer-unittest-kafka-task spec: sidecars: + - name: zookeeper + image: public.ecr.aws/ubuntu/zookeeper:3.1-22.04_edge + ports: + - containerPort: 9093 + env: + - name: TZ + value: "UTC" - name: kafka - image: public.ecr.aws/bitnami/kafka:3.9.0 + image: public.ecr.aws/ubuntu/kafka:3.1-22.04_edge env: - - name: KAFKA_CFG_NODE_ID - value: "0" - - name: KAFKA_CFG_PROCESS_ROLES - value: "controller,broker" - - name: KAFKA_CFG_LISTENERS - value: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094" - - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP - value: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT" - - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS - value: "0@kafka:9093" - - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES - value: "CONTROLLER" - - name: KAFKA_CFG_ADVERTISED_LISTENERS - value: "PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094" + - name: TZ + value: "UTC" + - name: ZOOKEEPER_HOST + value: localhost + - name: ZOOKEEPER_PORT + value: "2181" + ports: + - containerPort: 9093 + - containerPort: 9094 + command: + - /opt/kafka/bin/kafka-server-start.sh + - /opt/kafka/config/server.properties + - --override + - listeners=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9094 + - --override + - advertised.listeners=INTERNAL://localhost:9093,EXTERNAL://localhost:9094 + - --override + - listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + - --override + - inter.broker.listener.name=INTERNAL + - --override + - broker.id=1 + - --override + - offsets.topic.replication.factor=1 + - --override + - transaction.state.log.replication.factor=1 + - --override + - transaction.state.log.min.isr=1 + - --override + - auto.create.topics.enable=true params: - name: imageDigest type: string diff --git a/docker-compose.yml b/docker-compose.yml index 45393b76..299806a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,16 +61,39 @@ services: - "8681:8681" - "8682:8682" + # Sidecar container for Kafka + zookeeper: + image: public.ecr.aws/ubuntu/zookeeper:3.1-22.04_edge + ports: ["2181:2181"] + environment: [ "TZ=UTC" ] + kafka: - image: public.ecr.aws/bitnami/kafka:latest - ports: - - '9092:9092' - - '9094:9094' + image: public.ecr.aws/ubuntu/kafka:3.1-22.04_edge + depends_on: [zookeeper] + ports: + - "9094:9094" + - "9093:9093" environment: - - KAFKA_CFG_NODE_ID=0 - - KAFKA_CFG_PROCESS_ROLES=controller,broker - - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 - - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT - - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093 - - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094 + - TZ=UTC + - ZOOKEEPER_HOST=zookeeper + - ZOOKEEPER_PORT=2181 + command: + - /opt/kafka/config/server.properties + - --override + - listeners=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9094 + - --override + - advertised.listeners=INTERNAL://kafka:9093,EXTERNAL://127.0.0.1:9094 + - --override + - listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + - --override + - inter.broker.listener.name=INTERNAL + - --override + - broker.id=1 + - --override + - offsets.topic.replication.factor=1 + - --override + - transaction.state.log.replication.factor=1 + - --override + - transaction.state.log.min.isr=1 + - --override + - auto.create.topics.enable=true