doc: simplify running the example with docker-compose

tchrono · edsiper · commit a6b610409289 · 2022-02-16T14:27:27.000-06:00
Signed-off-by: Thiago Padilha &lt;thiago@calyptia.com&gt;
diff --git a/examples/kafka_filter/.env b/examples/kafka_filter/.env
@@ -0,0 +1,4 @@
+KAFKA_HOST=kafka-broker
+KAFKA_PORT=9092
+ZOOKEEPER_HOST=zookeeper
+ZOOKEEPER_PORT=2181
diff --git a/examples/kafka_filter/Dockerfile b/examples/kafka_filter/Dockerfile
@@ -0,0 +1,42 @@
+FROM debian:bullseye-slim as builder
+ENV DEBIAN_FRONTEND noninteractive
+ENV KAFKA_URL https://dlcdn.apache.org/kafka/3.0.0/kafka_2.13-3.0.0.tgz
+ENV KAFKA_SHA256 a82728166bbccf406009747a25e1fe52dbcb4d575e4a7a8616429b5818cd02d1
+
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    ca-certificates \
+    cmake \
+    pkg-config \
+    libsasl2-dev \
+    flex \
+    openjdk-11-jre-headless \
+    bison \
+    netcat-openbsd \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Download kafka to access "kafka-topics" script
+WORKDIR /kafka
+RUN bash -c 'curl -L $KAFKA_URL | tee kafka.tgz | sha256sum -c <(echo "$KAFKA_SHA256 -")' \
+    && tar --strip-components=1 -xf kafka.tgz \
+    && mv /kafka/bin/kafka-topics.sh /kafka/bin/kafka-topics
+ENV PATH="${PATH}:/kafka/bin"
+
+WORKDIR /build/
+COPY . /source
+RUN cmake -DFLB_DEV=On \
+          -DFLB_IN_KAFKA=On \
+          -DFLB_OUT_KAFKA=On \
+          /source && \
+    make -j "$(getconf _NPROCESSORS_ONLN)"
+
+FROM builder as runner
+COPY --from=builder /build/bin/fluent-bit /usr/local/bin/fluent-bit
+COPY examples/kafka_filter/kafka.conf /etc/kafka.conf
+COPY examples/kafka_filter/kafka.lua /etc/kafka.lua
+CMD ["/usr/local/bin/fluent-bit", "-c", "/etc/kafka.conf"]
diff --git a/examples/kafka_filter/Makefile b/examples/kafka_filter/Makefile
@@ -1,25 +1,11 @@
-CPU_COUNT := $(shell grep -c ^processor /proc/cpuinfo)
+build:
+	docker-compose build
 
-KAFKA_TGT = kafka/done.stamp
-KAFKA_DIR = $(shell dirname $(KAFKA_TGT))
-CMAKE_TGT = build/done.stamp
-FLB_TGT = build/bin/fluent-bit
+start:
+	docker-compose up -d
+	docker-compose logs -f fluent-bit kafka-consumer
 
-run: $(KAFKA_TGT) $(FLB_TGT)
-	./kafka-start.sh $(KAFKA_DIR) && tmux a
+stop:
+	docker-compose down
 
-$(FLB_TGT): $(CMAKE_TGT)
-	make -C build -j$(CPU_COUNT)
-
-$(CMAKE_TGT):
-	rm -rf build
-	mkdir -pv build
-	cd build && cmake -DFLB_DEV=On -DFLB_IN_KAFKA=On -DFLB_OUT_KAFKA=On ../../..
-	touch $@
-
-$(KAFKA_TGT):
-	rm -rf $(KAFKA_DIR)
-	./kafka-setup.sh $(KAFKA_DIR)
-	touch $@
-
-.PHONY: run
+.PHONY: build start stop
diff --git a/examples/kafka_filter/README.md b/examples/kafka_filter/README.md
diff --git a/examples/kafka_filter/common.sh b/examples/kafka_filter/common.sh
diff --git a/examples/kafka_filter/docker-compose.yml b/examples/kafka_filter/docker-compose.yml
@@ -0,0 +1,68 @@
+version: '3.4'
+
+x-service-common-fields: &service-common-fields
+    volumes:
+      - ./scripts:/scripts
+    env_file:
+        - ./.env
+
+
+services:
+  fluent-bit:
+      <<: *service-common-fields
+      build:
+          context: ../..
+          target: runner
+          dockerfile: examples/kafka_filter/Dockerfile
+      command: /scripts/flb-start.sh
+      depends_on:
+          - kafka-consumer
+
+
+  kafka-create-topics:
+      <<: *service-common-fields
+      image: confluentinc/cp-server:7.0.1
+      command: /scripts/create-topics.sh
+      depends_on:
+          - kafka-broker
+
+
+  kafka-producer:
+      <<: *service-common-fields
+      image: confluentinc/cp-server:7.0.1
+      command: /scripts/kafka-produce.sh
+      depends_on:
+          - fluent-bit
+
+
+  kafka-consumer:
+      <<: *service-common-fields
+      image: confluentinc/cp-server:7.0.1
+      command: /scripts/kafka-consume.sh
+      depends_on:
+          - kafka-create-topics
+
+
+  kafka-broker:
+      image: confluentinc/cp-server:7.0.1
+      hostname: broker
+      container_name: kafka-broker
+      depends_on:
+          - zookeeper
+      environment:
+          KAFKA_BROKER_ID: 1
+          KAFKA_ZOOKEEPER_CONNECT: "${ZOOKEEPER_HOST}:${ZOOKEEPER_PORT}"
+          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT"
+          KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://${KAFKA_HOST}:${KAFKA_PORT}"
+          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+          KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+          KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
+
+
+  zookeeper:
+      image: confluentinc/cp-zookeeper:7.0.1
+      hostname: zookeeper
+      container_name: zookeeper
+      environment:
+          ZOOKEEPER_CLIENT_PORT: "${ZOOKEEPER_PORT}"
+          ZOOKEEPER_TICK_TIME: 2000
diff --git a/examples/kafka_filter/example-data.json b/examples/kafka_filter/example-data.json
diff --git a/examples/kafka_filter/kafka-setup.sh b/examples/kafka_filter/kafka-setup.sh
diff --git a/examples/kafka_filter/kafka-start.sh b/examples/kafka_filter/kafka-start.sh
diff --git a/examples/kafka_filter/kafka.conf b/examples/kafka_filter/kafka.conf
@@ -5,7 +5,7 @@
 
 [INPUT]
     Name kafka
-    brokers localhost:9092
+    brokers kafka-broker:9092
     topics fb-source
 
 [FILTER]
@@ -16,5 +16,5 @@
 
 [OUTPUT]
     Name kafka
-    brokers localhost:9092
+    brokers kafka-broker:9092
     topics fb-sink
diff --git a/examples/kafka_filter/kafka.lua b/examples/kafka_filter/kafka.lua
@@ -2,6 +2,6 @@ local count = 0
 function modify_kafka_message(tag, timestamp, record)
     count = count + 1
     local payload = record.payload
-    payload.status = 'processed '..payload.name..', total records: '..tostring(count)
+    payload.status = 'processed by fluent-bit, total records: '..tostring(count)
     return 1, timestamp, payload
 end
diff --git a/examples/kafka_filter/scripts/common.sh b/examples/kafka_filter/scripts/common.sh
@@ -0,0 +1,17 @@
+#!/bin/bash -ue
+
+wait_kafka() {
+	while ! nc -z "$KAFKA_HOST" "$KAFKA_PORT"; do
+		sleep 0.1
+	done
+}
+
+wait_topic() {
+	wait_kafka
+	local topic=$1
+	[ -z "$topic" ] && return 1
+	while true; do
+		kafka-topics --list --bootstrap-server "$KAFKA_HOST:$KAFKA_PORT" | grep -q "^$topic$" && break
+		sleep 0.1
+	done
+}
diff --git a/examples/kafka_filter/scripts/create-topics.sh b/examples/kafka_filter/scripts/create-topics.sh
@@ -0,0 +1,9 @@
+#!/bin/bash -ue
+
+# shellcheck disable=SC1091
+. /scripts/common.sh
+
+wait_kafka
+
+kafka-topics --create --partitions 1 --replication-factor 1 --topic fb-source --bootstrap-server "$KAFKA_HOST:$KAFKA_PORT"
+kafka-topics --create --partitions 1 --replication-factor 1 --topic fb-sink --bootstrap-server "$KAFKA_HOST:$KAFKA_PORT"
diff --git a/examples/kafka_filter/scripts/flb-start.sh b/examples/kafka_filter/scripts/flb-start.sh
@@ -0,0 +1,8 @@
+#!/bin/bash -ue
+
+# shellcheck disable=SC1091
+. /scripts/common.sh
+
+wait_topic fb-sink
+
+exec /usr/local/bin/fluent-bit -c /etc/kafka.conf
diff --git a/examples/kafka_filter/scripts/kafka-consume.sh b/examples/kafka_filter/scripts/kafka-consume.sh
@@ -0,0 +1,9 @@
+#!/bin/bash -ue
+
+# shellcheck disable=SC1091
+. /scripts/common.sh
+
+wait_topic fb-sink
+
+kafka-console-consumer --topic fb-sink --bootstrap-server \
+	"$KAFKA_HOST:$KAFKA_PORT"
diff --git a/examples/kafka_filter/scripts/kafka-produce.sh b/examples/kafka_filter/scripts/kafka-produce.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -ue
+
+# shellcheck disable=SC1091
+. /scripts/common.sh
+
+wait_topic fb-sink
+
+for i in $(seq 1 100); do
+	sleep 1
+	echo "{ \"name\": \"object-$i\" }" | \
+		kafka-console-producer --topic fb-source \
+		--broker-list "$KAFKA_HOST:$KAFKA_PORT"
+done
