|
| 1 | +# Kafka Receiver Connector |
| 2 | + |
| 3 | +This directory contains the Axoflow Kafka receiver connector which helps collecting logs from Kafka topics. |
| 4 | + |
| 5 | +## Quickstart |
| 6 | + |
| 7 | +Create a topic for your logs, e.g: |
| 8 | + |
| 9 | +```sh |
| 10 | +kubectl apply -f - <<EOF |
| 11 | +apiVersion: v1 |
| 12 | +kind: Pod |
| 13 | +metadata: |
| 14 | + name: kafka-client |
| 15 | + namespace: kafka |
| 16 | +spec: |
| 17 | + containers: |
| 18 | + - name: kafka-client |
| 19 | + image: docker.io/apache/kafka:4.1.1 |
| 20 | + command: ["sleep", "infinity"] |
| 21 | +EOF |
| 22 | +kubectl wait --for=condition=ready pod/kafka-client -n kafka --timeout=60s |
| 23 | + |
| 24 | +kubectl exec -n kafka kafka-client -- /opt/kafka/bin/kafka-topics.sh \ |
| 25 | + --create \ |
| 26 | + --topic otlp_logs \ |
| 27 | + --partitions 3 \ |
| 28 | + --replication-factor 1 \ |
| 29 | + --bootstrap-server kafka.kafka.svc.cluster.local:9092 \ |
| 30 | + --if-not-exists |
| 31 | +``` |
| 32 | + |
| 33 | +Make sure the required environment variables are set before running the connector. |
| 34 | + |
| 35 | +### Using plaintext connection |
| 36 | + |
| 37 | +```bash |
| 38 | +UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())") |
| 39 | +AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1) |
| 40 | + |
| 41 | +docker run \ |
| 42 | + --rm \ |
| 43 | + -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \ |
| 44 | + -e KAFKA_BROKERS="${KAFKA_BROKERS}" \ |
| 45 | + -e KAFKA_LOGS_TOPIC="${KAFKA_LOGS_TOPIC}" \ |
| 46 | + -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \ |
| 47 | + -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \ |
| 48 | + -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \ |
| 49 | + ghcr.io/axoflow/axocloudconnectors:latest |
| 50 | +``` |
| 51 | + |
| 52 | +### Using TLS |
| 53 | + |
| 54 | +```bash |
| 55 | +UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())") |
| 56 | +AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1) |
| 57 | + |
| 58 | +docker run \ |
| 59 | + --rm \ |
| 60 | + -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \ |
| 61 | + -v "${KAFKA_CERTS_PATH}:/certs:ro" \ |
| 62 | + -e KAFKA_BROKERS="${KAFKA_BROKERS}" \ |
| 63 | + -e KAFKA_LOGS_TOPIC="${KAFKA_LOGS_TOPIC}" \ |
| 64 | + -e KAFKA_TLS_CA_FILE="/certs/ca.pem" \ |
| 65 | + -e KAFKA_TLS_CERT_FILE="/certs/cert.pem" \ |
| 66 | + -e KAFKA_TLS_KEY_FILE="/certs/key.pem" \ |
| 67 | + -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \ |
| 68 | + -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \ |
| 69 | + -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \ |
| 70 | + ghcr.io/axoflow/axocloudconnectors:latest |
| 71 | +``` |
| 72 | + |
| 73 | +## Deploy with Helm-chart |
| 74 | + |
| 75 | +1. Set the required environment-variables. |
| 76 | + |
| 77 | +### Example deploy with Axorouter in cluster |
| 78 | + |
| 79 | +```bash |
| 80 | +make minikube-cluster |
| 81 | +make docker-build |
| 82 | +make minikube-load-image |
| 83 | + |
| 84 | +kubectl create namespace cloudconnectors |
| 85 | +kubectl create secret generic kafka-credentials \ |
| 86 | + --from-literal=brokers="<YOUR-KAFKA-BROKERS>" \ |
| 87 | + --from-literal=logs-topic="<YOUR-KAFKA-LOGS-TOPIC>" \ |
| 88 | + --namespace cloudconnectors \ |
| 89 | + --dry-run=client -o yaml | kubectl apply -f - |
| 90 | + |
| 91 | +UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())") |
| 92 | +AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1) |
| 93 | + |
| 94 | +helm upgrade --install --wait --namespace cloudconnectors cloudconnectors ./charts/cloudconnectors \ |
| 95 | + --set image.repository="axocloudconnectors" \ |
| 96 | + --set image.tag="dev" \ |
| 97 | + --set 'env[0].name=AXOROUTER_ENDPOINT' \ |
| 98 | + --set 'env[0].value=axorouter.axoflow-local.svc.cluster.local:4317' \ |
| 99 | + --set 'env[1].name=AXOCLOUDCONNECTOR_DEVICE_ID' \ |
| 100 | + --set "env[1].value=${AXOCLOUDCONNECTOR_DEVICE_ID}" \ |
| 101 | + --set 'env[2].name=KAFKA_BROKERS' \ |
| 102 | + --set 'env[2].valueFrom.secretKeyRef.name=kafka-credentials' \ |
| 103 | + --set 'env[2].valueFrom.secretKeyRef.key=brokers' \ |
| 104 | + --set 'env[3].name=KAFKA_LOGS_TOPIC' \ |
| 105 | + --set 'env[3].valueFrom.secretKeyRef.name=kafka-credentials' \ |
| 106 | + --set 'env[3].valueFrom.secretKeyRef.key=logs-topic' |
| 107 | +``` |
0 commit comments