ext-rdkafka instrumentation (#284)

Instrumentation for RdKafka for context propagation

Author: tom-diacono

.github/workflows/php.yml

@@ -23,6 +23,7 @@ jobs:
           'Aws',
           'Context/Swoole',
           'Instrumentation/ExtAmqp',
+          'Instrumentation/ExtRdKafka',
           'Instrumentation/Guzzle',
           'Instrumentation/HttpAsyncClient',
           'Instrumentation/Slim',
@@ -94,6 +95,12 @@ jobs:
             php-version: 8.0
           - project: 'Instrumentation/ExtAmqp'
             php-version: 8.1
+          - project: 'Instrumentation/ExtRdKafka'
+            php-version: 7.4
+          - project: 'Instrumentation/ExtRdKafka'
+            php-version: 8.0
+          - project: 'Instrumentation/ExtRdKafka'
+            php-version: 8.1
           - project: 'Instrumentation/OpenAIPHP'
             php-version: 7.4
           - project: 'Instrumentation/OpenAIPHP'
@@ -112,7 +119,7 @@ jobs:
       with:
         php-version: ${{ matrix.php-version }}
         coverage: xdebug
-        extensions: ast, amqp, grpc, opentelemetry
+        extensions: ast, amqp, grpc, opentelemetry, rdkafka
 
     - name: Validate composer.json and composer.lock
       run: composer validate
@@ -167,6 +174,11 @@ jobs:
       run: |
         docker compose up rabbitmq -d --wait
 
+    - name: Start Kafka
+      if: ${{ matrix.project == 'Instrumentation/ExtRdKafka' }}
+      run: |
+        KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092 docker compose up  kafka -d --wait
+
     - name: Run PHPUnit
       working-directory: src/${{ matrix.project }}
       run: vendor/bin/phpunit

.gitsplit.yml

@@ -16,6 +16,8 @@ splits:
     target: "https://${GH_TOKEN}@github.com/opentelemetry-php/contrib-auto-codeigniter.git"
   - prefix: "src/Instrumentation/ExtAmqp"
     target: "https://${GH_TOKEN}@github.com/opentelemetry-php/contrib-auto-ext-amqp.git"
+  - prefix: "src/Instrumentation/ExtRdKafka"
+    target: "https://${GH_TOKEN}@github.com/opentelemetry-php/contrib-auto-ext-rdkafka.git"
   - prefix: "src/Instrumentation/Guzzle"
     target: "https://${GH_TOKEN}@github.com/opentelemetry-php/contrib-auto-guzzle.git"
   - prefix: "src/Instrumentation/HttpAsyncClient"

composer.json

@@ -18,6 +18,7 @@
       "OpenTelemetry\\Contrib\\Aws\\": "src/Aws/src",
       "OpenTelemetry\\Contrib\\Context\\Swoole\\": "src/Context/Swoole/src",
       "OpenTelemetry\\Contrib\\Instrumentation\\ExtAmqp\\": "src/Instrumentation/ExtAmqp/src",
+      "OpenTelemetry\\Contrib\\Instrumentation\\ExtRdKafka\\": "src/Instrumentation/ExtRdKafka/src",
       "OpenTelemetry\\Contrib\\Instrumentation\\Laravel\\": "src/Instrumentation/Laravel/src",
       "OpenTelemetry\\Contrib\\Instrumentation\\HttpAsyncClient\\": "src/Instrumentation/HttpAsyncClient/src",
       "OpenTelemetry\\Contrib\\Instrumentation\\IO\\": "src/Instrumentation/IO/src",
@@ -39,6 +40,7 @@
     },
     "files": [
       "src/Instrumentation/ExtAmqp/_register.php",
+      "src/Instrumentation/ExtRdKafka/_register.php",
       "src/Instrumentation/HttpAsyncClient/_register.php",
       "src/Instrumentation/IO/_register.php",
       "src/Instrumentation/Laravel/_register.php",

docker-compose.yaml

@@ -13,6 +13,7 @@ services:
       XDEBUG_CONFIG: ${XDEBUG_CONFIG:-''}
       PHP_IDE_CONFIG: ${PHP_IDE_CONFIG:-''}
       RABBIT_HOST: ${RABBIT_HOST:-rabbitmq}
+      KAFKA_HOST: ${KAFKA_HOST:-kafka}
 
   zipkin:
     image: openzipkin/zipkin-slim
@@ -42,3 +43,22 @@ services:
       retries: 3
     ports:
       - "5672:5672/tcp"
+  kafka:
+    image: confluentinc/cp-kafka:7.2.1
+    hostname: kafka
+    ports:
+      - "9092:9092/tcp"
+    environment:
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_NODE_ID: 1
+      KAFKA_ADVERTISED_LISTENERS: ${KAFKA_ADVERTISED_LISTENERS:-PLAINTEXT://kafka:29092,PLAINTEXT_HOST://kafka:9092}
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092'
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+    command: "bash -c '/tmp/update_run.sh && /etc/confluent/docker/run'"
+    volumes:
+      - ./docker/kafka/update_run.sh:/tmp/update_run.sh
+
+

docker/Dockerfile

@@ -6,6 +6,7 @@ USER root
 RUN install-php-extensions \
         opentelemetry \
         mongodb \
-        amqp
+        amqp \
+        rdkafka
 
 USER php

docker/kafka/update_run.sh

@@ -0,0 +1,11 @@
+# This script is required to run kafka cluster (without zookeeper)
+#!/bin/sh
+
+# Docker workaround: Remove check for KAFKA_ZOOKEEPER_CONNECT parameter
+sed -i '/KAFKA_ZOOKEEPER_CONNECT/d' /etc/confluent/docker/configure
+
+# Docker workaround: Ignore cub zk-ready
+sed -i 's/cub zk-ready/echo ignore zk-ready/' /etc/confluent/docker/ensure
+
+# KRaft required step: Format the storage directory with a new cluster ID
+echo "kafka-storage format --ignore-formatted -t $(kafka-storage random-uuid) -c /etc/kafka/kafka.properties" >> /etc/confluent/docker/ensure

src/Instrumentation/ExtRdKafka/.gitattributes

@@ -0,0 +1,13 @@
+* text=auto
+
+*.md diff=markdown
+*.php diff=php
+
+/.gitattributes export-ignore
+/.gitignore export-ignore
+/.phan export-ignore
+/.php-cs-fixer.php export-ignore
+/phpstan.neon.dist export-ignore
+/phpunit.xml.dist export-ignore
+/psalm.xml.dist export-ignore
+/tests export-ignore

src/Instrumentation/ExtRdKafka/.gitignore

@@ -0,0 +1 @@
+.idea/