kafka-python Instrument temporary fork, kafka-python-ng inside kafka-python's instrumentation (open-telemetry#2537)

* Instrument temporary fork, kafka-python-ng inside kafka-python's instrumentation

* Make kafkapython-ng run tests

Author: rjduffner

.github/workflows/instrumentations_1.yml

@@ -40,6 +40,7 @@ jobs:
           - "util-http"
           - "fastapislim"
           - "processor-baggage"
+          - "kafka-pythonng"
         os: [ubuntu-20.04]
         exclude:
           - python-version: pypy3

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Added
 
+- `opentelemetry-instrumentation-kafka-python` Instrument temporary fork, kafka-python-ng
+  inside kafka-python's instrumentation
+  ([#2537](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2537)))
+
 ## Breaking changes
 
 ## Fixed

instrumentation/README.md

@@ -24,7 +24,7 @@
 | [opentelemetry-instrumentation-grpc](./opentelemetry-instrumentation-grpc) | grpcio ~= 1.27 | No | experimental
 | [opentelemetry-instrumentation-httpx](./opentelemetry-instrumentation-httpx) | httpx >= 0.18.0 | No | migration
 | [opentelemetry-instrumentation-jinja2](./opentelemetry-instrumentation-jinja2) | jinja2 >= 2.7, < 4.0 | No | experimental
-| [opentelemetry-instrumentation-kafka-python](./opentelemetry-instrumentation-kafka-python) | kafka-python >= 2.0 | No | experimental
+| [opentelemetry-instrumentation-kafka-python](./opentelemetry-instrumentation-kafka-python) | kafka-python >= 2.0, < 3.0,kafka-python-ng >= 2.0, < 3.0 | No | experimental
 | [opentelemetry-instrumentation-logging](./opentelemetry-instrumentation-logging) | logging | No | experimental
 | [opentelemetry-instrumentation-mysql](./opentelemetry-instrumentation-mysql) | mysql-connector-python >= 8.0, < 10.0 | No | experimental
 | [opentelemetry-instrumentation-mysqlclient](./opentelemetry-instrumentation-mysqlclient) | mysqlclient < 3 | No | experimental

instrumentation/opentelemetry-instrumentation-kafka-python/pyproject.toml

@@ -22,6 +22,7 @@ classifiers = [
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
   "opentelemetry-api ~= 1.5",
@@ -31,7 +32,8 @@ dependencies = [
 
 [project.optional-dependencies]
 instruments = [
-  "kafka-python >= 2.0",
+  "kafka-python >= 2.0, < 3.0",
+  "kafka-python-ng >= 2.0, < 3.0"
 ]
 
 [project.entry-points.opentelemetry_instrumentor]

instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py

@@ -67,14 +67,19 @@ def consume_hook(span, record, args, kwargs):
 API
 ___
 """
+from importlib.metadata import PackageNotFoundError, distribution
 from typing import Collection
 
 import kafka
 from wrapt import wrap_function_wrapper
 
 from opentelemetry import trace
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
-from opentelemetry.instrumentation.kafka.package import _instruments
+from opentelemetry.instrumentation.kafka.package import (
+    _instruments,
+    _instruments_kafka_python,
+    _instruments_kafka_python_ng,
+)
 from opentelemetry.instrumentation.kafka.utils import _wrap_next, _wrap_send
 from opentelemetry.instrumentation.kafka.version import __version__
 from opentelemetry.instrumentation.utils import unwrap
@@ -86,6 +91,23 @@ class KafkaInstrumentor(BaseInstrumentor):
     """
 
     def instrumentation_dependencies(self) -> Collection[str]:
+        # Determine which package of kafka-python is installed
+        # Right now there are two packages, kafka-python and kafka-python-ng
+        # The latter is a fork of the former because the former is connected
+        # to a pypi namespace that the current maintainers cannot access
+        # https://github.com/dpkp/kafka-python/issues/2431
+        try:
+            distribution("kafka-python-ng")
+            return (_instruments_kafka_python_ng,)
+        except PackageNotFoundError:
+            pass
+
+        try:
+            distribution("kafka-python")
+            return (_instruments_kafka_python,)
+        except PackageNotFoundError:
+            pass
+
         return _instruments
 
     def _instrument(self, **kwargs):

instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py

@@ -13,4 +13,7 @@
 # limitations under the License.
 
 
-_instruments = ("kafka-python >= 2.0",)
+_instruments_kafka_python = "kafka-python >= 2.0, < 3.0"
+_instruments_kafka_python_ng = "kafka-python-ng >= 2.0, < 3.0"
+
+_instruments = (_instruments_kafka_python, _instruments_kafka_python_ng)

instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements-ng.txt

@@ -0,0 +1,15 @@
+asgiref==3.7.2
+Deprecated==1.2.14
+importlib-metadata==6.11.0
+iniconfig==2.0.0
+kafka-python-ng==2.2.2
+packaging==24.0
+pluggy==1.5.0
+py-cpuinfo==9.0.0
+pytest==7.4.4
+tomli==2.0.1
+typing_extensions==4.9.0
+wrapt==1.16.0
+zipp==3.19.2
+-e opentelemetry-instrumentation
+-e instrumentation/opentelemetry-instrumentation-kafka-python

opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py

@@ -105,7 +105,11 @@
         "instrumentation": "opentelemetry-instrumentation-jinja2==0.48b0.dev",
     },
     {
-        "library": "kafka-python >= 2.0",
+        "library": "kafka-python >= 2.0, < 3.0",
+        "instrumentation": "opentelemetry-instrumentation-kafka-python==0.48b0.dev",
+    },
+    {
+        "library": "kafka-python-ng >= 2.0, < 3.0",
         "instrumentation": "opentelemetry-instrumentation-kafka-python==0.48b0.dev",
     },
     {

tox.ini

@@ -350,7 +350,9 @@ envlist =
 
     ; opentelemetry-instrumentation-kafka-python
     py3{8,9,10,11}-test-instrumentation-kafka-python
+    py3{8,9,10,11,12}-test-instrumentation-kafka-pythonng
     pypy3-test-instrumentation-kafka-python
+    pypy3-test-instrumentation-kafka-pythonng
     lint-instrumentation-kafka-python
 
     ; opentelemetry-instrumentation-confluent-kafka
@@ -443,6 +445,10 @@ commands_pre =
   kafka-python: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions
   kafka-python: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk
   kafka-python: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements.txt
+  kafka-pythonng: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api
+  kafka-pythonng: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions
+  kafka-pythonng: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk
+  kafka-pythonng: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements-ng.txt
 
   confluent-kafka: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api
   confluent-kafka: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions
@@ -940,6 +946,9 @@ commands =
   lint-instrumentation-kafka-python: flake8 --config {toxinidir}/.flake8 {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python
   lint-instrumentation-kafka-python: sh -c "cd instrumentation && pylint --rcfile ../.pylintrc opentelemetry-instrumentation-kafka-python"
 
+  ; Test only for kafka-pythonng instrumentation as the only difference between kafka-python and kafka-pythonng is the version of kafka-python
+  test-instrumentation-kafka-pythonng: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python/tests {posargs}
+
   test-instrumentation-confluent-kafka: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-confluent-kafka/tests {posargs}
   lint-instrumentation-confluent-kafka: black --diff --check --config {toxinidir}/pyproject.toml {toxinidir}/instrumentation/opentelemetry-instrumentation-confluent-kafka
   lint-instrumentation-confluent-kafka: isort --diff --check-only --settings-path {toxinidir}/.isort.cfg {toxinidir}/instrumentation/opentelemetry-instrumentation-confluent-kafka