Add "instruments-any" feature: unblock multi-target instrumentations while fixing dependency conflict breakage.

CHANGELOG.md

@@ -11,6 +11,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Fixed
+
+- `opentelemetry-instrumentation`: Fix dependency conflict detection when instrumented packages are not installed by moving check back to before instrumentors are loaded. Add "instruments-any" feature for instrumentations that target multiple packages.
+  ([#3610](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3610))
+
+### Added
+
+- `opentelemetry-instrumentation-psycopg2` Utilize instruments-any functionality.
+  ([#3610](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3610))
+- `opentelemetry-instrumentation-kafka-python` Utilize instruments-any functionality.
+  ([#3610](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3610))
+
 ## Version 1.35.0/0.56b0 (2025-07-11)
 
 ### Added

CONTRIBUTING.md

@@ -325,10 +325,25 @@ Below is a checklist of things to be mindful of when implementing a new instrume
 ### Update supported instrumentation package versions
 
 - Navigate to the **instrumentation package directory:**
-  - Update **`pyproject.toml`** file by modifying _instruments_ entry in the `[project.optional-dependencies]` section with the new version constraint
-  - Update `_instruments` variable in instrumentation **`package.py`** file with the new version constraint
+  - Update **`pyproject.toml`** file by modifying `instruments` or `instruments-any` entry in the `[project.optional-dependencies]` section with the new version constraint
+  - Update `_instruments` or `_instruments_any` variable in instrumentation **`package.py`** file with the new version constraint
 - At the **root of the project directory**, run `tox -e generate` to regenerate necessary files
 
+Please note that `instruments-any` is an optional field that can be used instead of or in addition to `instruments`. While `instruments` is a list of dependencies, _all_ of which are expected by the instrumentation, `instruments-any` is a list _any_ of which but not all are expected. For example, the following entry requires both `util` and `common` plus either `foo` or `bar` to be present for the instrumentation to occur:
+```
+[project.optional-dependencies]
+instruments = [
+  "util ~= 1.0"
+  "common ~= 2.0"
+]
+instruments-any = [
+  "foo ~= 3.0"
+  "bar ~= 4.0"
+]
+```
+
+<!-- See https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3610 for details on instruments-any -->
+
 If you're adding support for a new version of the instrumentation package, follow these additional steps:
 
 - At the **instrumentation package directory:** Add new test-requirements.txt file with the respective package version required for testing

instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py

@@ -40,7 +40,6 @@
 )
 from opentelemetry.instrumentation.dependencies import (
     DependencyConflict,
-    DependencyConflictError,
 )
 from opentelemetry.sdk.metrics.export import (
     HistogramDataPoint,
@@ -1102,40 +1101,34 @@ def test_instruments_with_fastapi_installed(self, mock_logger):
             [self._instrumentation_loaded_successfully_call()]
         )
 
+    @patch(
+        "opentelemetry.instrumentation.auto_instrumentation._load.get_dist_dependency_conflicts"
+    )
     @patch("opentelemetry.instrumentation.auto_instrumentation._load._logger")
-    def test_instruments_with_old_fastapi_installed(self, mock_logger):  # pylint: disable=no-self-use
+    def test_instruments_with_old_fastapi_installed(
+        self, mock_logger, mock_dep
+    ):  # pylint: disable=no-self-use
         dependency_conflict = DependencyConflict("0.58", "0.57")
         mock_distro = Mock()
-        mock_distro.load_instrumentor.side_effect = DependencyConflictError(
-            dependency_conflict
-        )
+        mock_dep.return_value = dependency_conflict
         _load_instrumentors(mock_distro)
-        self.assertEqual(len(mock_distro.load_instrumentor.call_args_list), 1)
-        (ep,) = mock_distro.load_instrumentor.call_args.args
-        self.assertEqual(ep.name, "fastapi")
-        assert (
-            self._instrumentation_loaded_successfully_call()
-            not in mock_logger.debug.call_args_list
-        )
+        mock_distro.load_instrumentor.assert_not_called()
         mock_logger.debug.assert_has_calls(
             [self._instrumentation_failed_to_load_call(dependency_conflict)]
         )
 
+    @patch(
+        "opentelemetry.instrumentation.auto_instrumentation._load.get_dist_dependency_conflicts"
+    )
     @patch("opentelemetry.instrumentation.auto_instrumentation._load._logger")
-    def test_instruments_without_fastapi_installed(self, mock_logger):  # pylint: disable=no-self-use
+    def test_instruments_without_fastapi_installed(
+        self, mock_logger, mock_dep
+    ):  # pylint: disable=no-self-use
         dependency_conflict = DependencyConflict("0.58", None)
         mock_distro = Mock()
-        mock_distro.load_instrumentor.side_effect = DependencyConflictError(
-            dependency_conflict
-        )
+        mock_dep.return_value = dependency_conflict
         _load_instrumentors(mock_distro)
-        self.assertEqual(len(mock_distro.load_instrumentor.call_args_list), 1)
-        (ep,) = mock_distro.load_instrumentor.call_args.args
-        self.assertEqual(ep.name, "fastapi")
-        assert (
-            self._instrumentation_loaded_successfully_call()
-            not in mock_logger.debug.call_args_list
-        )
+        mock_distro.load_instrumentor.assert_not_called()
         mock_logger.debug.assert_has_calls(
             [self._instrumentation_failed_to_load_call(dependency_conflict)]
         )

instrumentation/opentelemetry-instrumentation-kafka-python/pyproject.toml

@@ -31,7 +31,8 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-instruments = [
+instruments = []
+instruments-any = [
   "kafka-python >= 2.0, < 3.0",
   "kafka-python-ng >= 2.0, < 3.0"
 ]

instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py

@@ -91,7 +91,7 @@ def process_msg(message):
 from opentelemetry import trace
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
 from opentelemetry.instrumentation.kafka.package import (
-    _instruments,
+    _instruments_any,
     _instruments_kafka_python,
     _instruments_kafka_python_ng,
 )
@@ -123,7 +123,7 @@ def instrumentation_dependencies(self) -> Collection[str]:
         except PackageNotFoundError:
             pass
 
-        return _instruments
+        return _instruments_any
 
     def _instrument(self, **kwargs):
         """Instruments the kafka module

instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py

@@ -16,4 +16,5 @@
 _instruments_kafka_python = "kafka-python >= 2.0, < 3.0"
 _instruments_kafka_python_ng = "kafka-python-ng >= 2.0, < 3.0"
 
-_instruments = (_instruments_kafka_python, _instruments_kafka_python_ng)
+_instruments = ()
+_instruments_any = (_instruments_kafka_python, _instruments_kafka_python_ng)

instrumentation/opentelemetry-instrumentation-kafka-python/tests/test_instrumentation.py

@@ -20,7 +20,6 @@
 
 from opentelemetry.instrumentation.kafka import KafkaInstrumentor
 from opentelemetry.instrumentation.kafka.package import (
-    _instruments,
     _instruments_kafka_python,
     _instruments_kafka_python_ng,
 )
@@ -134,4 +133,7 @@ def _distribution(name):
                 call("kafka-python"),
             ],
         )
-        self.assertEqual(package_to_instrument, _instruments)
+        self.assertEqual(
+            package_to_instrument,
+            (_instruments_kafka_python, _instruments_kafka_python_ng),
+        )

instrumentation/opentelemetry-instrumentation-psycopg2/pyproject.toml

@@ -31,7 +31,8 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-instruments = [
+instruments = []
+instruments-any = [
   "psycopg2 >= 2.7.3.1",
   "psycopg2-binary >= 2.7.3.1",
 ]

instrumentation/opentelemetry-instrumentation-psycopg2/src/opentelemetry/instrumentation/psycopg2/__init__.py

@@ -151,7 +151,7 @@
 from opentelemetry.instrumentation import dbapi
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
 from opentelemetry.instrumentation.psycopg2.package import (
-    _instruments,
+    _instruments_any,
     _instruments_psycopg2,
     _instruments_psycopg2_binary,
 )
@@ -187,7 +187,7 @@ def instrumentation_dependencies(self) -> Collection[str]:
         except PackageNotFoundError:
             pass
 
-        return _instruments
+        return _instruments_any
 
     def _instrument(self, **kwargs):
         """Integrate with PostgreSQL Psycopg library.

instrumentation/opentelemetry-instrumentation-psycopg2/src/opentelemetry/instrumentation/psycopg2/package.py

@@ -16,7 +16,8 @@
 _instruments_psycopg2 = "psycopg2 >= 2.7.3.1"
 _instruments_psycopg2_binary = "psycopg2-binary >= 2.7.3.1"
 
-_instruments = (
+_instruments = ()
+_instruments_any = (
     _instruments_psycopg2,
     _instruments_psycopg2_binary,
 )

instrumentation/opentelemetry-instrumentation-psycopg2/tests/test_psycopg2_instrumentation.py

@@ -20,7 +20,6 @@
 )
 from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
 from opentelemetry.instrumentation.psycopg2.package import (
-    _instruments,
     _instruments_psycopg2,
     _instruments_psycopg2_binary,
 )
@@ -130,19 +129,29 @@ def _distribution(name):
                 call("psycopg2-binary"),
             ],
         )
-        self.assertEqual(package_to_instrument, _instruments)
+        self.assertEqual(
+            package_to_instrument,
+            (
+                _instruments_psycopg2,
+                _instruments_psycopg2_binary,
+            ),
+        )
 
     # This test is to verify that the auto instrumentation path
     # will auto instrument psycopg2 or psycopg2-binary is installed.
     # Note there is only one test here but it is run twice in tox
     # once with the psycopg2 package installed and once with
     # psycopg2-binary installed.
+    @patch(
+        "opentelemetry.instrumentation.auto_instrumentation._load.get_dist_dependency_conflicts"
+    )
     @patch("opentelemetry.instrumentation.auto_instrumentation._load._logger")
-    def test_instruments_with_psycopg2_installed(self, mock_logger):
+    def test_instruments_with_psycopg2_installed(self, mock_logger, mock_dep):
         def _instrumentation_loaded_successfully_call():
             return call("Instrumented %s", "psycopg2")
 
         mock_distro = Mock()
+        mock_dep.return_value = None
         mock_distro.load_instrumentor.return_value = None
         _load_instrumentors(mock_distro)
         self.assertEqual(len(mock_distro.load_instrumentor.call_args_list), 1)

opentelemetry-instrumentation/src/opentelemetry/instrumentation/auto_instrumentation/_load.py

@@ -12,22 +12,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from functools import cached_property
 from logging import getLogger
 from os import environ
 
-from opentelemetry.instrumentation.dependencies import DependencyConflictError
+from opentelemetry.instrumentation.dependencies import (
+    DependencyConflictError,
+    get_dist_dependency_conflicts,
+)
 from opentelemetry.instrumentation.distro import BaseDistro, DefaultDistro
 from opentelemetry.instrumentation.environment_variables import (
     OTEL_PYTHON_CONFIGURATOR,
     OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
     OTEL_PYTHON_DISTRO,
 )
 from opentelemetry.instrumentation.version import __version__
-from opentelemetry.util._importlib_metadata import entry_points
+from opentelemetry.util._importlib_metadata import (
+    EntryPoint,
+    distributions,
+    entry_points,
+)
 
 _logger = getLogger(__name__)
 
 
+class _EntryPointDistFinder:
+    @cached_property
+    def _mapping(self):
+        return {
+            self._key_for(ep): dist
+            for dist in distributions()
+            for ep in dist.entry_points
+        }
+
+    def dist_for(self, entry_point: EntryPoint):
+        dist = getattr(entry_point, "dist", None)
+        if dist:
+            return dist
+
+        return self._mapping.get(self._key_for(entry_point))
+
+    @staticmethod
+    def _key_for(entry_point: EntryPoint):
+        return f"{entry_point.group}:{entry_point.name}:{entry_point.value}"
+
+
 def _load_distro() -> BaseDistro:
     distro_name = environ.get(OTEL_PYTHON_DISTRO, None)
     for entry_point in entry_points(group="opentelemetry_distro"):
@@ -55,6 +84,7 @@ def _load_distro() -> BaseDistro:
 
 def _load_instrumentors(distro):
     package_to_exclude = environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, [])
+    entry_point_finder = _EntryPointDistFinder()
     if isinstance(package_to_exclude, str):
         package_to_exclude = package_to_exclude.split(",")
         # to handle users entering "requests , flask" or "requests, flask" with spaces
@@ -71,11 +101,24 @@ def _load_instrumentors(distro):
             continue
 
         try:
-            distro.load_instrumentor(
-                entry_point, raise_exception_on_conflict=True
-            )
+            entry_point_dist = entry_point_finder.dist_for(entry_point)
+            conflict = get_dist_dependency_conflicts(entry_point_dist)
+            if conflict:
+                _logger.debug(
+                    "Skipping instrumentation %s: %s",
+                    entry_point.name,
+                    conflict,
+                )
+                continue
+
+            # tell instrumentation to not run dep checks again as we already did it above
+            distro.load_instrumentor(entry_point, skip_dep_check=True)
             _logger.debug("Instrumented %s", entry_point.name)
         except DependencyConflictError as exc:
+            # Dependency conflicts are generally caught from get_dist_dependency_conflicts
+            # returning a DependencyConflict. Keeping this error handling in case custom
+            # distro and instrumentor behavior raises a DependencyConflictError later.
+            # See https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3610
             _logger.debug(
                 "Skipping instrumentation %s: %s",
                 entry_point.name,