fix(internal): improve performance of package resolution [backport 2.21] (#12375)

Co-authored-by: Brett Langdon <[email protected]>
Co-authored-by: Federico Mon <[email protected]>

Author: 3 people

.gitlab/benchmarks.yml

@@ -56,6 +56,7 @@ microbenchmarks:
         - "http_propagation_extract"
         - "http_propagation_inject"
         - "rate_limiter"
+        - "packages_update_imported_dependencies"
 
 benchmarks-pr-comment:
   image: $MICROBENCHMARKS_CI_IMAGE

ddtrace/internal/packages.py

@@ -20,6 +20,8 @@
 
 Distribution = t.NamedTuple("Distribution", [("name", str), ("version", str), ("path", t.Optional[str])])
 
+_PACKAGE_DISTRIBUTIONS: t.Optional[t.Mapping[str, t.List[str]]] = None
+
 
 @callonce
 def get_distributions():
@@ -45,18 +47,21 @@ def get_distributions():
     return pkgs
 
 
-@cached(maxsize=1)
 def get_package_distributions() -> t.Mapping[str, t.List[str]]:
     """a mapping of importable package names to their distribution name(s)"""
-    try:
-        import importlib.metadata as importlib_metadata
-    except ImportError:
-        import importlib_metadata  # type: ignore[no-redef]
-
-    # Prefer the official API if available, otherwise fallback to the vendored version
-    if hasattr(importlib_metadata, "packages_distributions"):
-        return importlib_metadata.packages_distributions()
-    return _packages_distributions()
+    global _PACKAGE_DISTRIBUTIONS
+    if _PACKAGE_DISTRIBUTIONS is None:
+        try:
+            import importlib.metadata as importlib_metadata
+        except ImportError:
+            import importlib_metadata  # type: ignore[no-redef]
+
+        # Prefer the official API if available, otherwise fallback to the vendored version
+        if hasattr(importlib_metadata, "packages_distributions"):
+            _PACKAGE_DISTRIBUTIONS = importlib_metadata.packages_distributions()
+        else:
+            _PACKAGE_DISTRIBUTIONS = _packages_distributions()
+    return _PACKAGE_DISTRIBUTIONS
 
 
 @cached(maxsize=1024)

releasenotes/notes/fix-internal-packages-perf-c270b1ff7e68b46a.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    internal: Fix performance overhead of Python distribution parsing for internal telemetry.