Apply patched only when dependency detected. (#122)

*Issue #, if available:*
We face `botocore` version conflict and python module miss localization
when both sample app and aws distro install `botocore` pkg with their
own version.

*Description of changes:*
Move `botocore` to `[project.optional-dependencies]` as additional
packages. The user will need to manually install botocore themselves,
otherwise, it won’t be automatically installed when users install
`aws-opentelemetry-distro`. This has the same behaviour as upstream
[opentelemetry-instrumentation-botocore](https://github.com/open-telemetry/opentelemetry-python-contrib/tree/b84d779104b8c511ae7b876f5a9f0db3f21654e6/instrumentation/opentelemetry-instrumentation-botocore).

Note:
1. To check whether `botocore` is installed in the system, we don't read
`pyproject.toml` file directly since it is only used when building the
pkg, and won't be copied into the container. Instead, we define a
`patch_libraries` dict.
2. we move `import apply_instrumentation_patches ` into conditional
check since the `botocore` only available if condition pass. Otherwise,
if we put it on the top of the file, the instrumentation will throw
error when `botocore` not installed.

Test:
Tested on ECS and locally.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: zzhlogin

aws-opentelemetry-distro/pyproject.toml

@@ -68,12 +68,14 @@ dependencies = [
   "opentelemetry-instrumentation-urllib3 == 0.43b0",
   "opentelemetry-instrumentation-wsgi == 0.43b0",
   "opentelemetry-instrumentation-cassandra == 0.43b0",
-
-  # Patch dependencies
-  "botocore ~= 1.0",
 ]
 
 [project.optional-dependencies]
+# The 'patch' optional dependency is used for applying patches to specific libraries.
+# If a new patch is added into the list, it must also be added into tox.ini, dev-requirements.txt and _instrumentation_patch
+patch = [
+  "botocore ~= 1.0",
+]
 test = []
 
 [project.entry-points.opentelemetry_configurator]

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 import os
 
-from amazon.opentelemetry.distro._instrumentation_patch import apply_instrumentation_patches
+from amazon.opentelemetry.distro.patches._instrumentation_patch import apply_instrumentation_patches
 from opentelemetry.distro import OpenTelemetryDistro
 from opentelemetry.environment_variables import OTEL_PROPAGATORS, OTEL_PYTHON_ID_GENERATOR
 from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_instrumentation_patch.py renamed to aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py

@@ -2,31 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 # Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
 import importlib
-import ssl
-from urllib.request import Request, urlopen
 
-import opentelemetry.sdk.extension.aws.resource.ec2 as ec2_resource
-import opentelemetry.sdk.extension.aws.resource.eks as eks_resource
 from opentelemetry.instrumentation.botocore.extensions import _KNOWN_EXTENSIONS
 from opentelemetry.instrumentation.botocore.extensions.sqs import _SqsExtension
 from opentelemetry.instrumentation.botocore.extensions.types import _AttributeMapT, _AwsSdkExtension
 from opentelemetry.semconv.trace import SpanAttributes
 
 
-def apply_instrumentation_patches() -> None:
-    """Apply patches to upstream instrumentation libraries.
-
-    This method is invoked to apply changes to upstream instrumentation libraries, typically when changes to upstream
-    are required on a timeline that cannot wait for upstream release. Generally speaking, patches should be short-term
-    local solutions that are comparable to long-term upstream solutions.
-
-    Where possible, automated testing should be run to catch upstream changes resulting in broken patches
-    """
-    _apply_botocore_instrumentation_patches()
-
-    _apply_resource_detector_patches()
-
-
 def _apply_botocore_instrumentation_patches() -> None:
     """Botocore instrumentation patches
 
@@ -37,38 +19,6 @@ def _apply_botocore_instrumentation_patches() -> None:
     _apply_botocore_sqs_patch()
 
 
-# The OpenTelemetry Authors code
-def _apply_resource_detector_patches() -> None:
-    """AWS Resource Detector patches for getting the following unreleased change (as of v2.0.1) in the upstream:
-    https://github.com/open-telemetry/opentelemetry-python-contrib/commit/a5ec3f7f55494cb80b4b53c652e31c465b8d5e80
-    """
-
-    def patch_ec2_aws_http_request(method, path, headers):
-        with urlopen(
-            Request("http://169.254.169.254" + path, headers=headers, method=method),
-            timeout=5,
-        ) as response:
-            return response.read().decode("utf-8")
-
-    def patch_eks_aws_http_request(method, path, cred_value):
-        with urlopen(
-            Request(
-                "https://kubernetes.default.svc" + path,
-                headers={"Authorization": cred_value},
-                method=method,
-            ),
-            timeout=5,
-            context=ssl.create_default_context(cafile="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"),
-        ) as response:
-            return response.read().decode("utf-8")
-
-    ec2_resource._aws_http_request = patch_ec2_aws_http_request
-    eks_resource._aws_http_request = patch_eks_aws_http_request
-
-
-# END The OpenTelemetry Authors code
-
-
 def _apply_botocore_kinesis_patch() -> None:
     """Botocore instrumentation patch for Kinesis
 

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_instrumentation_patch.py

@@ -0,0 +1,45 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+# Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
+import sys
+from logging import Logger, getLogger
+
+import pkg_resources
+
+from amazon.opentelemetry.distro.patches._resource_detector_patches import _apply_resource_detector_patches
+
+_logger: Logger = getLogger(__name__)
+
+
+def apply_instrumentation_patches() -> None:
+    """Apply patches to upstream instrumentation libraries.
+
+    This method is invoked to apply changes to upstream instrumentation libraries, typically when changes to upstream
+    are required on a timeline that cannot wait for upstream release. Generally speaking, patches should be short-term
+    local solutions that are comparable to long-term upstream solutions.
+
+    Where possible, automated testing should be run to catch upstream changes resulting in broken patches
+    """
+
+    if _is_installed("botocore ~= 1.0"):
+        # pylint: disable=import-outside-toplevel
+        # Delay import to only occur if patches is safe to apply (e.g. the instrumented library is installed).
+        from amazon.opentelemetry.distro.patches._botocore_patches import _apply_botocore_instrumentation_patches
+
+        _apply_botocore_instrumentation_patches()
+
+    # No need to check if library is installed as this patches opentelemetry.sdk,
+    # which must be installed for the distro to work at all.
+    _apply_resource_detector_patches()
+
+
+def _is_installed(req: str) -> bool:
+    if req in sys.modules:
+        return True
+
+    try:
+        pkg_resources.get_distribution(req)
+    except Exception as exc:  # pylint: disable=broad-except
+        _logger.debug("Skipping instrumentation patch: package %s, exception: %s", req, exc)
+        return False
+    return True

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_resource_detector_patches.py

@@ -0,0 +1,40 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+# Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
+import ssl
+from urllib.request import Request, urlopen
+
+import opentelemetry.sdk.extension.aws.resource.ec2 as ec2_resource
+import opentelemetry.sdk.extension.aws.resource.eks as eks_resource
+
+
+# The OpenTelemetry Authors code
+def _apply_resource_detector_patches() -> None:
+    """AWS Resource Detector patches for getting the following unreleased change (as of v2.0.1) in the upstream:
+    https://github.com/open-telemetry/opentelemetry-python-contrib/commit/a5ec3f7f55494cb80b4b53c652e31c465b8d5e80
+    """
+
+    def patch_ec2_aws_http_request(method, path, headers):
+        with urlopen(
+            Request("http://169.254.169.254" + path, headers=headers, method=method),
+            timeout=5,
+        ) as response:
+            return response.read().decode("utf-8")
+
+    def patch_eks_aws_http_request(method, path, cred_value):
+        with urlopen(
+            Request(
+                "https://kubernetes.default.svc" + path,
+                headers={"Authorization": cred_value},
+                method=method,
+            ),
+            timeout=5,
+            context=ssl.create_default_context(cafile="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"),
+        ) as response:
+            return response.read().decode("utf-8")
+
+    ec2_resource._aws_http_request = patch_ec2_aws_http_request
+    eks_resource._aws_http_request = patch_eks_aws_http_request
+
+
+# END The OpenTelemetry Authors code

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_instrumentation_patch.py

@@ -2,9 +2,11 @@
 # SPDX-License-Identifier: Apache-2.0
 from typing import Dict
 from unittest import TestCase
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
-from amazon.opentelemetry.distro._instrumentation_patch import apply_instrumentation_patches
+import pkg_resources
+
+from amazon.opentelemetry.distro.patches._instrumentation_patch import apply_instrumentation_patches
 from opentelemetry.instrumentation.botocore.extensions import _KNOWN_EXTENSIONS
 from opentelemetry.semconv.trace import SpanAttributes
 
@@ -15,10 +17,43 @@
 
 
 class TestInstrumentationPatch(TestCase):
-    def test_apply_instrumentation_patches(self):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.mock_get_distribution = patch(
+            "amazon.opentelemetry.distro.patches._instrumentation_patch.pkg_resources.get_distribution"
+        ).start()
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+        cls.mock_get_distribution.stop()
+
+    def test_botocore_not_installed(self):
+        # Test scenario 1: Botocore package not installed
+        self.mock_get_distribution.side_effect = pkg_resources.DistributionNotFound
+        apply_instrumentation_patches()
+        with patch(
+            "amazon.opentelemetry.distro.patches._botocore_patches._apply_botocore_instrumentation_patches"
+        ) as mock_apply_patches:
+            mock_apply_patches.assert_not_called()
+
+    def test_botocore_installed_wrong_version(self):
+        # Test scenario 2: Botocore package installed with wrong version
+        self.mock_get_distribution.side_effect = pkg_resources.VersionConflict("botocore==1.0.0", "botocore==0.0.1")
+        apply_instrumentation_patches()
+        with patch(
+            "amazon.opentelemetry.distro.patches._botocore_patches._apply_botocore_instrumentation_patches"
+        ) as mock_apply_patches:
+            mock_apply_patches.assert_not_called()
+
+    def test_botocore_installed_correct_version(self):
+        # Test scenario 3: Botocore package installed with correct version
         # Validate unpatched upstream behaviour - important to detect upstream changes that may break instrumentation
         self._validate_unpatched_botocore_instrumentation()
 
+        self.mock_get_distribution.return_value = "CorrectDistributionObject"
+
         # Apply patches
         apply_instrumentation_patches()
 

dev-requirements.txt

@@ -14,3 +14,4 @@ codespell==2.1.0
 requests==2.31.0
 ruamel.yaml==0.17.21
 flaky==3.7.0
+botocore==1.34.67

tox.ini

@@ -29,6 +29,7 @@ commands_pre =
   3.{7,8,9,10,11}: python -m pip install -U pip setuptools wheel
 ; Install common packages for all the tests. These are not needed in all the
 ; cases but it saves a lot of boilerplate in this file.
+  test: pip install botocore
   test: pip install "opentelemetry-api[test] @ {env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api"
   test: pip install "opentelemetry-sdk[test] @ {env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk"
   test: pip install "opentelemetry-instrumentation[test] @ {env:CONTRIB_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation"