Add configuration for custom span processors (#34326)

Author: lzchen

sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### Features Added
 
+- Add custom span processors configuration option
+    ([#34326](https://github.com/Azure/azure-sdk-for-python/pull/34326))
+
 ### Bugs Fixed
 
 ### Other Changes

sdk/monitor/azure-monitor-opentelemetry/README.md

@@ -60,7 +60,7 @@ You can use `configure_azure_monitor` to set up instrumentation for your app to
 | `connection_string` | The [connection string][connection_string_doc] for your Application Insights resource. The connection string will be automatically populated from the `APPLICATIONINSIGHTS_CONNECTION_STRING` environment variable if not explicitly passed in. | `APPLICATIONINSIGHTS_CONNECTION_STRING` |
 | `logger_name` | The name of the [Python logger][python_logger] under which telemetry is collected. | `N/A` |
 | `instrumentation_options` | A nested dictionary that determines which instrumentations to enable or disable. Instrumentations are referred to by their [Library Names](#officially-supported-instrumentations). For example, `{"azure_sdk": {"enabled": False}, "flask": {"enabled": False}, "django": {"enabled": True}}` will disable Azure Core Tracing and the Flask instrumentation but leave Django and the other default instrumentations enabled. The `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS` environment variable explained below can also be used to disable instrumentations. | `N/A` |
-
+| `span_processors` | A list of [span processors][ot_span_processor] that will perform processing on each of your spans before they are exported. Useful for filtering/modifying telemetry. | `N/A` |
 
 You can configure further with [OpenTelemetry environment variables][ot_env_vars] such as:
 | Environment Variable | Description |
@@ -217,6 +217,7 @@ contact [[email protected]](mailto:[email protected]) with any additio
 [ot_python_docs]: https://opentelemetry.io/docs/instrumentation/python/
 [ot_sdk_python]: https://github.com/open-telemetry/opentelemetry-python
 [ot_sdk_python_metric_reader]: https://opentelemetry-python.readthedocs.io/en/stable/sdk/metrics.export.html#opentelemetry.sdk.metrics.export.MetricReader
+[ot_span_processor]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#span-processor
 [ot_sdk_python_view_examples]: https://github.com/open-telemetry/opentelemetry-python/tree/main/docs/examples/metrics/views
 [ot_instrumentation_django]: https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation/opentelemetry-instrumentation-django
 [ot_instrumentation_django_version]: https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/package.py#L16
@@ -249,4 +250,4 @@ contact [[email protected]](mailto:[email protected]) with any additio
 [python_logger]: https://docs.python.org/3/library/logging.html#logger-objects
 [python_logging_level]: https://docs.python.org/3/library/logging.html#levels
 [samples]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples
-[samples_manual]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/manual.py
+[samples_manual]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/manually_instrumented.py

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_configure.py

@@ -35,6 +35,7 @@
     LOGGER_NAME_ARG,
     RESOURCE_ARG,
     SAMPLING_RATIO_ARG,
+    SPAN_PROCESSORS_ARG,
 )
 from azure.monitor.opentelemetry._types import ConfigurationValue
 from azure.monitor.opentelemetry.exporter import (  # pylint: disable=import-error,no-name-in-module
@@ -68,6 +69,8 @@ def configure_azure_monitor(**kwargs) -> None:
      `{"azure_sdk": {"enabled": False}, "flask": {"enabled": False}, "django": {"enabled": True}}`
      will disable Azure Core Tracing and the Flask instrumentation but leave Django and the other default
      instrumentations enabled.
+    :keyword list[~opentelemetry.sdk.trace.SpanProcessor] span_processors: List of `SpanProcessor` objects
+     to process every span prior to exporting. Will be run sequentially.
     :keyword str storage_directory: Storage directory in which to store retry files. Defaults to
      `<tempfile.gettempdir()>/Microsoft/AzureMonitor/opentelemetry-python-<your-instrumentation-key>`.
     :rtype: None
@@ -105,11 +108,13 @@ def _setup_tracing(configurations: Dict[str, ConfigurationValue]):
         resource=resource
     )
     set_tracer_provider(tracer_provider)
+    for span_processor in configurations[SPAN_PROCESSORS_ARG]: # type: ignore
+        get_tracer_provider().add_span_processor(span_processor) # type: ignore
     trace_exporter = AzureMonitorTraceExporter(**configurations)
-    span_processor = BatchSpanProcessor(
+    bsp = BatchSpanProcessor(
         trace_exporter,
     )
-    get_tracer_provider().add_span_processor(span_processor) # type: ignore
+    get_tracer_provider().add_span_processor(bsp) # type: ignore
     if _is_instrumentation_enabled(configurations, _AZURE_SDK_INSTRUMENTATION_NAME):
         settings.tracing_implementation = OpenTelemetrySpan
 

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py

@@ -29,6 +29,7 @@
 INSTRUMENTATION_OPTIONS_ARG = "instrumentation_options"
 RESOURCE_ARG = "resource"
 SAMPLING_RATIO_ARG = "sampling_ratio"
+SPAN_PROCESSORS_ARG = "span_processors"
 
 
 # --------------------Diagnostic/status logging------------------------------

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_util/configurations.py

@@ -35,6 +35,7 @@
     LOGGER_NAME_ARG,
     RESOURCE_ARG,
     SAMPLING_RATIO_ARG,
+    SPAN_PROCESSORS_ARG,
 )
 from azure.monitor.opentelemetry._types import ConfigurationValue
 from azure.monitor.opentelemetry._version import VERSION
@@ -66,6 +67,7 @@ def _get_configurations(**kwargs) -> Dict[str, ConfigurationValue]:
     _default_resource(configurations)
     _default_sampling_ratio(configurations)
     _default_instrumentation_options(configurations)
+    _default_span_processors(configurations)
 
     return configurations
 
@@ -141,6 +143,11 @@ def _default_instrumentation_options(configurations):
     configurations[INSTRUMENTATION_OPTIONS_ARG] = merged_instrumentation_options
 
 
+def _default_span_processors(configurations):
+    if SPAN_PROCESSORS_ARG not in configurations:
+        configurations[SPAN_PROCESSORS_ARG] = []
+
+
 def _get_otel_disabled_instrumentations():
     disabled_instrumentation = environ.get(
         OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, ""

sdk/monitor/azure-monitor-opentelemetry/samples/tracing/filter_spans.py

@@ -0,0 +1,49 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+import requests
+from azure.monitor.opentelemetry import configure_azure_monitor
+from opentelemetry.sdk.trace import SpanProcessor
+from opentelemetry.trace import get_tracer, SpanContext, SpanKind, TraceFlags
+
+# Define a custom processor to filter your spans
+class SpanFilteringProcessor(SpanProcessor):
+
+    # Prevents exporting spans that are of kind INTERNAL
+    def on_start(self, span, parent_context):
+        # Check if the span is an internal activity.
+        if span._kind is SpanKind.INTERNAL:
+            # Create a new span context with the following properties:
+            #   * The trace ID is the same as the trace ID of the original span.
+            #   * The span ID is the same as the span ID of the original span.
+            #   * The is_remote property is set to `False`.
+            #   * The trace flags are set to `DEFAULT`.
+            #   * The trace state is the same as the trace state of the original span.
+            span._context = SpanContext(
+                span.context.trace_id,
+                span.context.span_id,
+                span.context.is_remote,
+                TraceFlags(TraceFlags.DEFAULT),
+                span.context.trace_state,
+            )
+
+# Create a SpanFilteringProcessor instance.
+span_filter_processor = SpanFilteringProcessor()
+
+# Pass in your processor to configuration options
+configure_azure_monitor(
+    span_processors=[span_filter_processor]
+)
+
+tracer = get_tracer(__name__)
+
+with tracer.start_as_current_span("this_span_is_ignored"):
+    # Requests made using the requests library will be automatically captured
+    # The span generated from this request call will be tracked since it is not an INTERNAL span
+    response = requests.get("https://azure.microsoft.com/", timeout=5)
+    print("Hello, World!")
+
+input()

sdk/monitor/azure-monitor-opentelemetry/samples/tracing/manual.py renamed to sdk/monitor/azure-monitor-opentelemetry/samples/tracing/manually_instrumented.py

@@ -0,0 +1,35 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+from azure.monitor.opentelemetry import configure_azure_monitor
+from opentelemetry import trace
+from opentelemetry.sdk.trace import SpanProcessor
+
+# Define a custom processor to modify your spans
+class SpanEnrichingProcessor(SpanProcessor):
+
+    def on_end(self, span):
+        # Prefix the span name with the string "Updated-".
+        span._name = "Updated-" + span.name
+        # Add the custom dimension "CustomDimension1" with the value "Value1".
+        span._attributes["CustomDimension1"] = "Value1"
+         # Add the custom dimension "CustomDimension2" with the value "Value2".
+        span._attributes["CustomDimension2"] = "Value2"
+
+# Create a SpanEnrichingProcessor instance.
+span_enrich_processor = SpanEnrichingProcessor()
+
+# Pass in your processor to configuration options
+configure_azure_monitor(
+    span_processors=[span_enrich_processor]
+)
+
+tracer = trace.get_tracer(__name__)
+
+with tracer.start_as_current_span("this_span_will_be_modified"):
+    print("Hello, World!")
+
+input()

sdk/monitor/azure-monitor-opentelemetry/samples/tracing/modify_spans.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import unittest
-from unittest.mock import Mock, patch
+from unittest.mock import Mock, call, patch
 
 from opentelemetry.sdk.resources import Resource
 
@@ -218,13 +218,15 @@ def test_setup_tracing(
         trace_exporter_mock.return_value = trace_exp_init_mock
         bsp_init_mock = Mock()
         bsp_mock.return_value = bsp_init_mock
+        custom_sp = Mock()
 
         configurations = {
             "connection_string": "test_cs",
             "instrumentation_options": {
                 "azure_sdk": {"enabled": True}
             },
             "sampling_ratio": 0.5,
+            "span_processors": [custom_sp],
             "resource": TEST_RESOURCE,
         }
         _setup_tracing(configurations)
@@ -237,7 +239,8 @@ def test_setup_tracing(
         get_tracer_provider_mock.assert_called()
         trace_exporter_mock.assert_called_once_with(**configurations)
         bsp_mock.assert_called_once_with(trace_exp_init_mock)
-        tp_init_mock.add_span_processor.assert_called_once_with(bsp_init_mock)
+        self.assertEqual(tp_init_mock.add_span_processor.call_count, 2)
+        tp_init_mock.add_span_processor.assert_has_calls([call(custom_sp), call(bsp_init_mock)])
         self.assertEqual(
             azure_core_mock.tracing_implementation, OpenTelemetrySpan
         )