Telemetry migration from opencensus to azure monitor opentelemetry (#37392)

* pre-commit hook changes

* use prod credentials

* fix unit tests

* add missing dependency

* remove unnecessary dependency

* add dependency

* remove opentelemetry-api

* add azure monitor dependency in setup

* add and remove dependencies

* remove from dev_requirements

* add new line

* readd dependencies due to azure identity issues

Author: PratibhaShrivastav18

sdk/ml/azure-ai-ml/CHANGELOG.md

@@ -8,6 +8,8 @@
 ### Bugs Fixed
   - #35820 - using compute location attribute to fill compute location to align the experience with UI.
 
+### Other Changes
+  - Added dependency on `azure-monitor-opentelemetry`.
 
 ## 1.20.0 (2024-09-10)
 

sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py

@@ -10,7 +10,7 @@
 from functools import singledispatch
 from itertools import product
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple, TypeVar, Union
+from typing import Any, Optional, Tuple, TypeVar, Union
 
 from azure.ai.ml._azure_environments import (
     CloudArgumentKeys,
@@ -43,7 +43,7 @@
     AzureMachineLearningWorkspaces as ServiceClientWorkspaceDataplane,
 )
 from azure.ai.ml._scope_dependent_operations import OperationConfig, OperationsContainer, OperationScope
-from azure.ai.ml._telemetry.logging_handler import get_appinsights_log_handler
+from azure.ai.ml._telemetry.logging_handler import set_appinsights_distro
 from azure.ai.ml._user_agent import USER_AGENT
 from azure.ai.ml._utils._experimental import experimental
 from azure.ai.ml._utils._http_utils import HttpPipeline
@@ -278,12 +278,11 @@ def __init__(
 
         user_agent = kwargs.get("user_agent", None)
 
-        app_insights_handler: Tuple = get_appinsights_log_handler(
+        set_appinsights_distro(
             user_agent,
             **{"properties": properties},
             enable_telemetry=self._operation_config.enable_telemetry,
         )
-        app_insights_handler_kwargs: Dict[str, Tuple] = {"app_insights_handler": app_insights_handler}
 
         base_url = _get_base_url_from_metadata(cloud_name=cloud_name, is_local_mfe=True)
         self._base_url = base_url
@@ -292,7 +291,7 @@ def __init__(
         self._operation_container = OperationsContainer()
 
         # kwargs related to operations alone not all kwargs passed to MLClient are needed by operations
-        ops_kwargs = app_insights_handler_kwargs
+        ops_kwargs = {}
         if base_url:
             ops_kwargs["enforce_https"] = _is_https_url(base_url)
 
@@ -479,7 +478,6 @@ def __init__(
             self._credential,
             requests_pipeline=self._requests_pipeline,
             dataplane_client=self._service_client_workspace_dataplane,
-            **app_insights_handler_kwargs,
         )
         self._operation_container.add(AzureMLResourceType.WORKSPACE, self._workspaces)  # type: ignore[arg-type]
 
@@ -497,7 +495,6 @@ def __init__(
             self._service_client_10_2022_preview,
             self._operation_container,
             self._credential,
-            **app_insights_handler_kwargs,  # type: ignore[arg-type]
         )
         self._operation_container.add(AzureMLResourceType.REGISTRY, self._registries)  # type: ignore[arg-type]
 
@@ -519,7 +516,6 @@ def __init__(
             self._operation_config,
             self._service_client_08_2023_preview,
             self._service_client_04_2024_preview,
-            **app_insights_handler_kwargs,  # type: ignore[arg-type]
         )
         self._operation_container.add(AzureMLResourceType.COMPUTE, self._compute)
         self._datastores = DatastoreOperations(
@@ -545,7 +541,6 @@ def __init__(
             workspace_rg=self._ws_rg,
             workspace_sub=self._ws_sub,
             registry_reference=registry_reference,
-            **app_insights_handler_kwargs,  # type: ignore[arg-type]
         )
         # Evaluators
         self._evaluators = EvaluatorOperations(
@@ -563,7 +558,6 @@ def __init__(
             workspace_rg=self._ws_rg,
             workspace_sub=self._ws_sub,
             registry_reference=registry_reference,
-            **app_insights_handler_kwargs,  # type: ignore[arg-type]
         )
 
         self._operation_container.add(AzureMLResourceType.MODEL, self._models)
@@ -705,7 +699,6 @@ def __init__(
             self._service_client_07_2024_preview,
             self._operation_container,
             self._credential,
-            **app_insights_handler_kwargs,  # type: ignore[arg-type]
         )
 
         self._featuresets = FeatureSetOperations(

sdk/ml/azure-ai-ml/azure/ai/ml/_telemetry/__init__.py

@@ -5,13 +5,13 @@
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)
 
 from .activity import ActivityType, log_activity, monitor_with_activity, monitor_with_telemetry_mixin
-from .logging_handler import AML_INTERNAL_LOGGER_NAMESPACE, get_appinsights_log_handler
+from .logging_handler import AML_INTERNAL_LOGGER_NAMESPACE, set_appinsights_distro
 
 __all__ = [
     "monitor_with_activity",
     "monitor_with_telemetry_mixin",
     "log_activity",
     "ActivityType",
-    "get_appinsights_log_handler",
+    "set_appinsights_distro",
     "AML_INTERNAL_LOGGER_NAMESPACE",
 ]

sdk/ml/azure-ai-ml/azure/ai/ml/_telemetry/activity.py

@@ -31,6 +31,8 @@
 # Get environment variable IS_IN_CI_PIPELINE to decide whether it's in CI test
 IS_IN_CI_PIPELINE = _str_to_bool(os.environ.get("IS_IN_CI_PIPELINE", "False"))
 
+ACTIVITY_SPAN = "activity_span"
+
 
 class ActivityType(object):
     """The type of activity (code) monitored.
@@ -93,10 +95,7 @@ def process(self, msg: str, kwargs: Dict) -> Tuple[str, Dict]:  # type: ignore[o
         if "extra" not in kwargs:
             kwargs["extra"] = {}
 
-        if "properties" not in kwargs["extra"]:
-            kwargs["extra"]["properties"] = {}
-
-        kwargs["extra"]["properties"].update(self._activity_info)
+        kwargs["extra"].update(self._activity_info)
 
         return msg, kwargs
 
@@ -282,7 +281,7 @@ def monitor(f):
         def wrapper(*args, **kwargs):
             tracer = logger.package_tracer if isinstance(logger, OpsLogger) else None
             if tracer:
-                with tracer.span():
+                with tracer.start_as_current_span(ACTIVITY_SPAN):
                     with log_activity(
                         logger.package_logger, activity_name or f.__name__, activity_type, custom_dimensions
                     ):