Skip to content

Commit 5d2f42b

Browse files
PratibhaShrivastav18PratibhaShrivastav18
authored
Azure monitor opentelemetry migration (Azure#37663)
* pre-commit hook changes * use prod credentials * fix unit tests * add missing dependency * remove unnecessary dependency * add dependency * remove opentelemetry-api * add azure monitor dependency in setup * add and remove dependencies * remove from dev_requirements * add new line * readd dependencies due to azure identity issues * update * update * update * fix tests * remove opencensus dependencies * address review comments * fix change * add docstrings
1 parent 1cabd16 commit 5d2f42b

36 files changed

+251
-403
lines changed

‎sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py‎

Lines changed: 20 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from functools import singledispatch
1111
from itertools import product
1212
from pathlib import Path
13-
from typing import Any, Dict, Optional, Tuple, TypeVar, Union
13+
from typing import Any, Optional, Tuple, TypeVar, Union
1414

1515
from azure.ai.ml._azure_environments import (
1616
CloudArgumentKeys,
@@ -24,57 +24,27 @@
2424
from azure.ai.ml._restclient.v2020_09_01_dataplanepreview import (
2525
AzureMachineLearningWorkspaces as ServiceClient092020DataplanePreview,
2626
)
27-
from azure.ai.ml._restclient.v2022_02_01_preview import (
28-
AzureMachineLearningWorkspaces as ServiceClient022022Preview,
29-
)
30-
from azure.ai.ml._restclient.v2022_05_01 import (
31-
AzureMachineLearningWorkspaces as ServiceClient052022,
32-
)
33-
from azure.ai.ml._restclient.v2022_10_01 import (
34-
AzureMachineLearningWorkspaces as ServiceClient102022,
35-
)
36-
from azure.ai.ml._restclient.v2022_10_01_preview import (
37-
AzureMachineLearningWorkspaces as ServiceClient102022Preview,
38-
)
39-
from azure.ai.ml._restclient.v2023_02_01_preview import (
40-
AzureMachineLearningWorkspaces as ServiceClient022023Preview,
41-
)
42-
from azure.ai.ml._restclient.v2023_04_01 import (
43-
AzureMachineLearningWorkspaces as ServiceClient042023,
44-
)
45-
from azure.ai.ml._restclient.v2023_04_01_preview import (
46-
AzureMachineLearningWorkspaces as ServiceClient042023Preview,
47-
)
48-
from azure.ai.ml._restclient.v2023_06_01_preview import (
49-
AzureMachineLearningWorkspaces as ServiceClient062023Preview,
50-
)
51-
from azure.ai.ml._restclient.v2023_08_01_preview import (
52-
AzureMachineLearningWorkspaces as ServiceClient082023Preview,
53-
)
27+
from azure.ai.ml._restclient.v2022_02_01_preview import AzureMachineLearningWorkspaces as ServiceClient022022Preview
28+
from azure.ai.ml._restclient.v2022_05_01 import AzureMachineLearningWorkspaces as ServiceClient052022
29+
from azure.ai.ml._restclient.v2022_10_01 import AzureMachineLearningWorkspaces as ServiceClient102022
30+
from azure.ai.ml._restclient.v2022_10_01_preview import AzureMachineLearningWorkspaces as ServiceClient102022Preview
31+
from azure.ai.ml._restclient.v2023_02_01_preview import AzureMachineLearningWorkspaces as ServiceClient022023Preview
32+
from azure.ai.ml._restclient.v2023_04_01 import AzureMachineLearningWorkspaces as ServiceClient042023
33+
from azure.ai.ml._restclient.v2023_04_01_preview import AzureMachineLearningWorkspaces as ServiceClient042023Preview
34+
from azure.ai.ml._restclient.v2023_06_01_preview import AzureMachineLearningWorkspaces as ServiceClient062023Preview
35+
from azure.ai.ml._restclient.v2023_08_01_preview import AzureMachineLearningWorkspaces as ServiceClient082023Preview
5436

5537
# Same object, but was renamed starting in v2023_08_01_preview
5638
from azure.ai.ml._restclient.v2023_10_01 import AzureMachineLearningServices as ServiceClient102023
57-
from azure.ai.ml._restclient.v2024_01_01_preview import (
58-
AzureMachineLearningWorkspaces as ServiceClient012024Preview,
59-
)
60-
from azure.ai.ml._restclient.v2024_04_01_preview import (
61-
AzureMachineLearningWorkspaces as ServiceClient042024Preview,
62-
)
63-
from azure.ai.ml._restclient.v2024_07_01_preview import (
64-
AzureMachineLearningWorkspaces as ServiceClient072024Preview,
65-
)
66-
from azure.ai.ml._restclient.v2024_10_01_preview import (
67-
AzureMachineLearningWorkspaces as ServiceClient102024Preview,
68-
)
39+
from azure.ai.ml._restclient.v2024_01_01_preview import AzureMachineLearningWorkspaces as ServiceClient012024Preview
40+
from azure.ai.ml._restclient.v2024_04_01_preview import AzureMachineLearningWorkspaces as ServiceClient042024Preview
41+
from azure.ai.ml._restclient.v2024_07_01_preview import AzureMachineLearningWorkspaces as ServiceClient072024Preview
42+
from azure.ai.ml._restclient.v2024_10_01_preview import AzureMachineLearningWorkspaces as ServiceClient102024Preview
6943
from azure.ai.ml._restclient.workspace_dataplane import (
7044
AzureMachineLearningWorkspaces as ServiceClientWorkspaceDataplane,
7145
)
72-
from azure.ai.ml._scope_dependent_operations import (
73-
OperationConfig,
74-
OperationsContainer,
75-
OperationScope,
76-
)
77-
from azure.ai.ml._telemetry.logging_handler import get_appinsights_log_handler
46+
from azure.ai.ml._scope_dependent_operations import OperationConfig, OperationsContainer, OperationScope
47+
from azure.ai.ml._telemetry.logging_handler import configure_appinsights_logging
7848
from azure.ai.ml._user_agent import USER_AGENT
7949
from azure.ai.ml._utils._experimental import experimental
8050
from azure.ai.ml._utils._http_utils import HttpPipeline
@@ -126,17 +96,15 @@
12696
WorkspaceConnectionsOperations,
12797
WorkspaceOperations,
12898
)
99+
from azure.ai.ml.operations._capability_hosts_operations import CapabilityHostsOperations
129100
from azure.ai.ml.operations._code_operations import CodeOperations
130101
from azure.ai.ml.operations._feature_set_operations import FeatureSetOperations
131102
from azure.ai.ml.operations._feature_store_entity_operations import FeatureStoreEntityOperations
132103
from azure.ai.ml.operations._feature_store_operations import FeatureStoreOperations
133104
from azure.ai.ml.operations._local_deployment_helper import _LocalDeploymentHelper
134105
from azure.ai.ml.operations._local_endpoint_helper import _LocalEndpointHelper
135106
from azure.ai.ml.operations._schedule_operations import ScheduleOperations
136-
from azure.ai.ml.operations._capability_hosts_operations import CapabilityHostsOperations
137-
from azure.ai.ml.operations._workspace_outbound_rule_operations import (
138-
WorkspaceOutboundRuleOperations,
139-
)
107+
from azure.ai.ml.operations._workspace_outbound_rule_operations import WorkspaceOutboundRuleOperations
140108
from azure.core.credentials import TokenCredential
141109
from azure.core.polling import LROPoller
142110

@@ -316,12 +284,11 @@ def __init__(
316284

317285
user_agent = kwargs.get("user_agent", None)
318286

319-
app_insights_handler: Tuple = get_appinsights_log_handler(
287+
configure_appinsights_logging(
320288
user_agent,
321289
**{"properties": properties},
322290
enable_telemetry=self._operation_config.enable_telemetry,
323291
)
324-
app_insights_handler_kwargs: Dict[str, Tuple] = {"app_insights_handler": app_insights_handler}
325292

326293
base_url = _get_base_url_from_metadata(cloud_name=cloud_name, is_local_mfe=True)
327294
self._base_url = base_url
@@ -330,7 +297,7 @@ def __init__(
330297
self._operation_container = OperationsContainer()
331298

332299
# kwargs related to operations alone not all kwargs passed to MLClient are needed by operations
333-
ops_kwargs = app_insights_handler_kwargs
300+
ops_kwargs = {}
334301
if base_url:
335302
ops_kwargs["enforce_https"] = _is_https_url(base_url)
336303

@@ -532,7 +499,6 @@ def __init__(
532499
self._credential,
533500
requests_pipeline=self._requests_pipeline,
534501
dataplane_client=self._service_client_workspace_dataplane,
535-
**app_insights_handler_kwargs,
536502
)
537503
self._operation_container.add(AzureMLResourceType.WORKSPACE, self._workspaces) # type: ignore[arg-type]
538504

@@ -550,7 +516,6 @@ def __init__(
550516
self._service_client_10_2022_preview,
551517
self._operation_container,
552518
self._credential,
553-
**app_insights_handler_kwargs, # type: ignore[arg-type]
554519
)
555520
self._operation_container.add(AzureMLResourceType.REGISTRY, self._registries) # type: ignore[arg-type]
556521

@@ -582,7 +547,6 @@ def __init__(
582547
self._operation_config,
583548
self._service_client_08_2023_preview,
584549
self._service_client_04_2024_preview,
585-
**app_insights_handler_kwargs, # type: ignore[arg-type]
586550
)
587551
self._operation_container.add(AzureMLResourceType.COMPUTE, self._compute)
588552
self._datastores = DatastoreOperations(
@@ -608,7 +572,6 @@ def __init__(
608572
workspace_rg=self._ws_rg,
609573
workspace_sub=self._ws_sub,
610574
registry_reference=registry_reference,
611-
**app_insights_handler_kwargs, # type: ignore[arg-type]
612575
)
613576
# Evaluators
614577
self._evaluators = EvaluatorOperations(
@@ -626,7 +589,6 @@ def __init__(
626589
workspace_rg=self._ws_rg,
627590
workspace_sub=self._ws_sub,
628591
registry_reference=registry_reference,
629-
**app_insights_handler_kwargs, # type: ignore[arg-type]
630592
)
631593

632594
self._operation_container.add(AzureMLResourceType.MODEL, self._models)
@@ -770,7 +732,6 @@ def __init__(
770732
self._service_client_10_2024_preview,
771733
self._operation_container,
772734
self._credential,
773-
**app_insights_handler_kwargs, # type: ignore[arg-type]
774735
)
775736

776737
self._featuresets = FeatureSetOperations(

‎sdk/ml/azure-ai-ml/azure/ai/ml/_telemetry/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,13 @@
55
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
66

77
from .activity import ActivityType, log_activity, monitor_with_activity, monitor_with_telemetry_mixin
8-
from .logging_handler import AML_INTERNAL_LOGGER_NAMESPACE, get_appinsights_log_handler
8+
from .logging_handler import AML_INTERNAL_LOGGER_NAMESPACE, configure_appinsights_logging
99

1010
__all__ = [
1111
"monitor_with_activity",
1212
"monitor_with_telemetry_mixin",
1313
"log_activity",
1414
"ActivityType",
15-
"get_appinsights_log_handler",
15+
"configure_appinsights_logging",
1616
"AML_INTERNAL_LOGGER_NAMESPACE",
1717
]

‎sdk/ml/azure-ai-ml/azure/ai/ml/_telemetry/activity.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
# Get environment variable IS_IN_CI_PIPELINE to decide whether it's in CI test
3232
IS_IN_CI_PIPELINE = _str_to_bool(os.environ.get("IS_IN_CI_PIPELINE", "False"))
3333

34+
ACTIVITY_SPAN = "activity_span"
35+
3436

3537
class ActivityType(object):
3638
"""The type of activity (code) monitored.
@@ -93,10 +95,7 @@ def process(self, msg: str, kwargs: Dict) -> Tuple[str, Dict]: # type: ignore[o
9395
if "extra" not in kwargs:
9496
kwargs["extra"] = {}
9597

96-
if "properties" not in kwargs["extra"]:
97-
kwargs["extra"]["properties"] = {}
98-
99-
kwargs["extra"]["properties"].update(self._activity_info)
98+
kwargs["extra"].update(self._activity_info)
10099

101100
return msg, kwargs
102101

@@ -282,7 +281,7 @@ def monitor(f):
282281
def wrapper(*args, **kwargs):
283282
tracer = logger.package_tracer if isinstance(logger, OpsLogger) else None
284283
if tracer:
285-
with tracer.span():
284+
with tracer.start_as_current_span(ACTIVITY_SPAN):
286285
with log_activity(
287286
logger.package_logger, activity_name or f.__name__, activity_type, custom_dimensions
288287
):

0 commit comments

Comments
 (0)