Momo GA Changes (#34029)

Author: nemanjarajic

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/monitoring/monitor_definition.py

@@ -16,6 +16,7 @@
     FeatureAttributionDriftSignalSchema,
     CustomMonitoringSignalSchema,
     GenerationSafetyQualitySchema,
+    ModelPerformanceSignalSchema,
 )
 from azure.ai.ml._schema.monitoring.alert_notification import AlertNotificationSchema
 from azure.ai.ml._schema.core.fields import NestedField, UnionField, StringTransformedEnum
@@ -35,6 +36,7 @@ class MonitorDefinitionSchema(metaclass=PatchedSchemaMeta):
                 NestedField(FeatureAttributionDriftSignalSchema),
                 NestedField(CustomMonitoringSignalSchema),
                 NestedField(GenerationSafetyQualitySchema),
+                NestedField(ModelPerformanceSignalSchema),
             ]
         ),
     )

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/monitoring/signals.py

@@ -75,6 +75,7 @@ class ProductionDataSchema(metaclass=PatchedSchemaMeta):
     data_context = StringTransformedEnum(allowed_values=[o.value for o in MonitorDatasetContext])
     pre_processing_component = fields.Str()
     data_window = NestedField(BaselineDataRangeSchema)
+    data_column_names = fields.Dict(keys=fields.Str(), values=fields.Str())
 
     @post_load
     def make(self, data, **kwargs):
@@ -89,6 +90,7 @@ class ReferenceDataSchema(metaclass=PatchedSchemaMeta):
     pre_processing_component = fields.Str()
     target_column_name = fields.Str()
     data_window = NestedField(BaselineDataRangeSchema)
+    data_column_names = fields.Dict(keys=fields.Str(), values=fields.Str())
 
     @post_load
     def make(self, data, **kwargs):
@@ -221,10 +223,14 @@ def make(self, data, **kwargs):
         return FeatureAttributionDriftSignal(**data)
 
 
-class ModelPerformanceSignalSchema(ModelSignalSchema):
+class ModelPerformanceSignalSchema(metaclass=PatchedSchemaMeta):
     type = StringTransformedEnum(allowed_values=MonitorSignalType.MODEL_PERFORMANCE, required=True)
+    production_data = NestedField(ProductionDataSchema)
+    reference_data = NestedField(ReferenceDataSchema)
     data_segment = NestedField(DataSegmentSchema)
+    alert_enabled = fields.Bool()
     metric_thresholds = NestedField(ModelPerformanceMetricThresholdSchema)
+    properties = fields.Dict()
 
     @pre_dump
     def predump(self, data, **kwargs):
@@ -299,7 +305,7 @@ class GenerationSafetyQualitySchema(metaclass=PatchedSchemaMeta):
     metric_thresholds = NestedField(GenerationSafetyQualityMetricThresholdSchema)
     alert_enabled = fields.Bool()
     properties = fields.Dict()
-    sampling_rate = fields.Int()
+    sampling_rate = fields.Float()
 
     @pre_dump
     def predump(self, data, **kwargs):

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/monitoring/thresholds.py

@@ -2,11 +2,11 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-# pylint: disable=unused-argument, line-too-long
+# pylint: disable=unused-argument, line-too-long, name-too-long
 
 from marshmallow import fields, post_load
 
-from azure.ai.ml.constants._monitoring import MonitorFeatureType, MonitorMetricName
+from azure.ai.ml.constants._monitoring import MonitorFeatureType
 from azure.ai.ml._schema.core.fields import StringTransformedEnum, NestedField
 from azure.ai.ml._schema.core.schema import PatchedSchemaMeta
 
@@ -112,18 +112,33 @@ def make(self, data, **kwargs):
         return FeatureAttributionDriftMetricThreshold(**data)
 
 
+class ModelPerformanceClassificationThresholdsSchema(metaclass=PatchedSchemaMeta):
+    accuracy = fields.Number()
+    precision = fields.Number()
+    recall = fields.Number()
+
+    @post_load
+    def make(self, data, **kwargs):
+        from azure.ai.ml.entities._monitoring.thresholds import ModelPerformanceClassificationThresholds
+
+        return ModelPerformanceClassificationThresholds(**data)
+
+
+class ModelPerformanceRegressionThresholdsSchema(metaclass=PatchedSchemaMeta):
+    mae = fields.Number()
+    mse = fields.Number()
+    rmse = fields.Number()
+
+    @post_load
+    def make(self, data, **kwargs):
+        from azure.ai.ml.entities._monitoring.thresholds import ModelPerformanceRegressionThresholds
+
+        return ModelPerformanceRegressionThresholds(**data)
+
+
 class ModelPerformanceMetricThresholdSchema(MetricThresholdSchema):
-    metric_name = StringTransformedEnum(
-        allowed_values=[
-            MonitorMetricName.ACCURACY,
-            MonitorMetricName.PRECISION,
-            MonitorMetricName.RECALL,
-            MonitorMetricName.F1_SCORE,
-            MonitorMetricName.MAE,
-            MonitorMetricName.MSE,
-            MonitorMetricName.RMSE,
-        ]
-    )
+    classification = NestedField(ModelPerformanceClassificationThresholdsSchema)
+    regression = NestedField(ModelPerformanceRegressionThresholdsSchema)
 
     @post_load
     def make(self, data, **kwargs):

sdk/ml/azure-ai-ml/azure/ai/ml/entities/__init__.py

@@ -169,6 +169,7 @@
     PredictionDriftSignal,
     ProductionData,
     ReferenceData,
+    ModelPerformanceSignal,
 )
 from ._monitoring.target import MonitoringTarget
 from ._monitoring.thresholds import (
@@ -182,6 +183,9 @@
     GenerationSafetyQualityMonitoringMetricThreshold,
     NumericalDriftMetrics,
     PredictionDriftMetricThreshold,
+    ModelPerformanceMetricThreshold,
+    ModelPerformanceClassificationThresholds,
+    ModelPerformanceRegressionThresholds,
 )
 from ._notification.notification import Notification
 from ._registry.registry import Registry
@@ -434,6 +438,7 @@
     "FeatureAttributionDriftSignal",
     "CustomMonitoringSignal",
     "GenerationSafetyQualitySignal",
+    "ModelPerformanceSignal",
     "MonitorFeatureFilter",
     "DataSegment",
     "FADProductionData",
@@ -455,6 +460,9 @@
     "NumericalDriftMetrics",
     "DataQualityMetricsNumerical",
     "DataQualityMetricsCategorical",
+    "ModelPerformanceMetricThreshold",
+    "ModelPerformanceClassificationThresholds",
+    "ModelPerformanceRegressionThresholds",
     "DataCollector",
     "IntellectualProperty",
     "DeploymentCollection",

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_monitoring/alert_notification.py

@@ -9,11 +9,9 @@
     EmailNotificationEnableType,
     NotificationSetting,
 )
-from azure.ai.ml._utils._experimental import experimental
 from azure.ai.ml.entities._mixins import RestTranslatableMixin
 
 
-@experimental
 class AlertNotification(RestTranslatableMixin):
     """Alert notification configuration for monitoring jobs
 

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_monitoring/compute.py

@@ -4,11 +4,9 @@
 
 from azure.ai.ml._exception_helper import log_and_raise_error
 from azure.ai.ml._restclient.v2023_06_01_preview.models import AmlTokenComputeIdentity, MonitorServerlessSparkCompute
-from azure.ai.ml._utils._experimental import experimental
 from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, ValidationErrorType, ValidationException
 
 
-@experimental
 class ServerlessSparkCompute:
     """Serverless Spark compute.
 

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_monitoring/definition.py

@@ -10,7 +10,6 @@
 
 from azure.ai.ml._restclient.v2023_06_01_preview.models import AzMonMonitoringAlertNotificationSettings
 from azure.ai.ml._restclient.v2023_06_01_preview.models import MonitorDefinition as RestMonitorDefinition
-from azure.ai.ml._utils._experimental import experimental
 from azure.ai.ml.constants._monitoring import (
     AZMONITORING,
     DEFAULT_DATA_DRIFT_SIGNAL_NAME,
@@ -29,7 +28,6 @@
 from azure.ai.ml.entities._monitoring.target import MonitoringTarget
 
 
-@experimental
 class MonitorDefinition(RestTranslatableMixin):
     """Monitor definition
 
@@ -44,7 +42,8 @@ class MonitorDefinition(RestTranslatableMixin):
         , ~azure.ai.ml.entities.DataQualitySignal, ~azure.ai.ml.entities.PredictionDriftSignal
         , ~azure.ai.ml.entities.FeatureAttributionDriftSignal
         , ~azure.ai.ml.entities.CustomMonitoringSignal
-        , ~azure.ai.ml.entities.GenerationSafetyQualitySignal]]]
+        , ~azure.ai.ml.entities.GenerationSafetyQualitySignal
+        , ~azure.ai.ml.entities.ModelPerformanceSignal]]]
     :keyword alert_notification: The alert configuration for the monitor.
     :paramtype alert_notification: Optional[Union[Literal['azmonitoring'], ~azure.ai.ml.entities.AlertNotification]]
 
@@ -73,6 +72,7 @@ def __init__(
 
     def _to_rest_object(self, **kwargs: Any) -> RestMonitorDefinition:
         default_data_window_size = kwargs.get("default_data_window_size")
+        ref_data_window_size = kwargs.get("ref_data_window_size")
         rest_alert_notification = None
         if self.alert_notification:
             if isinstance(self.alert_notification, str) and self.alert_notification.lower() == AZMONITORING:
@@ -85,6 +85,7 @@ def _to_rest_object(self, **kwargs: Any) -> RestMonitorDefinition:
             _signals = {
                 signal_name: signal._to_rest_object(
                     default_data_window_size=default_data_window_size,
+                    ref_data_window_size=ref_data_window_size,
                 )
                 for signal_name, signal in self.monitoring_signals.items()
             }

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_monitoring/input_data.py

@@ -13,13 +13,11 @@
 from azure.ai.ml._restclient.v2023_06_01_preview.models import MonitoringInputDataBase as RestMonitorInputBase
 from azure.ai.ml._restclient.v2023_06_01_preview.models import StaticInputData as RestStaticInputData
 from azure.ai.ml._restclient.v2023_06_01_preview.models import TrailingInputData as RestTrailingInputData
-from azure.ai.ml._utils._experimental import experimental
 from azure.ai.ml._utils.utils import camel_to_snake, snake_to_camel
 from azure.ai.ml.constants._monitoring import MonitorDatasetContext, MonitorInputDataType
 from azure.ai.ml.entities._mixins import RestTranslatableMixin
 
 
-@experimental
 class MonitorInputData(RestTranslatableMixin):
     """Monitor input data.
 
@@ -64,7 +62,6 @@ def _from_rest_object(cls, obj: RestMonitorInputBase) -> Optional["MonitorInputD
         return None
 
 
-@experimental
 class FixedInputData(MonitorInputData):
     """
     :ivar type: Specifies the type of monitoring input data. Set automatically to "Fixed" for this class.
@@ -105,7 +102,6 @@ def _from_rest_object(cls, obj: RestFixedInputData) -> "FixedInputData":
         )
 
 
-@experimental
 class TrailingInputData(MonitorInputData):
     """
     :ivar type: Specifies the type of monitoring input data. Set automatically to "Trailing" for this class.
@@ -158,7 +154,6 @@ def _from_rest_object(cls, obj: RestTrailingInputData) -> "TrailingInputData":
         )
 
 
-@experimental
 class StaticInputData(MonitorInputData):
     """
     :ivar type: Specifies the type of monitoring input data. Set automatically to "Static" for this class.

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_monitoring/schedule.py

@@ -13,7 +13,6 @@
 from azure.ai.ml._restclient.v2023_06_01_preview.models import Schedule as RestSchedule
 from azure.ai.ml._restclient.v2023_06_01_preview.models import ScheduleProperties
 from azure.ai.ml._schema.monitoring.schedule import MonitorScheduleSchema
-from azure.ai.ml._utils._experimental import experimental
 from azure.ai.ml._utils.utils import dump_yaml_to_file
 from azure.ai.ml.constants._common import BASE_PATH_CONTEXT_KEY, PARAMS_OVERRIDE_KEY, ScheduleType
 from azure.ai.ml.entities._mixins import RestTranslatableMixin
@@ -26,7 +25,6 @@
 module_logger = logging.getLogger(__name__)
 
 
-@experimental
 class MonitorSchedule(Schedule, RestTranslatableMixin):
     """Monitor schedule.
 
@@ -98,25 +96,31 @@ def _to_rest_object(self) -> RestSchedule:
         # by default 7 days if user provides incorrect recurrence frequency
         # or a cron expression
         default_data_window_size = "P7D"
+        ref_data_window_size = "P14D"
         if isinstance(self.trigger, RecurrenceTrigger):
             frequency = self.trigger.frequency.lower()
             interval = self.trigger.interval
             if frequency == RecurrenceFrequency.MINUTE.lower() or frequency == RecurrenceFrequency.HOUR.lower():
                 default_data_window_size = "P1D"
+                ref_data_window_size = "P2D"
             elif frequency == RecurrenceFrequency.DAY.lower():
                 default_data_window_size = f"P{interval}D"
+                ref_data_window_size = f"P{interval * 2}D"
             elif frequency == RecurrenceFrequency.WEEK.lower():
                 default_data_window_size = f"P{interval * 7}D"
+                ref_data_window_size = f"P{(interval * 7) * 2}D"
             elif frequency == RecurrenceFrequency.MONTH.lower():
                 default_data_window_size = f"P{interval * 30}D"
+                ref_data_window_size = f"P{(interval * 30) * 2}D"
+
         return RestSchedule(
             properties=ScheduleProperties(
                 description=self.description,
                 properties=self.properties,
                 tags=tags,
                 action=CreateMonitorAction(
                     monitor_definition=self.create_monitor._to_rest_object(
-                        default_data_window_size=default_data_window_size
+                        default_data_window_size=default_data_window_size, ref_data_window_size=ref_data_window_size
                     )
                 ),
                 display_name=self.display_name,