fix: ParamValidationError when scheduling a Clarify model monitor (#2596)

Co-authored-by: Ahsan Khan <[email protected]>
Co-authored-by: Shreya Pandit <[email protected]>

Author: 3 people

src/sagemaker/clarify.py

@@ -228,7 +228,7 @@ def __init__(
         probability_threshold=None,
         label_headers=None,
     ):
-        """Initializes a model output config to extract the predicted label.
+        """Initializes a model output config to extract the predicted label or predicted score(s).
 
         The following examples show different parameter configurations depending on the endpoint:
             * Regression Task: The model returns the score, e.g. 1.2. we don't need to specify
@@ -255,11 +255,11 @@ def __init__(
                     'label_headers=['cat','dog','fish']' and infer the predicted label to be 'fish.'
 
         Args:
-            label (str or int or list[int]): Optional index or JSONPath location in the model
-                output for the prediction. In case, this is a predicted label of the same type as
-                the label in the dataset no further arguments need to be specified.
-            probability (str or int or list[int]): Optional index or JSONPath location in the model
-                output for the predicted scores.
+            label (str or int): Index or JSONPath location in the model output for the prediction.
+                In case, this is a predicted label of the same type as the label in the dataset,
+                no further arguments need to be specified.
+            probability (str or int): Index or JSONPath location in the model output
+                for the predicted score(s).
             probability_threshold (float): An optional value for binary prediction tasks in which
                 the model returns a probability, to indicate the threshold to convert the
                 prediction to a boolean value. Default is 0.5.

src/sagemaker/model_monitor/clarify_model_monitoring.py

@@ -476,9 +476,15 @@ def suggest_baseline(
             features_attribute=data_config.features,
         )
         if model_predicted_label_config is not None:
-            latest_baselining_job_config.inference_attribute = model_predicted_label_config.label
+            latest_baselining_job_config.inference_attribute = (
+                model_predicted_label_config.label
+                if model_predicted_label_config.label is None
+                else str(model_predicted_label_config.label)
+            )
             latest_baselining_job_config.probability_attribute = (
                 model_predicted_label_config.probability
+                if model_predicted_label_config.probability is None
+                else str(model_predicted_label_config.probability)
             )
             latest_baselining_job_config.probability_threshold_attribute = (
                 model_predicted_label_config.probability_threshold
@@ -827,8 +833,9 @@ def suggest_baseline(
                 specific explainability method. Currently, only SHAP is supported.
             model_config (:class:`~sagemaker.clarify.ModelConfig`): Config of the model and its
                 endpoint to be created.
-            model_scores:  Index or JSONPath location in the model output for the predicted scores
-                to be explained. This is not required if the model output is a single score.
+            model_scores (int or str): Index or JSONPath location in the model output for the
+                predicted scores to be explained. This is not required if the model output is
+                a single score.
             wait (bool): Whether the call should wait until the job completes (default: False).
             logs (bool): Whether to show the logs produced by the job.
                 Only meaningful when wait is True (default: False).
@@ -865,7 +872,7 @@ def suggest_baseline(
                 headers=headers,
             ),
             features_attribute=data_config.features,
-            inference_attribute=model_scores,
+            inference_attribute=model_scores if model_scores is None else str(model_scores),
         )
         self.latest_baselining_job_name = baselining_job_name
         self.latest_baselining_job = ClarifyBaseliningJob(

tests/unit/sagemaker/monitor/test_clarify_model_monitor.py

@@ -93,8 +93,8 @@
 OUTPUT_S3_URI = "s3://bucket/output"
 CONSTRAINTS = Constraints("", "s3://bucket/analysis.json")
 FEATURES_ATTRIBUTE = "features"
-INFERENCE_ATTRIBUTE = "predicted_label"
-PROBABILITY_ATTRIBUTE = "probabilities"
+INFERENCE_ATTRIBUTE = 0
+PROBABILITY_ATTRIBUTE = 1
 PROBABILITY_THRESHOLD_ATTRIBUTE = 0.6
 APP_SPECIFICATION = {
     "ConfigUri": ANALYSIS_CONFIG_S3_URI,
@@ -131,8 +131,8 @@
         "StartTimeOffset": START_TIME_OFFSET,
         "EndTimeOffset": END_TIME_OFFSET,
         "FeaturesAttribute": FEATURES_ATTRIBUTE,
-        "InferenceAttribute": INFERENCE_ATTRIBUTE,
-        "ProbabilityAttribute": PROBABILITY_ATTRIBUTE,
+        "InferenceAttribute": str(INFERENCE_ATTRIBUTE),
+        "ProbabilityAttribute": str(PROBABILITY_ATTRIBUTE),
         "ProbabilityThresholdAttribute": PROBABILITY_THRESHOLD_ATTRIBUTE,
     },
     "GroundTruthS3Input": {"S3Uri": GROUND_TRUTH_S3_URI},
@@ -155,7 +155,7 @@
         "S3InputMode": S3_INPUT_MODE,
         "S3DataDistributionType": S3_DATA_DISTRIBUTION_TYPE,
         "FeaturesAttribute": FEATURES_ATTRIBUTE,
-        "InferenceAttribute": INFERENCE_ATTRIBUTE,
+        "InferenceAttribute": str(INFERENCE_ATTRIBUTE),
     }
 }
 EXPLAINABILITY_JOB_DEFINITION = {
@@ -665,6 +665,13 @@ def test_model_bias_monitor_suggest_baseline(
         sagemaker_session=sagemaker_session,
         analysis_config=None,  # will pick up config from baselining job
         baseline_job_name=BASELINING_JOB_NAME,
+        endpoint_input=EndpointInput(
+            endpoint_name=ENDPOINT_NAME,
+            destination=ENDPOINT_INPUT_LOCAL_PATH,
+            start_time_offset=START_TIME_OFFSET,
+            end_time_offset=END_TIME_OFFSET,
+            #  will pick up attributes from baselining job
+        ),
     )
 
     # update schedule
@@ -837,8 +844,8 @@ def _test_model_bias_monitor_create_schedule(
         start_time_offset=START_TIME_OFFSET,
         end_time_offset=END_TIME_OFFSET,
         features_attribute=FEATURES_ATTRIBUTE,
-        inference_attribute=INFERENCE_ATTRIBUTE,
-        probability_attribute=PROBABILITY_ATTRIBUTE,
+        inference_attribute=str(INFERENCE_ATTRIBUTE),
+        probability_attribute=str(PROBABILITY_ATTRIBUTE),
         probability_threshold_attribute=PROBABILITY_THRESHOLD_ATTRIBUTE,
     ),
 ):
@@ -1074,6 +1081,7 @@ def test_model_explainability_monitor_suggest_baseline(
         analysis_config=None,  # will pick up config from baselining job
         baseline_job_name=BASELINING_JOB_NAME,
         endpoint_input=ENDPOINT_NAME,
+        #  will pick up attributes from baselining job
     )
 
     # update schedule
@@ -1253,7 +1261,7 @@ def _test_model_explainability_monitor_create_schedule(
         endpoint_name=ENDPOINT_NAME,
         destination=ENDPOINT_INPUT_LOCAL_PATH,
         features_attribute=FEATURES_ATTRIBUTE,
-        inference_attribute=INFERENCE_ATTRIBUTE,
+        inference_attribute=str(INFERENCE_ATTRIBUTE),
     ),
 ):
     # create schedule