[evaluation] Refactor exceptions to use custom exception class (#37436)

* Remove Optional type annotation from get() methods

* Remove duplicate overloads

* Include xpia in handled_metrics for evaluation aggregation

* Rename class from PromptflowEvalsException to EvaluationException

* Update exceptions to use custom exception class

* Revert "Include xpia in handled_metrics for evaluation aggregation"

This reverts commit 8629de8.

* Revert irrelevant commits

* Update output.py

* Update tests

* Fix typo in error message

* Address bandit error

Author: diondrapeck

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py

@@ -15,6 +15,7 @@
 from azure.identity import DefaultAzureCredential
 
 from azure.ai.evaluation._http_utils import get_async_http_client
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation._model_configurations import AzureAIProject
 
 from .constants import (
@@ -72,15 +73,25 @@ async def ensure_service_availability(rai_svc_url: str, token: str, capability:
         )
 
     if response.status_code != 200:
-        raise Exception(  # pylint: disable=broad-exception-raised
-            f"RAI service is not available in this region. Status Code: {response.status_code}"
+        msg = f"RAI service is not available in this region. Status Code: {response.status_code}"
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=ErrorTarget.UNKNOWN,
+            category=ErrorCategory.SERVICE_UNAVAILABLE,
+            blame=ErrorBlame.USER_ERROR,
         )
 
     capabilities = response.json()
 
     if capability and capability not in capabilities:
-        raise Exception(  # pylint: disable=broad-exception-raised
-            f"Capability '{capability}' is not available in this region"
+        msg = f"Capability '{capability}' is not available in this region"
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=ErrorTarget.RAI_CLIENT,
+            category=ErrorCategory.SERVICE_UNAVAILABLE,
+            blame=ErrorBlame.USER_ERROR,
         )
 
 
@@ -337,7 +348,15 @@ async def _get_service_discovery_url(azure_ai_project: AzureAIProject, token: st
         )
 
     if response.status_code != 200:
-        raise Exception("Failed to retrieve the discovery service URL")  # pylint: disable=broad-exception-raised
+        msg = f"Failed to retrieve the discovery service URL."
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=ErrorTarget.RAI_CLIENT,
+            category=ErrorCategory.SERVICE_UNAVAILABLE,
+            blame=ErrorBlame.UNKNOWN,
+        )
+
     base_url = urlparse(response.json()["properties"]["discoveryUrl"])
     return f"{base_url.scheme}://{base_url.netloc}"
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run_client/code_client.py

@@ -10,6 +10,7 @@
 from promptflow.contracts.types import AttrDict
 from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _has_aggregator, get_int_env_var, load_jsonl
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 
 from ..._constants import PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT
 
@@ -119,8 +120,12 @@ def run(self, flow, data, evaluator_name=None, column_mapping=None, **kwargs):
             try:
                 json_data = load_jsonl(data)
             except json.JSONDecodeError as exc:
-                raise ValueError(
-                    f"Failed to parse data as JSON: {data}. Please provide a valid json lines data."
+                raise EvaluationException(
+                    message = f"Failed to parse data as JSON: {data}. Provide valid json lines data.",
+                    internal_message="Failed to parse data as JSON",
+                    target=ErrorTarget.CODE_CLIENT,
+                    category=ErrorCategory.INVALID_VALUE,
+                    blame=ErrorBlame.USER_ERROR,
                 ) from exc
 
             input_df = pd.DataFrame(json_data)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_eval_run.py

@@ -18,6 +18,7 @@
 from promptflow._sdk.entities import Run
 from azure.ai.evaluation._http_utils import get_http_client
 from azure.ai.evaluation._version import VERSION
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 
 LOGGER = logging.getLogger(__name__)
 
@@ -199,7 +200,7 @@ def _end_run(self, reason: str) -> None:
 
         :param reason: Reason for run termination. Possible values are "FINISHED" "FAILED", and "KILLED"
         :type reason: str
-        :raises ValueError: Raised if the run is not in ("FINISHED", "FAILED", "KILLED")
+        :raises EvaluationException: Raised if the run is not in ("FINISHED", "FAILED", "KILLED")
         """
         if not self._check_state_and_log(
             "stop run", {RunStatus.BROKEN, RunStatus.NOT_STARTED, RunStatus.TERMINATED}, False
@@ -210,8 +211,12 @@ def _end_run(self, reason: str) -> None:
             self._status = RunStatus.TERMINATED
             return
         if reason not in ("FINISHED", "FAILED", "KILLED"):
-            raise ValueError(
-                f"Incorrect terminal status {reason}. " 'Valid statuses are "FINISHED", "FAILED" and "KILLED".'
+            raise EvaluationException(
+                message=f"Incorrect terminal status {reason}. Valid statuses are 'FINISHED', 'FAILED' and 'KILLED'.",
+                internal_message="Incorrect terminal status. Valid statuses are 'FINISHED', 'FAILED' and 'KILLED'",
+                target=ErrorTarget.EVAL_RUN,
+                category=ErrorCategory.FAILED_EXECUTION,
+                blame=ErrorBlame.UNKNOWN
             )
         url = f"https://{self._url_base}/mlflow/v2.0" f"{self._get_scope()}/api/2.0/mlflow/runs/update"
         body = {
@@ -337,14 +342,20 @@ def _check_state_and_log(self, action: str, bad_states: Set[RunStatus], should_r
         :type bad_states: Set[RunStatus]
         :param should_raise: Should we raise an error if the bad state has been encountered
         :type should_raise: bool
-        :raises: RuntimeError if should_raise is True and invalid state was encountered.
+        :raises: ~azure.ai.evaluations._exceptions.EvaluationException if should_raise is True and invalid state was encountered.
         :return: Whether or not run is in the correct state.
         :rtype: bool
         """
         if self._status in bad_states:
             msg = f"Unable to {action} due to Run status={self._status}."
             if should_raise:
-                raise RuntimeError(msg)
+                raise EvaluationException(
+                    message=msg,
+                    internal_message=msg,
+                    target=ErrorTarget.EVAL_RUN,
+                    category=ErrorCategory.FAILED_EXECUTION,
+                    blame=ErrorBlame.UNKNOWN
+                )
             LOGGER.warning(msg)
             return False
         return True

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py

@@ -28,6 +28,7 @@
     _trace_destination_from_project_scope,
     _write_output,
 )
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 
 
 # pylint: disable=line-too-long
@@ -160,44 +161,111 @@ def _validate_input_data_for_evaluator(evaluator, evaluator_name, df_data, is_ta
     missing_inputs = [col for col in required_inputs if col not in df_data.columns]
     if missing_inputs:
         if not is_target_fn:
-            raise ValueError(f"Missing required inputs for evaluator {evaluator_name} : {missing_inputs}.")
-        raise ValueError(f"Missing required inputs for target : {missing_inputs}.")
+            msg = f"Missing required inputs for evaluator {evaluator_name} : {missing_inputs}."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        msg = f"Missing required inputs for target : {missing_inputs}."
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=ErrorTarget.EVALUATE,
+            category=ErrorCategory.MISSING_FIELD,
+            blame=ErrorBlame.USER_ERROR,
+        )
 
 
 def _validate_and_load_data(target, data, evaluators, output_path, azure_ai_project, evaluation_name):
     if data is None:
-        raise ValueError("data must be provided for evaluation.")
+        msg = "data parameter must be provided for evaluation."
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=ErrorTarget.EVALUATE,
+            category=ErrorCategory.MISSING_FIELD,
+            blame=ErrorBlame.USER_ERROR,
+        )
 
     if target is not None:
         if not callable(target):
-            raise ValueError("target must be a callable function.")
+            msg = "target parameter must be a callable function."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
 
     if data is not None:
         if not isinstance(data, str):
-            raise ValueError("data must be a string.")
+            msg = "data parameter must be a string."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
 
     if evaluators is not None:
         if not isinstance(evaluators, dict):
-            raise ValueError("evaluators must be a dictionary.")
+            msg = "evaluators parameter must be a dictionary."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
 
     if output_path is not None:
         if not isinstance(output_path, str):
-            raise ValueError("output_path must be a string.")
+            msg = "output_path parameter must be a string."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
 
     if azure_ai_project is not None:
         if not isinstance(azure_ai_project, Dict):
-            raise ValueError("azure_ai_project must be a Dict.")
+            msg = "azure_ai_project parameter must be a dictionary."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
 
     if evaluation_name is not None:
         if not isinstance(evaluation_name, str):
-            raise ValueError("evaluation_name must be a string.")
+            msg = "evaluation_name parameter must be a string."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
 
     try:
         initial_data_df = pd.read_json(data, lines=True)
     except Exception as e:
-        raise ValueError(
-            f"Failed to load data from {data}. Please validate it is a valid jsonl data. Error: {str(e)}."
-        ) from e
+        raise EvaluationException(
+                message=f"Failed to load data from {data}. Confirm that it is valid jsonl data. Error: {str(e)}.",
+                internal_message="Failed to load data. Confirm that it is valid jsonl data.",
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            ) from e
 
     return initial_data_df
 
@@ -219,11 +287,18 @@ def _validate_columns(
     :type target: Optional[Callable]
     :param evaluator_config: The configuration for evaluators.
     :type evaluator_config: Dict[str, Dict[str, str]]
-    :raises ValueError: If column starts from "__outputs." while target is defined.
+    :raises EvaluationException: If column starts from "__outputs." while target is defined.
     """
     if target:
         if any(c.startswith(Prefixes.TSG_OUTPUTS) for c in df.columns):
-            raise ValueError("The column cannot start from " f'"{Prefixes.TSG_OUTPUTS}" if target was defined.')
+            msg = "The column cannot start from " f'"{Prefixes.TSG_OUTPUTS}" if target was defined.'
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
         # If the target function is given, it may return
         # several columns and hence we cannot check the availability of columns
         # without knowing target function semantics.
@@ -319,9 +394,13 @@ def _process_evaluator_config(evaluator_config: Dict[str, Dict[str, str]]) -> Di
                 for map_to_key, map_value in mapping_config.items():
                     # Check if there's any unexpected reference other than ${target.} or ${data.}
                     if unexpected_references.search(map_value):
-                        raise ValueError(
-                            "Unexpected references detected in 'evaluator_config'. "
-                            "Ensure only ${target.} and ${data.} are used."
+                        msg = "Unexpected references detected in 'evaluator_config'. Ensure only ${target.} and ${data.} are used."
+                        raise EvaluationException(
+                            message=msg,
+                            internal_message=msg,
+                            target=ErrorTarget.EVALUATE,
+                            category=ErrorCategory.INVALID_VALUE,
+                            blame=ErrorBlame.USER_ERROR,
                         )
 
                     # Replace ${target.} with ${run.outputs.}
@@ -455,7 +534,13 @@ def evaluate(
                 "    if __name__ == '__main__':\n"
                 "        evaluate(...)"
             )
-            raise RuntimeError(error_message) from e
+            raise EvaluationException(
+                message=error_message,
+                internal_message=error_message,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.FAILED_EXECUTION,
+                blame=ErrorBlame.UNKNOWN,
+            ) from e
 
         raise e
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py

@@ -12,6 +12,7 @@
 import pandas as pd
 
 from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, Prefixes
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation._evaluate._eval_run import EvalRun
 
 LOGGER = logging.getLogger(__name__)
@@ -31,11 +32,17 @@ def is_none(value):
 def extract_workspace_triad_from_trace_provider(trace_provider: str):  # pylint: disable=name-too-long
     match = re.match(AZURE_WORKSPACE_REGEX_FORMAT, trace_provider)
     if not match or len(match.groups()) != 5:
-        raise ValueError(
-            "Malformed trace provider string, expected azureml://subscriptions/<subscription_id>/"
-            "resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/"
-            f"workspaces/<workspace_name>, got {trace_provider}"
-        )
+        raise EvaluationException(
+                message="Malformed trace provider string, expected azureml://subscriptions/<subscription_id>/"
+                "resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/"
+                f"workspaces/<workspace_name>, got {trace_provider}",
+                internal_message="Malformed trace provider string, expected azureml://subscriptions/<subscription_id>/"
+                "resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/"
+                "workspaces/<workspace_name>,",
+                target=ErrorTarget.UNKNOWN,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.UNKNOWN,
+            )
     subscription_id = match.group(1)
     resource_group_name = match.group(3)
     workspace_name = match.group(5)