Update client validation error messages to align with Heuristic evaluation criteria (Azure#27718)

* Reformat and update colors, raise MlException instead of generic

Author: diondrapeck

sdk/ml/azure-ai-ml/azure/ai/ml/_artifacts/_blob_storage_helper.py

@@ -32,7 +32,7 @@
     upload_file,
 )
 from azure.ai.ml.constants._common import STORAGE_AUTH_MISMATCH_ERROR
-from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MlException, ValidationException
+from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MLException, ValidationException
 from azure.core.exceptions import ResourceNotFoundError
 from azure.storage.blob import BlobServiceClient, ContainerClient
 
@@ -229,7 +229,7 @@ def download(
             raise ex
         except Exception as e:
             msg = "Saving blob with prefix {} was unsuccessful. exception={}"
-            raise MlException(
+            raise MLException(
                 message=msg.format(starts_with, e),
                 no_personal_data_message=msg.format("[starts_with]", "[exception]"),
                 target=ErrorTarget.ARTIFACT,

sdk/ml/azure-ai-ml/azure/ai/ml/_artifacts/_fileshare_storage_helper.py

@@ -26,7 +26,7 @@
     get_directory_size,
     traverse_directory,
 )
-from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MlException
+from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MLException
 from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
 from azure.storage.fileshare import ShareDirectoryClient, ShareFileClient
 
@@ -346,7 +346,7 @@ def recursive_download(
             recursive_download(sub_client, destination=destination, max_concurrency=max_concurrency)
     except Exception:
         msg = f"Saving fileshare directory with prefix {starts_with} was unsuccessful."
-        raise MlException(
+        raise MLException(
             message=msg.format(starts_with),
             no_personal_data_message=msg.format("[prefix]"),
             target=ErrorTarget.ARTIFACT,

sdk/ml/azure-ai-ml/azure/ai/ml/_artifacts/_gen2_storage_helper.py

@@ -24,7 +24,7 @@
     upload_file,
 )
 from azure.ai.ml.constants._common import STORAGE_AUTH_MISMATCH_ERROR
-from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MlException, ValidationException
+from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MLException, ValidationException
 from azure.core.exceptions import ResourceExistsError
 from azure.storage.filedatalake import DataLakeServiceClient
 
@@ -179,7 +179,7 @@ def download(self, starts_with: str, destination: str = Path.home()) -> None:
             raise ex
         except Exception as e:
             msg = "Saving output with prefix {} was unsuccessful. exception={}"
-            raise MlException(
+            raise MLException(
                 message=msg.format(starts_with, e),
                 no_personal_data_message=msg.format("[starts_with]", "[exception]"),
                 target=ErrorTarget.ARTIFACT,

sdk/ml/azure-ai-ml/azure/ai/ml/_exception_helper.py

@@ -7,7 +7,7 @@
 import traceback
 from typing import Dict, Tuple, Union
 
-from colorama import Fore, init
+from colorama import Fore, init, Style
 from marshmallow.exceptions import ValidationError as SchemaValidationError
 
 from azure.ai.ml.constants._common import (
@@ -86,7 +86,7 @@ def format_details_section(
 
     if hasattr(error, "message"):
         error_types[error.error_type] = True
-        details += f"\n\n{error.message}\n"
+        details += f"\n\n{Fore.RED}(x) {error.message}{Fore.RESET}\n"
     else:
         if (
             entity_type == ErrorTarget.COMMAND_JOB
@@ -139,12 +139,15 @@ def format_details_section(
             if isinstance(field_error, dict):
                 field_error = f"{list(field_error.keys())[0]}:\n    - {list(field_error.values())[0][0]}"
 
-            details += f"\n{field}:\n- {field_error}\n"
-
+            details += f"{Fore.RED}\n(x) {field}:\n- {field_error}{Fore.RESET}\n"
     return error_types, details
 
 
-def format_errors_and_resolutions_sections(entity_type: str, error_types: Dict[str, bool]) -> Tuple[str, str]:
+def format_errors_and_resolutions_sections(
+    entity_type: str,
+    error_types: Dict[str, bool],
+    cli: bool
+) -> Tuple[str, str]:
     """Builds strings for details of the error message template's Errors and Resolutions sections."""
 
     resolutions = ""
@@ -153,40 +156,50 @@ def format_errors_and_resolutions_sections(entity_type: str, error_types: Dict[s
 
     if error_types[ValidationErrorType.INVALID_VALUE]:
         errors += f"\n{count}) One or more fields are invalid"
-        resolutions += f"Double-check that all specified parameters are of the correct types and formats \
-        prescribed by the {entity_type} schema."
+        resolutions += f"\n{count}) Double-check that all specified parameters are of the correct types and formats "\
+            f"prescribed by the {entity_type} schema."
         count += 1
     if error_types[ValidationErrorType.UNKNOWN_FIELD]:
         errors += f"\n{count}) A least one unrecognized parameter is specified"
-        resolutions += f"Remove any parameters not prescribed by the {entity_type} schema."
+        resolutions += f"\n{count}) Remove any parameters not prescribed by the {entity_type} schema."
         count += 1
     if error_types[ValidationErrorType.MISSING_FIELD]:
         errors += f"\n{count}) At least one required parameter is missing"
-        resolutions += f"Ensure all parameters required by the {entity_type} schema are specified."
+        resolutions += f"\n{count}) Ensure all parameters required by the {entity_type} schema are specified."
         count += 1
     if error_types[ValidationErrorType.FILE_OR_FOLDER_NOT_FOUND]:
         errors += f"\n{count}) One or more files or folders do not exist.\n"
-        resolutions += "Double-check the directory paths you provided and enter the correct paths."
+        resolutions += f"\n{count}) Double-check the directory path you provided and enter the correct path."
         count += 1
     if error_types[ValidationErrorType.CANNOT_SERIALIZE]:
         errors += f"\n{count}) One or more fields cannot be serialized.\n"
-        resolutions += f"Double-check that all specified parameters are of the correct types and formats \
+        resolutions += f"\n{count}) Double-check that all specified parameters are of the correct types and formats \
         prescribed by the {entity_type} schema."
         count += 1
     if error_types[ValidationErrorType.CANNOT_PARSE]:
         errors += f"\n{count}) YAML file cannot be parsed.\n"
-        resolutions += "Double-check your YAML file for syntax and formatting errors."
+        resolutions += f"\n{count}) Double-check your YAML file for syntax and formatting errors."
         count += 1
     if error_types[ValidationErrorType.RESOURCE_NOT_FOUND]:
         errors += f"\n{count}) Resource was not found.\n"
-        resolutions += "Double-check that the resource has been specified correctly and that you have access to it."
+        resolutions += f"\n{count}) Double-check that the resource has been specified correctly and "\
+            "that you have access to it."
         count += 1
 
+    if cli:
+        errors = "Error: " + errors
+    else:
+        errors = Fore.BLACK + errors + Fore.RESET
+
     return errors, resolutions
 
 
 def format_create_validation_error(
-    error: Union[SchemaValidationError, ValidationException], yaml_operation: bool
+    error: Union[SchemaValidationError,
+    ValidationException],
+    yaml_operation: bool,
+    cli: bool = False,
+    raw_error: str = None,
 ) -> str:
     """
     Formats a detailed error message for validation errors.
@@ -204,12 +217,15 @@ def format_create_validation_error(
     from azure.ai.ml._schema.assets.environment import EnvironmentSchema
     from azure.ai.ml._schema.assets.model import ModelSchema
 
+    if raw_error:
+        error = raw_error
     entity_type, details = get_entity_type(error)
     error_types, details = format_details_section(error, details, entity_type)
-    errors, resolutions = format_errors_and_resolutions_sections(entity_type, error_types)
+    errors, resolutions = format_errors_and_resolutions_sections(entity_type, error_types, cli)
 
     if yaml_operation:
         description = YAML_CREATION_ERROR_DESCRIPTION.format(entity_type=entity_type)
+        description = Style.BRIGHT + description + Style.RESET_ALL
 
         if entity_type == ErrorTarget.MODEL:
             schema_type = ModelSchema
@@ -242,9 +258,11 @@ def format_create_validation_error(
         error_msg=errors,
         parsed_error_details=details,
         resolutions=resolutions,
+        text_color=Fore.WHITE,
+        link_color=Fore.CYAN,
+        reset=Fore.RESET
     )
 
-    formatted_error = Fore.RED + formatted_error + Fore.RESET
     return formatted_error
 
 
@@ -265,10 +283,16 @@ def log_and_raise_error(error, debug=False, yaml_operation=False):
     elif isinstance(error, ValidationException):
         module_logger.debug(traceback.format_exc())
         try:
-            if error.error_type == ValidationErrorType.GENERIC:
+            error_type = error.error_type
+            if error_type == ValidationErrorType.GENERIC:
                 formatted_error = error
             else:
                 formatted_error = format_create_validation_error(error, yaml_operation=yaml_operation)
+            raise ValidationException(
+                message=formatted_error,
+                no_personal_data_message="",
+                error_type=error_type,
+            )
         except NotImplementedError:
             formatted_error = error
     else:

sdk/ml/azure-ai-ml/azure/ai/ml/_local_endpoints/utilities/commandline_utility.py

@@ -8,7 +8,7 @@
 import sys
 import time
 
-from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MlException
+from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MLException
 
 
 def _print_command_results(test_passed, time_taken, output):
@@ -67,7 +67,7 @@ def run_cli_command(
                 return json.loads(exclude_warnings(output))
             except Exception:
                 msg = "Expected JSON, instead got: \n{}\n"
-                raise MlException(
+                raise MLException(
                     message=msg.format(output),
                     no_personal_data_message=msg.format("[something else]"),
                     target=ErrorTarget.LOCAL_ENDPOINT,

sdk/ml/azure-ai-ml/azure/ai/ml/_telemetry/activity.py

@@ -25,7 +25,7 @@
 from azure.core.exceptions import HttpResponseError
 
 from azure.ai.ml._utils.utils import _is_user_error_from_exception_type, _is_user_error_from_status_code, _str_to_bool
-from azure.ai.ml.exceptions import ErrorCategory, MlException
+from azure.ai.ml.exceptions import ErrorCategory, MLException
 
 # Get environment variable IS_IN_CI_PIPELINE to decide whether it's in CI test
 IS_IN_CI_PIPELINE = _str_to_bool(os.environ.get("IS_IN_CI_PIPELINE", "False"))
@@ -122,7 +122,7 @@ def error_preprocess(activityLogger, exception):
         activityLogger.activity_info["errorCategory"] = error_category
         if exception.inner_exception:
             activityLogger.activity_info["innerException"] = type(exception.inner_exception).__name__
-    elif isinstance(exception, MlException):
+    elif isinstance(exception, MLException):
         # If exception is MLException, it will have error_category, message and target attributes and will log those
         # information in log_activity, no need more actions here.
         pass
@@ -192,7 +192,7 @@ def log_activity(
         # All the system and unknown errors except for NotImplementedError will be wrapped with a new exception.
         if IS_IN_CI_PIPELINE and not isinstance(e, NotImplementedError):
             if (
-                isinstance(exception, MlException)
+                isinstance(exception, MLException)
                 and exception.error_category in [ErrorCategory.SYSTEM_ERROR, ErrorCategory.UNKNOWN] # pylint: disable=no-member
             ) or (
                 "errorCategory" in activityLogger.activity_info
@@ -214,7 +214,7 @@ def log_activity(
             if exception:
                 message += ", Exception={}".format(type(exception).__name__)
                 activityLogger.activity_info["exception"] = type(exception).__name__
-                if isinstance(exception, MlException):
+                if isinstance(exception, MLException):
                     activityLogger.activity_info["errorMessage"] = exception.no_personal_data_message # pylint: disable=no-member
                     activityLogger.activity_info["errorTarget"] = exception.target # pylint: disable=no-member
                     activityLogger.activity_info["errorCategory"] = exception.error_category # pylint: disable=no-member

sdk/ml/azure-ai-ml/azure/ai/ml/_utils/_endpoint_utils.py

@@ -26,7 +26,7 @@
     ResourceNotFoundError,
     map_error,
 )
-from azure.ai.ml.exceptions import ErrorTarget, ValidationErrorType, ValidationException, MlException, ErrorCategory
+from azure.ai.ml.exceptions import ErrorTarget, ValidationErrorType, ValidationException, MLException, ErrorCategory
 from azure.core.polling import LROPoller
 from azure.core.rest import HttpResponse
 from azure.mgmt.core.exceptions import ARMErrorFormat
@@ -206,7 +206,7 @@ def validate_scoring_script(deployment):
     except Exception as err:
         if isinstance(err, ValidationException):
             raise err
-        raise MlException(
+        raise MLException(
             message= f"Failed to open scoring script {err.filename}.",
             no_personal_data_message= "Failed to open scoring script.",
         )

sdk/ml/azure-ai-ml/azure/ai/ml/constants/_common.py

@@ -104,7 +104,7 @@
 EXPERIMENTAL_LINK_MESSAGE = (
     "and may change at any time. Please see https://aka.ms/azuremlexperimental for more information."
 )
-REF_DOC_YAML_SCHEMA_ERROR_MSG_FORMAT = "Visit this link to refer to the {} schema if needed: {}."
+REF_DOC_YAML_SCHEMA_ERROR_MSG_FORMAT = "\nVisit this link to refer to the {} schema if needed: {}."
 STORAGE_AUTH_MISMATCH_ERROR = "AuthorizationPermissionMismatch"
 SWEEP_JOB_BEST_CHILD_RUN_ID_PROPERTY_NAME = "best_child_run_id"
 BATCH_JOB_CHILD_RUN_NAME = "batchscoring"
@@ -139,20 +139,21 @@
 ANONYMOUS_COMPONENT_NAME = "azureml_anonymous"
 GIT_PATH_PREFIX = "git+"
 SCHEMA_VALIDATION_ERROR_TEMPLATE = (
-    "\n\nError: {description}\n{error_msg}\n\n"
+    "\n{text_color}{description}\n{error_msg}{reset}\n\n"
     "Details: {parsed_error_details}\n"
-    "Resolutions:\n{resolutions}"
+    "Resolutions: {resolutions}"
     "If using the CLI, you can also check the full log in debug mode for more details by adding --debug "
     "to the end of your command\n"
-    "Additional Resources: The easiest way to author a yaml specification file is using IntelliSense and "
+    "\nAdditional Resources: The easiest way to author a yaml specification file is using IntelliSense and "
     "auto-completion Azure ML VS code extension provides: "
-    "https://code.visualstudio.com/docs/datascience/azure-machine-learning. "
-    "To set up VS Code, visit https://docs.microsoft.com/azure/machine-learning/how-to-setup-vs-code\n"
+    "{link_color}https://code.visualstudio.com/docs/datascience/azure-machine-learning.{reset} "
+    "To set up VS Code, visit {link_color}https://docs.microsoft.com/azure/machine-learning/how-to-setup-vs-"
+    "code{reset}\n"
 )
 
 YAML_CREATION_ERROR_DESCRIPTION = (
     "The yaml file you provided does not match the prescribed schema "
-    "for {entity_type} yaml files and/or has the following issues:"
+    "for {entity_type} yaml files and/or has the following issues:\n"
 )
 DATASTORE_SCHEMA_TYPES = [
     "AzureFileSchema",