add responsibleai-vision package to responsible-ai-toolbox (#2135)

Author: imatiach-msft

responsibleai_vision/README.md

@@ -0,0 +1,16 @@
+# Responsible AI Vision SDK for Python
+
+### This package has been tested with Python 3.6, 3.7, 3.8 and 3.9
+
+The Responsible AI Vision SDK enables users to analyze their machine learning models for computer vision in one API. Users will be able to analyze errors, explain the most important features, and understand their data using a single API.
+
+Highlights of the package include:
+
+- `explainer.add()` explains the model
+
+### Supported scenarios, models and datasets
+
+The Responsible AI Vision SDK supports multiclass classification models on image data currently.
+
+The open source code for the visualization dashboard can be found here:
+https://github.com/microsoft/responsible-ai-widgets

responsibleai_vision/requirements-automl.txt

@@ -0,0 +1,2 @@
+opencv-python==4.3.0.36
+azureml-automl-dnn-vision>=1.47.0

responsibleai_vision/requirements-dev.txt

@@ -0,0 +1,30 @@
+# Requirements for development
+
+pytest==7.0.1
+pytest-cov
+pytest-mock==3.6.1
+requests==2.25.1
+
+requirements-parser==0.2.0
+
+wheel
+
+# Required for notebook tests
+nbformat
+papermill
+scrapbook
+jupyter
+nbval
+
+docutils<0.18
+sphinx==3.1.1
+sphinx-gallery==0.8.1
+pydata-sphinx-theme==0.3.0
+
+transformers
+datasets
+tensorflow<2.11.0
+opencv-python
+
+fastai
+mlflow

responsibleai_vision/requirements.txt

@@ -0,0 +1,8 @@
+numpy>=1.17.2
+pandas>=0.25.1,<2.0.0 # TODO: remove ceiling on version.
+scikit-learn>=0.22.1
+scipy>=1.4.1
+semver~=2.13.0
+responsibleai>=0.27.0
+torchmetrics
+vision_explanation_methods

responsibleai_vision/responsibleai_vision/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) Microsoft Corporation
+# Licensed under the MIT License.
+
+"""Responsible AI Vision SDK package."""
+
+from responsibleai_vision.common.constants import ModelTask
+from responsibleai_vision.rai_vision_insights import RAIVisionInsights
+
+from .version import name, version
+
+__name__ = name
+__version__ = version
+
+__all__ = ['ModelTask', 'RAIVisionInsights']

responsibleai_vision/responsibleai_vision/common/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation
+# Licensed under the MIT License.
+
+"""Common infrastructure, constants and utilities."""

responsibleai_vision/responsibleai_vision/common/constants.py

@@ -0,0 +1,101 @@
+# Copyright (c) Microsoft Corporation
+# Licensed under the MIT License.
+
+from enum import Enum
+
+
+class ModelTask(str, Enum):
+    """Provide model task constants.
+
+    Can be 'image_classification', 'object_detection' or 'unknown'.
+    """
+
+    IMAGE_CLASSIFICATION = 'image_classification'
+    MULTILABEL_IMAGE_CLASSIFICATION = 'multilabel_image_classification'
+    OBJECT_DETECTION = 'object_detection'
+    UNKNOWN = 'unknown'
+
+
+class ImageColumns(str, Enum):
+    """Provide constants related to the input image dataframe columns.
+
+    Can be 'image_url', 'image' or 'label'.
+    """
+
+    IMAGE_URL = 'image_url'
+    IMAGE = 'image'
+    LABEL = 'label'
+    IMAGE_DETAILS = 'image_details'
+
+
+class ExplainabilityLiterals:
+    """Parameters for explainability method names."""
+
+    MODEL_EXPLAINABILITY = 'model_explainability'
+    XAI_PARAMETERS = 'xai_parameters'
+    XAI_ALGORITHM = 'xai_algorithm'
+    SHAP_METHOD_NAME = 'shap'
+    XRAI_METHOD_NAME = 'xrai'
+    INTEGRATEDGRADIENTS_METHOD_NAME = 'integrated_gradients'
+    GUIDEDGRADCAM_METHOD_NAME = 'guided_gradcam'
+    GUIDEDBACKPROP_METHOD_NAME = 'guided_backprop'
+    CONFIDENCE_SCORE_THRESHOLD_MULTILABEL = (
+        'confidence_score_threshold_multilabel'
+    )
+    N_STEPS = "n_steps"
+    APPROXIMATION_METHOD = "approximation_method"
+    XRAI_FAST = "xrai_fast"
+    XAI_ARGS_GROUP = [
+        XAI_ALGORITHM,
+        N_STEPS,
+        APPROXIMATION_METHOD,
+        XRAI_FAST,
+        CONFIDENCE_SCORE_THRESHOLD_MULTILABEL,
+    ]
+    SHAP = 'shap'
+
+
+class ExplainabilityDefaults:
+    """DEFAULT values for explainability parameters."""
+
+    MODEL_EXPLAINABILITY = False
+    XAI_ALGORITHM = ExplainabilityLiterals.GUIDEDGRADCAM_METHOD_NAME
+    OUTPUT_VISUALIZATIONS = True
+    OUTPUT_ATTRIBUTIONS = False
+    CONFIDENCE_SCORE_THRESHOLD_MULTILABEL = 0.5
+    DEFAULT_MAX_EVALS = 100
+    DEFAULT_MASK_RES = 4
+    DEFAULT_NUM_MASKS = 50
+
+
+class XAIPredictionLiterals:
+    """Strings that will be keys in the output json during prediction."""
+
+    VISUALIZATIONS_KEY_NAME = 'visualizations'
+    ATTRIBUTIONS_KEY_NAME = 'attributions'
+
+
+class MLFlowSchemaLiterals:
+    """MLFlow model signature related schema"""
+
+    INPUT_IMAGE_KEY = 'image_base64'
+    INPUT_COLUMN_IMAGE = 'image'
+    INPUT_IMAGE_SIZE = 'image_size'
+
+
+class CommonTags:
+    """Common constants"""
+
+    IMAGE_DECODE_UTF_FORMAT = 'utf-8'
+
+
+class AutoMLImagesModelIdentifier:
+    """AutoML model object types"""
+
+    AUTOML_IMAGE_CLASSIFICATION_MODEL = (
+        "WrappedMlflowAutomlImagesClassificationModel'>"
+    )
+
+    AUTOML_OBJECT_DETECTION_MODEL = (
+        "WrappedMlflowAutomlObjectDetectionModel'>"
+    )

responsibleai_vision/responsibleai_vision/common/interfaces.py

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation
+# Licensed under the MIT License.
+
+from typing import List
+
+
+class VisionExplanationData:
+    classNames: List[str]
+    images: List[str]
+    predictedY: List[str]
+    trueY: List[str]

responsibleai_vision/responsibleai_vision/managers/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation
+# Licensed under the MIT License.
+
+"""Contains all of the managers."""