Unified as_df and as_html methods between SHAP and LIME (#131)

RobGeada · web-flow · commit 03e04527152a · 2022-12-12T10:13:43.000Z
diff --git a/src/trustyai/explainers/lime.py b/src/trustyai/explainers/lime.py
@@ -6,9 +6,11 @@
 import bokeh.models
 import matplotlib.pyplot as plt
 import matplotlib as mpl
+import numpy as np
 from bokeh.models import ColumnDataSource, HoverTool
 from bokeh.plotting import figure
 import pandas as pd
+from matplotlib.colors import LinearSegmentedColormap
 
 from trustyai import _default_initializer  # pylint: disable=unused-import
 from trustyai.utils._visualisation import (
@@ -77,42 +79,62 @@ def as_dataframe(self) -> pd.DataFrame:
         Returns
         -------
         pandas.DataFrame
-            DataFrame containing the results of the LIME explanation. For each model output, the
-            table will contain the following columns:
+            Dictionary of DataFrames, keyed by output name, containing the results of the LIME
+            explanation. For each model output, the table will contain the following columns:
+
+            * ``Feature``: The name of the feature
+            * ``Value``: The value of the feature for this particular input.
+            * ``Saliency``: The importance of this feature to the output.
+            * ``Confidence``: The confidence of this explanation as returned by the explainer.
 
-            * ``${output_name}_features``: The names of each input feature.
-            * ``${output_name}_score``: The LIME saliency of this feature.
-            * ``${output_name}_value``: The original value of each feature.
-            * ``${output_name}_confidence``: The confidence of the reported saliency.
         """
         outputs = self.saliency_map().keys()
 
         data = {}
         for output in outputs:
-            pfis = self.saliency_map().get(output).getPerFeatureImportance()
-            data[f"{output}_features"] = [
-                f"{pfi.getFeature().getName()}" for pfi in pfis
-            ]
-            data[f"{output}_score"] = [pfi.getScore() for pfi in pfis]
-            data[f"{output}_value"] = [
-                pfi.getFeature().getValue().as_number() for pfi in pfis
-            ]
-            data[f"{output}_confidence"] = [pfi.getConfidence() for pfi in pfis]
-
-        return pd.DataFrame.from_dict(data)
+            output_rows = []
+            for pfi in self.saliency_map().get(output).getPerFeatureImportance():
+                output_rows.append(
+                    {
+                        "Feature": str(pfi.getFeature().getName().toString()),
+                        "Value": pfi.getFeature().getValue().getUnderlyingObject(),
+                        "Saliency": pfi.getScore(),
+                        "Confidence": pfi.getConfidence(),
+                    }
+                )
+            data[output] = pd.DataFrame(output_rows)
+        return data
 
     def as_html(self) -> pd.io.formats.style.Styler:
         """
-        Return the LIME result as a Pandas Styler object.
+        Return the LIME results as Pandas Styler objects.
 
         Returns
         -------
-        pandas.Styler
-            Styler containing the results of the LIME explanation, in the same
-            schema as in :func:`as_dataframe`. Currently, no default styles are applied
-            in this particular function, making it equivalent to :code:`self.as_dataframe().style`.
+        Dict[str, pandas.Styler]
+            Dictionary of stylers keyed by output name. Each styler containing the results of the
+            LIME explanation for that particular output, in the same
+            schema as in :func:`as_dataframe`. This will:
+
+            * Color each ``Saliency`` based on how their magnitude.
         """
-        return self.as_dataframe().style
+
+        htmls = {}
+        for k, df in self.as_dataframe().items():
+            htmls[k] = df.style.background_gradient(
+                LinearSegmentedColormap.from_list(
+                    name="rwg",
+                    colors=[
+                        ds["negative_primary_colour"],
+                        ds["neutral_primary_colour"],
+                        ds["positive_primary_colour"],
+                    ],
+                ),
+                subset="Saliency",
+                vmin=-1 * max(np.abs(df["Saliency"])),
+                vmax=max(np.abs(df["Saliency"])),
+            )
+        return htmls
 
     def _matplotlib_plot(self, output_name: str, block=True) -> None:
         """Plot the LIME saliencies."""
diff --git a/src/trustyai/explainers/shap.py b/src/trustyai/explainers/shap.py
@@ -101,44 +101,27 @@ def _saliency_to_dataframe(self, saliency, output_name):
             ],
             0,
         ).tolist()
-        feature_values = [
-            pfi.getFeature().getValue().asNumber()
-            for pfi in saliency.getPerFeatureImportance()[:-1]
-        ]
-        shap_values = [
-            pfi.getScore() for pfi in saliency.getPerFeatureImportance()[:-1]
-        ]
-        feature_names = [
-            str(pfi.getFeature().getName())
-            for pfi in saliency.getPerFeatureImportance()[:-1]
-        ]
-
-        columns = ["Mean Background Value", "Feature Value", "SHAP Value"]
-        visualizer_data_frame = pd.DataFrame(
-            [background_mean_feature_values, feature_values, shap_values],
-            index=columns,
-            columns=feature_names,
-        ).T
-        fnull = self.get_fnull()[output_name]
 
-        return (
-            pd.concat(
-                [
-                    pd.DataFrame(
-                        [["-", "-", fnull]], index=["Background"], columns=columns
-                    ),
-                    visualizer_data_frame,
-                    pd.DataFrame(
-                        [[fnull, sum(shap_values) + fnull, sum(shap_values) + fnull]],
-                        index=["Prediction"],
-                        columns=columns,
-                    ),
-                ]
-            ),
-            feature_names,
-            shap_values,
-            background_mean_feature_values,
-        )
+        data_rows = []
+        for i, pfi in enumerate(saliency.getPerFeatureImportance()[:-1]):
+            data_rows.append(
+                {
+                    "Feature": str(pfi.getFeature().getName().toString()),
+                    "Value": pfi.getFeature().getValue().getUnderlyingObject(),
+                    "Mean Background Value": background_mean_feature_values[i],
+                    "SHAP Value": pfi.getScore(),
+                    "Confidence": pfi.getConfidence(),
+                }
+            )
+
+        fnull = {
+            "Feature": "Background",
+            "Value": None,
+            "Mean Background Value": None,
+            "SHAP Value": self.get_fnull()[output_name],
+        }
+
+        return pd.DataFrame([fnull] + data_rows)
 
     def as_dataframe(self) -> Dict[str, pd.DataFrame]:
         """
@@ -148,16 +131,18 @@ def as_dataframe(self) -> Dict[str, pd.DataFrame]:
         -------
         Dict[str, pandas.DataFrame]
             Dictionary of DataFrames, keyed by output name, containing the results of the SHAP
-            explanation. For each model output, the table will contain the following columns,
-            indexed by feature name:
+            explanation. For each model output, the table will contain the following columns:
 
-            * ``Mean Background Value``: The mean value this feature took in the background
+            * ``Feature``: The name of the feature
             * ``Feature Value``: The value of the feature for this particular input.
+            * ``Mean Background Value``: The mean value this feature took in the background
             * ``SHAP Value``: The found SHAP value of this feature.
+            * ``Confidence``: The confidence of this explanation as returned by the explainer.
+
         """
         df_dict = {}
         for output_name, saliency in self.saliency_map().items():
-            df_dict[output_name] = self._saliency_to_dataframe(saliency, output_name)[0]
+            df_dict[output_name] = self._saliency_to_dataframe(saliency, output_name)
         return df_dict
 
     def as_html(self) -> Dict[str, pd.io.formats.style.Styler]:
@@ -179,23 +164,21 @@ def as_html(self) -> Dict[str, pd.io.formats.style.Styler]:
         def _color_feature_values(feature_values, background_vals):
             """Internal function for the dataframe visualization"""
             formats = []
-            for i, feature_value in enumerate(feature_values[1:-1]):
+            for i, feature_value in enumerate(feature_values[1:]):
                 if feature_value < background_vals[i]:
                     formats.append(f"background-color:{ds['negative_primary_colour']}")
                 elif feature_value > background_vals[i]:
                     formats.append(f"background-color:{ds['positive_primary_colour']}")
                 else:
                     formats.append(None)
-            return [None] + formats + [None]
+            return [None] + formats
 
         df_dict = {}
-        for i, (output_name, saliency) in enumerate(self.saliency_map().items()):
-            (
-                df,
-                feature_names,
-                shap_values,
-                background_mean_feature_values,
-            ) = self._saliency_to_dataframe(saliency, i)
+        for output_name, saliency in self.saliency_map().items():
+            df = self._saliency_to_dataframe(saliency, output_name)
+            shap_values = df["SHAP Value"].values[1:]
+            background_mean_feature_values = df["Mean Background Value"].values[1:]
+
             style = df.style.background_gradient(
                 LinearSegmentedColormap.from_list(
                     name="rwg",
@@ -205,15 +188,15 @@ def _color_feature_values(feature_values, background_vals):
                         ds["positive_primary_colour"],
                     ],
                 ),
-                subset=(slice(feature_names[0], feature_names[-1]), "SHAP Value"),
+                subset=(slice(1, None), "SHAP Value"),
                 vmin=-1 * max(np.abs(shap_values)),
                 vmax=max(np.abs(shap_values)),
             )
             style.set_caption(f"Explanation of {output_name}")
             df_dict[output_name] = style.apply(
                 _color_feature_values,
                 background_vals=background_mean_feature_values,
-                subset="Feature Value",
+                subset="Value",
                 axis=0,
             )
         return df_dict
diff --git a/tests/general/test_limeexplainer.py b/tests/general/test_limeexplainer.py
@@ -118,16 +118,24 @@ def test_lime_plots():
 
 def test_lime_v2():
     np.random.seed(0)
-    data = pd.DataFrame(np.random.rand(1, 5))
+    data = pd.DataFrame(np.random.rand(1, 5)).values
+
     model_weights = np.random.rand(5)
-    predict_function = lambda x: np.dot(x.values, model_weights)
+    predict_function = lambda x: np.stack([np.dot(x, model_weights), 2 * np.dot(x, model_weights)], -1)
+    model = Model(predict_function)
 
-    model = Model(predict_function, dataframe_input=True)
     explainer = LimeExplainer(samples=100, perturbations=2, seed=23, normalise_weights=False)
     explanation = explainer.explain(inputs=data, outputs=model(data), model=model)
-    for score in explanation.as_dataframe()["output-0_score"]:
+
+    for score in explanation.as_dataframe()["output-0"]['Saliency']:
         assert score != 0
 
+    for out_name, df in explanation.as_dataframe().items():
+        assert "Feature" in df
+        assert "output" in out_name
+        assert all([x in str(df) for x in "01234"])
+
+
 def test_impact_score():
     np.random.seed(0)
     data = pd.DataFrame(np.random.rand(1, 5))
@@ -143,3 +151,17 @@ def test_impact_score():
     impact = ExplainabilityMetrics.impactScore(model, pred, top_features_t)
     assert impact > 0
     return impact
+
+
+def test_lime_as_html():
+    np.random.seed(0)
+    data = np.random.rand(1, 5)
+
+    model_weights = np.random.rand(5)
+    predict_function = lambda x: np.stack([np.dot(x, model_weights), 2 * np.dot(x, model_weights)], -1)
+
+    model = Model(predict_function, disable_arrow=True)
+
+    explainer = LimeExplainer()
+    explainer.explain(inputs=data, outputs=model(data), model=model)
+    assert True
diff --git a/tests/general/test_shap.py b/tests/general/test_shap.py
@@ -96,3 +96,19 @@ def test_shap_as_df():
         assert "Mean Background Value" in df
         assert "output" in out_name
         assert all([x in str(df) for x in "01234"])
+
+
+def test_shap_as_html():
+    np.random.seed(0)
+    data = pd.DataFrame(np.random.rand(101, 5))
+    background = data.iloc[:100].values
+    to_explain = data.iloc[100:101].values
+
+    model_weights = np.random.rand(5)
+    predict_function = lambda x: np.stack([np.dot(x, model_weights), 2 * np.dot(x, model_weights)], -1)
+
+    model = Model(predict_function, disable_arrow=True)
+
+    shap_explainer = SHAPExplainer(background=background)
+    explanation = shap_explainer.explain(inputs=to_explain, outputs=model(to_explain), model=model)
+    assert True
