RHOAIENG-24600 : Ensure Python 3.12 support for Python TrustyAI

.github/workflows/workflow.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.8, 3.9, 3.11 ]
+        python-version: [ '3.10', '3.11', '3.12' ]
         java-version: [ 17 ]
         maven-version: [ '3.8.6' ]
     steps:

pyproject.toml

@@ -5,7 +5,7 @@ description = "Python bindings to the TrustyAI explainability library."
 authors = [{ name = "Rui Vieira", email = "[email protected]" }]
 license = { text = "Apache License Version 2.0" }
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 
 keywords = ["trustyai", "xai", "explainability", "ml"]
 
@@ -15,19 +15,20 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Programming Language :: Java",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "Topic :: Software Development :: Libraries :: Java Libraries",
 ]
 
 dependencies = [
     "Jpype1==1.5.0",
-    "pyarrow==17.0.0",
-    "matplotlib~=3.6.3",
+    "pyarrow==20.0.0",
+    "matplotlib~=3.10.3",
     "pandas~=1.5.3",
-    "numpy~=1.24.1",
-    "jupyter-bokeh~=3.0.5",
+    "numpy==1.26.4",
+    "jupyter-bokeh~=3.0.7",
 ]
 
 [project.optional-dependencies]
@@ -38,15 +39,15 @@ dev = [
     "joblib~=1.2.0",
     "jupyterlab~=3.5.3",
     "numpydoc==1.5.0",
-    "pylint==2.15.6",
+    "pylint==3.2.0",
     "pytest~=7.2.1",
     "pytest-benchmark==4.0.0",
     "pytest-forked~=1.6.0",
-    "scikit-learn~=1.2.1",
+    "scikit-learn~=1.7.0",
     "setuptools",
     "twine==3.4.2",
     "wheel~=0.38.4",
-    "xgboost==1.4.2",
+    "xgboost~=3.0.2",
 ]
 extras = ["aix360[default,tsice,tslime,tssaliency]==0.3.0"]
 

src/trustyai/__init__.py

@@ -21,7 +21,8 @@
 def init():
     """Deprecated manual initializer for the JVM. This function has been replaced by
     automatic initialization when importing the components of the module that require
-    JVM access, or by manual user initialization via :func:`trustyai`initializer.init`."""
+    JVM access, or by manual user initialization via :func:`trustyai`initializer.init`.
+    """
     warnings.warn(
         "trustyai.init() is now deprecated; the trustyai library will now "
         + "automatically initialize. For manual initialization options, see "

src/trustyai/explainers/__init__.py

@@ -1,4 +1,5 @@
 """Explainers module"""
+
 # pylint: disable=duplicate-code
 from .counterfactuals import CounterfactualResult, CounterfactualExplainer
 from .lime import LimeExplainer, LimeResults

src/trustyai/explainers/counterfactuals.py

@@ -1,4 +1,5 @@
 """Explainers.countefactual module"""
+
 # pylint: disable = import-error, too-few-public-methods, wrong-import-order, line-too-long,
 # pylint: disable = unused-argument
 from typing import Optional, Union, List

src/trustyai/explainers/explanation_results.py

@@ -1,4 +1,5 @@
 """Generic class for Explanation and Saliency results"""
+
 from abc import ABC, abstractmethod
 
 import pandas as pd

src/trustyai/explainers/extras/tsice.py

@@ -2,6 +2,7 @@
 Wrapper module for TSICEExplainer from aix360.
 Original at https://github.com/Trusted-AI/AIX360/
 """
+
 # pylint: disable=too-many-arguments,import-error
 from typing import Callable, List, Optional, Union
 
@@ -46,7 +47,8 @@ def as_html(self) -> pd.io.formats.style.Styler:
 
     def plot_forecast(self, variable):  # pylint: disable=too-many-locals
         """Plots the explanation.
-        Based on https://github.com/Trusted-AI/AIX360/blob/master/examples/tsice/plots.py"""
+        Based on https://github.com/Trusted-AI/AIX360/blob/master/examples/tsice/plots.py
+        """
         forecast_horizon = self.explanation["current_forecast"].shape[0]
         original_ts = pd.DataFrame(
             data={variable: self.explanation["data_x"][variable]}
@@ -161,7 +163,8 @@ def _add_timeseries(
 
     def plot_impact(self, feature_per_row=2):
         """Plot the impace.
-        Based on https://github.com/Trusted-AI/AIX360/blob/master/examples/tsice/plots.py"""
+        Based on https://github.com/Trusted-AI/AIX360/blob/master/examples/tsice/plots.py
+        """
 
         n_row = int(np.ceil(len(self.explanation["feature_names"]) / feature_per_row))
         feat_values = np.array(self.explanation["feature_values"])

src/trustyai/explainers/extras/tslime.py

@@ -35,7 +35,8 @@ def as_html(self) -> Styler:
 
     def plot(self):
         """Plot TSLime explanation for the time-series instance. Based on
-        https://github.com/Trusted-AI/AIX360/blob/master/examples/tslime/tslime_univariate_demo.ipynb"""
+        https://github.com/Trusted-AI/AIX360/blob/master/examples/tslime/tslime_univariate_demo.ipynb
+        """
         relevant_history = self.explanation["history_weights"].shape[0]
         input_data = self.explanation["input_data"]
         relevant_df = input_data[-relevant_history:]

src/trustyai/explainers/lime.py

@@ -1,4 +1,5 @@
 """Explainers.lime module"""
+
 # pylint: disable = import-error, too-few-public-methods, wrong-import-order, line-too-long,
 # pylint: disable = unused-argument, duplicate-code, consider-using-f-string, invalid-name
 from typing import Dict, Union

src/trustyai/explainers/pdp.py

@@ -1,4 +1,5 @@
 """Explainers.pdp module"""
+
 import math
 import pandas as pd
 from pandas.io.formats.style import Styler

src/trustyai/explainers/shap.py

@@ -1,4 +1,5 @@
 """Explainers.shap module"""
+
 # pylint: disable = import-error, too-few-public-methods, wrong-import-order, line-too-long,
 # pylint: disable = unused-argument, consider-using-f-string, invalid-name
 from typing import Dict, Optional, Union

src/trustyai/language/detoxify/__init__.py

@@ -1,2 +1,3 @@
 """Language detoxification module."""
+
 from trustyai.language.detoxify.tmarco import TMaRCo

src/trustyai/language/detoxify/tmarco.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# pylint: disable = invalid-name, line-too-long, use-dict-literal, consider-using-f-string, too-many-nested-blocks, self-assigning-variable
+# pylint: disable = invalid-name, line-too-long, use-dict-literal, consider-using-f-string, too-many-nested-blocks, self-assigning-variable, broad-exception-raised, possibly-used-before-assignment
 """TMaRCo detoxification."""
 import os
 import math

src/trustyai/metrics/distance.py

@@ -1,4 +1,5 @@
-""""Distance metrics"""
+""" "Distance metrics"""
+
 # pylint: disable = import-error
 from dataclasses import dataclass
 from typing import List, Optional, Union, Callable

src/trustyai/metrics/fairness/group.py

@@ -1,4 +1,5 @@
 """Group fairness metrics"""
+
 # pylint: disable = import-error
 from typing import List, Optional, Any, Union
 

src/trustyai/metrics/language.py

@@ -1,4 +1,5 @@
-""""Language metrics"""
+""" "Language metrics"""
+
 # pylint: disable = import-error
 from dataclasses import dataclass
 

src/trustyai/model/__init__.py

@@ -242,7 +242,8 @@ def __init__(self, predict_fun):
 
     def predict(self, inbound_bytearray):
         """The function called internally by :func:`predictAsync` when communicating
-        between Java and Python. This function should never need to be called manually."""
+        between Java and Python. This function should never need to be called manually.
+        """
         with pa.ipc.open_file(inbound_bytearray) as reader:
             batch = reader.get_batch(0)
         arr = batch.to_pandas()

src/trustyai/utils/_visualisation.py

@@ -1,4 +1,5 @@
 """Visualiser utilies for explainer results"""
+
 # pylint: disable = consider-using-f-string
 
 

src/trustyai/utils/data_conversions.py

@@ -381,7 +381,7 @@ def numpy_to_prediction_object(
 
 
 def prediction_object_to_numpy(
-    objects: Union[List[PredictionInput], List[PredictionOutput]]
+    objects: Union[List[PredictionInput], List[PredictionOutput]],
 ) -> np.array:
     """
     Convert a list of TrustyAI
@@ -410,7 +410,7 @@ def prediction_object_to_numpy(
 
 
 def prediction_object_to_pandas(
-    objects: Union[List[PredictionInput], List[PredictionOutput]]
+    objects: Union[List[PredictionInput], List[PredictionOutput]],
 ) -> pd.DataFrame:
     """
     Convert a list of TrustyAI

src/trustyai/utils/extras/models.py

@@ -1,2 +1,3 @@
 """AIX360 model wrappers"""
+
 from aix360.algorithms.tsutils.model_wrappers import *  # pylint: disable=wildcard-import,unused-wildcard-import

src/trustyai/utils/extras/timeseries.py

@@ -1,3 +1,4 @@
 """Extra time series utilities."""
+
 from aix360.algorithms.tsutils.tsframe import tsFrame  # pylint: disable=unused-import
 from aix360.algorithms.tsutils.tsperturbers import *  # pylint: disable=wildcard-import,unused-wildcard-import

src/trustyai/utils/text.py

@@ -1,4 +1,5 @@
 """Utility methods for text data handling"""
+
 from typing import List, Callable
 
 from jpype import _jclass

src/trustyai/utils/tokenizers.py

@@ -1,4 +1,5 @@
-""""Default tokenizers for TrustyAI."""
+""" "Default tokenizers for TrustyAI."""
+
 # pylint: disable = import-error
 
 from org.apache.commons.text import StringTokenizer as _StringTokenizer

src/trustyai/utils/tyrus.py

@@ -1,4 +1,5 @@
 """Tyrus module"""
+
 # pylint: disable = too-few-public-methods, wrong-import-order, protected-access, cell-var-from-loop
 # pylint: disable = too-many-instance-attributes, import-error. too-many-locals
 # pylint: disable = consider-using-f-string
@@ -98,7 +99,7 @@ def __init__(
         inputs: OneInputUnionType,
         outputs: OneOutputUnionType,
         background: ManyInputsUnionType,
-        **kwargs
+        **kwargs,
     ):
         r"""Initialize the :class:`Tyrus` TrustyAI assistant and dashboard.
 

src/trustyai/version.py

@@ -1,2 +1,3 @@
 """TrustyAI version"""
+
 __version__ = "0.6.1"

src/trustyai/visualizations/__init__.py

@@ -1,4 +1,5 @@
 """Generates visualization according to explanation type"""
+
 # pylint: disable=import-error, wrong-import-order, protected-access, missing-final-newline
 from typing import Union, Optional
 

src/trustyai/visualizations/distance.py

@@ -1,4 +1,5 @@
 """Visualizations.distance module"""
+
 # pylint: disable = import-error, too-few-public-methods, line-too-long, missing-final-newline
 import numpy as np
 import matplotlib.pyplot as plt
@@ -9,7 +10,7 @@ class DistanceViz:
 
     def plot(self, explanations):
         """Plot the Levenshtein distance matrix"""
-        cmap = plt.cm.viridis
+        cmap = plt.cm.viridis  # pylint: disable=no-member
 
         _, axes = plt.subplots()
         cax = axes.imshow(explanations.matrix, cmap=cmap, interpolation="nearest")

src/trustyai/visualizations/lime.py

@@ -1,4 +1,5 @@
 """Visualizations.lime module"""
+
 # pylint: disable = import-error, too-few-public-methods, consider-using-f-string, missing-final-newline
 import matplotlib.pyplot as plt
 import matplotlib as mpl
@@ -34,9 +35,11 @@ def _matplotlib_plot(
                 ] = feature_importance.getScore()
 
             colours = [
-                ds["negative_primary_colour"]
-                if i < 0
-                else ds["positive_primary_colour"]
+                (
+                    ds["negative_primary_colour"]
+                    if i < 0
+                    else ds["positive_primary_colour"]
+                )
                 for i in dictionary.values()
             ]
             plt.title(f"LIME: Feature Importances to {output_name}")
@@ -65,18 +68,22 @@ def _get_bokeh_plot(self, explanations, output_name):
             ]
         )
         lime_data_source["color"] = lime_data_source["saliency"].apply(
-            lambda x: ds["positive_primary_colour"]
-            if x >= 0
-            else ds["negative_primary_colour"]
+            lambda x: (
+                ds["positive_primary_colour"]
+                if x >= 0
+                else ds["negative_primary_colour"]
+            )
         )
         lime_data_source["saliency_colored"] = lime_data_source["saliency"].apply(
             lambda x: (bold_green_html if x >= 0 else bold_red_html)("{:.2f}".format(x))
         )
 
         lime_data_source["color_faded"] = lime_data_source["saliency"].apply(
-            lambda x: ds["positive_primary_colour_faded"]
-            if x >= 0
-            else ds["negative_primary_colour_faded"]
+            lambda x: (
+                ds["positive_primary_colour_faded"]
+                if x >= 0
+                else ds["negative_primary_colour_faded"]
+            )
         )
         source = ColumnDataSource(lime_data_source)
         htool = HoverTool(

src/trustyai/visualizations/pdp.py

@@ -1,4 +1,5 @@
 """Visualizations.pdp module"""
+
 # pylint: disable = import-error, wrong-import-order, too-few-public-methods, missing-final-newline
 # pylint: disable = protected-access
 import matplotlib.pyplot as plt

src/trustyai/visualizations/shap.py

@@ -1,4 +1,5 @@
 """Visualizations.shap module"""
+
 # pylint: disable = import-error, consider-using-f-string, too-few-public-methods, missing-final-newline
 import matplotlib.pyplot as plt
 import matplotlib as mpl
@@ -100,14 +101,18 @@ def _get_bokeh_plot(self, explanations, output_name):
         prediction = fnull + data_source["saliency"].sum()
 
         data_source["color"] = data_source["saliency"].apply(
-            lambda x: ds["positive_primary_colour"]
-            if x >= 0
-            else ds["negative_primary_colour"]
+            lambda x: (
+                ds["positive_primary_colour"]
+                if x >= 0
+                else ds["negative_primary_colour"]
+            )
         )
         data_source["color_faded"] = data_source["saliency"].apply(
-            lambda x: ds["positive_primary_colour_faded"]
-            if x >= 0
-            else ds["negative_primary_colour_faded"]
+            lambda x: (
+                ds["positive_primary_colour_faded"]
+                if x >= 0
+                else ds["negative_primary_colour_faded"]
+            )
         )
         data_source["index"] = data_source.index
         data_source["saliency_text"] = data_source["saliency"].apply(

src/trustyai/visualizations/visualization_results.py

@@ -1,4 +1,5 @@
 """Generic class for Visualization results"""
+
 # pylint: disable = import-error, too-few-public-methods, line-too-long, missing-final-newline
 from abc import ABC, abstractmethod
 from typing import Dict