fix flake8 errors due to new pypi dependency/rule updates (#597)

imatiach-msft · web-flow · commit d11712b8d535 · 2024-05-31T13:45:28.000-04:00
diff --git a/.github/workflows/CI-python.yml b/.github/workflows/CI-python.yml
@@ -53,12 +53,15 @@ jobs:
         pip install -r requirements-dev.txt
     - name: Install visualization dependencies
       shell: bash -l {0}
-      # install scikit-learn to workaround raiwidgets dependency
       run: |
         pip install raiwidgets
         pip install -r requirements-vis.txt
-        pip install --upgrade scikit-learn
         pip install --upgrade "shap<=0.44.0"
+    - if: ${{ matrix.pythonVersion == '3.9' }}
+      name: Install scikit-learn to work around raiwidgets dependency
+      shell: bash -l {0}
+      run: |
+        pip install "scikit-learn==1.4.2"
     - name: Install test dependencies
       shell: bash -l {0}
       run: |
diff --git a/devops/PR-Gate.yml b/devops/PR-Gate.yml
@@ -18,7 +18,7 @@ jobs:
   parameters:
     platforms:  { MacOS: macos-latest }
     testRunTypes: ['Notebooks']
-    pyVersions: [3.8]
+    pyVersions: [3.9]
     installationType: PipLocal
     envArtifactStem: $(EnvArtifactStem)
     envFileStem: $(EnvFileStem)
diff --git a/devops/nightly.yml b/devops/nightly.yml
@@ -23,7 +23,7 @@ jobs:
 - template: templates/all-tests-job-template.yml
   parameters:
     platforms:  { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest }
-    pyVersions: [3.7, 3.8]
+    pyVersions: [3.8, 3.9]
     installationType: PipLocal
     envArtifactStem: $(EnvArtifactStem)
     envFileStem: $(EnvFileStem)
diff --git a/devops/templates/create-env-step-template.yml b/devops/templates/create-env-step-template.yml
@@ -19,14 +19,14 @@ steps:
 
   - bash: |
       source activate ${{parameters.condaEnv}}
-      conda install --yes --quiet --name  ${{parameters.condaEnv}} numpy==1.19.5 -c conda-forge
+      conda install --yes --quiet --name  ${{parameters.condaEnv}} "numpy<1.24.0" -c conda-forge
       conda install --yes --quiet --name  ${{parameters.condaEnv}} pytorch torchvision cpuonly -c pytorch
     displayName: Install Anaconda packages
     condition:  ne(variables['Agent.OS'], 'Darwin')
 
   - bash: |
       source activate ${{parameters.condaEnv}}
-      conda install --yes --quiet --name  ${{parameters.condaEnv}} numpy==1.19.5 -c conda-forge
+      conda install --yes --quiet --name  ${{parameters.condaEnv}} "numpy<1.24.0" -c conda-forge
       conda install --yes --quiet --name  ${{parameters.condaEnv}} pytorch torchvision -c pytorch
     displayName: Install Anaconda packages on MacOS, which should not include cpuonly according to official docs
     condition:  eq(variables['Agent.OS'], 'Darwin')
diff --git a/devops/templates/test-run-step-template.yml b/devops/templates/test-run-step-template.yml
@@ -69,7 +69,7 @@ steps:
 - bash: |
     source activate  ${{parameters.condaEnv}}
     pip install responsibleai
-    pip install rai-core-flask==0.5.0
+    pip install rai-core-flask==0.7.6
     pip install raiwidgets --no-deps
     pip install --upgrade "shap<=0.44.0"
     pip install -r requirements-vis.txt
diff --git a/notebooks/advanced-feature-transformations-explain-local.ipynb b/notebooks/advanced-feature-transformations-explain-local.ipynb
@@ -197,8 +197,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import sklearn\n",
     "from sklearn.compose import ColumnTransformer\n",
-    "\n",
+    "from packaging import version\n",
+    "# for older scikit-learn versions use sparse, for newer sparse_output:\n",
+    "if version.parse(sklearn.__version__) < version.parse('1.2'):\n",
+    "    ohe_params = {\"sparse\": False}\n",
+    "else:\n",
+    "    ohe_params = {\"sparse_output\": False}\n",
     "transformations = ColumnTransformer([\n",
     "    (\"age_fare_1\", Pipeline(steps=[\n",
     "        ('imputer', SimpleImputer(strategy='median')),\n",
@@ -208,8 +214,8 @@
     "    (\"age_fare_3\", many_to_many_transformer, [\"age\", \"fare\"]),\n",
     "    (\"embarked\", Pipeline(steps=[\n",
     "        (\"imputer\", SimpleImputer(strategy='constant', fill_value='missing')), \n",
-    "        (\"encoder\", OneHotEncoder(sparse=False))]), [\"embarked\"]),\n",
-    "    (\"sex_pclass\", OneHotEncoder(sparse=False), [\"sex\", \"pclass\"])    \n",
+    "        (\"encoder\", OneHotEncoder(**ohe_params))]), [\"embarked\"]),\n",
+    "    (\"sex_pclass\", OneHotEncoder(**ohe_params), [\"sex\", \"pclass\"])    \n",
     "])\n"
    ]
   },
@@ -222,7 +228,14 @@
     "'''\n",
     "# Uncomment below if sklearn-pandas is not installed\n",
     "#!pip install sklearn-pandas\n",
+    "import sklearn\n",
     "from sklearn_pandas import DataFrameMapper\n",
+    "from packaging import version\n",
+    "# for older scikit-learn versions use sparse, for newer sparse_output:\n",
+    "if version.parse(sklearn.__version__) < version.parse('1.2'):\n",
+    "    ohe_params = {\"sparse\": False}\n",
+    "else:\n",
+    "    ohe_params = {\"sparse_output\": False}\n",
     "\n",
     "# Impute, standardize the numeric features and one-hot encode the categorical features.    \n",
     "\n",
@@ -235,8 +248,8 @@
     "    ([\"age\", \"fare\"], many_to_many_transformer),\n",
     "    ([\"embarked\"], Pipeline(steps=[\n",
     "        (\"imputer\", SimpleImputer(strategy='constant', fill_value='missing')), \n",
-    "        (\"encoder\", OneHotEncoder(sparse=False))])),\n",
-    "    ([\"sex\", \"pclass\"], OneHotEncoder(sparse=False))    \n",
+    "        (\"encoder\", OneHotEncoder(**ohe_params))])),\n",
+    "    ([\"sex\", \"pclass\"], OneHotEncoder(**ohe_params))    \n",
     "]\n",
     "\n",
     "\n",
diff --git a/notebooks/simple-feature-transformations-explain-local.ipynb b/notebooks/simple-feature-transformations-explain-local.ipynb
@@ -240,24 +240,27 @@
     "## Fitted Transformer tuples\n",
     "# Uncomment below if sklearn-pandas is not installed\n",
     "#!pip install sklearn-pandas\n",
+    "import sklearn\n",
     "from sklearn_pandas import DataFrameMapper\n",
+    "from packaging import version\n",
+    "# for older scikit-learn versions use sparse, for newer sparse_output:\n",
+    "if version.parse(sklearn.__version__) < version.parse('1.2'):\n",
+    "    ohe_params = {\"sparse\": False}\n",
+    "else:\n",
+    "    ohe_params = {\"sparse_output\": False}\n",
     "\n",
     "# Impute, standardize the numeric features and one-hot encode the categorical features.    \n",
     "\n",
-    "\n",
     "numeric_transformations = [([f], Pipeline(steps=[('imputer', SimpleImputer(strategy='median')), ('scaler', StandardScaler())])) for f in numerical]\n",
     "\n",
-    "categorical_transformations = [([f], OneHotEncoder(handle_unknown='ignore', sparse=False)) for f in categorical]\n",
+    "categorical_transformations = [([f], OneHotEncoder(handle_unknown='ignore', **ohe_params)) for f in categorical]\n",
     "\n",
     "transformations = numeric_transformations + categorical_transformations\n",
     "\n",
     "# Append classifier to preprocessing pipeline.\n",
     "# Now we have a full prediction pipeline.\n",
     "clf = Pipeline(steps=[('preprocessor', transformations),\n",
-    "                      ('classifier', SVC(C = 1.0, probability=True, gamma='auto'))]) \n",
-    "\n",
-    "\n",
-    "\n",
+    "                      ('classifier', SVC(C = 1.0, probability=True, gamma='auto'))])\n",
     "'''"
    ]
   },
diff --git a/python/docs/transformations.rst b/python/docs/transformations.rst
@@ -64,7 +64,7 @@ In case you want to run the example with the list of fitted transformer tuples,
           strategy='median')), ('scaler', StandardScaler())])) for f in numerical]
 
       categorical_transformations = [([f], OneHotEncoder(
-          handle_unknown='ignore', sparse=False)) for f in categorical]
+          handle_unknown='ignore', sparse_output=False)) for f in categorical]
 
       transformations = numeric_transformations + categorical_transformations
 
diff --git a/python/interpret_community/common/gpu_kmeans.py b/python/interpret_community/common/gpu_kmeans.py
@@ -26,7 +26,7 @@
     from cuml import KMeans
     from cuml.preprocessing import SimpleImputer
     rapids_installed = True
-except BaseException:
+except BaseException:  # noqa: B036
     rapids_installed = False
 from scipy.sparse import issparse
 
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -5,4 +5,4 @@ hdbscan
 lightgbm
 xgboost
 sklearn_pandas
-lime>=0.2.0.0
+lime>=0.2.0.0
diff --git a/tests/common_tabular_tests.py b/tests/common_tabular_tests.py
@@ -21,7 +21,7 @@
                           create_sklearn_random_forest_classifier,
                           create_sklearn_random_forest_regressor,
                           create_sklearn_svm_classifier,
-                          create_xgboost_classifier)
+                          create_xgboost_classifier, get_ohe_params)
 from constants import ModelType
 from datasets import retrieve_dataset
 from interpret_community.common.constants import (ExplainParams, InterpretData,
@@ -860,7 +860,8 @@ def transform(self, X):
                 return X.astype('U')
 
         custom_text = CustomTextTransformer()
-        encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
+        ohe_params = get_ohe_params(use_simple_features_combiner=True)
+        encoder = OneHotEncoder(handle_unknown='ignore', **ohe_params)
         ct1 = ColumnTransformer([('cu', custom_text, categorical_col_indices)], remainder='passthrough')
         ct2 = ColumnTransformer([('ord', encoder, slice(0, len(categorical_col_indices)))], remainder='passthrough')
         pipeline = Pipeline([('cu', ct1), ('ct', ct2), ('lgbm', lgbm_regressor)])
diff --git a/tests/common_utils.py b/tests/common_utils.py
@@ -5,7 +5,9 @@
 # Defines common utilities for explanations
 import numpy as np
 import pandas as pd
+import sklearn
 from lightgbm import LGBMClassifier, LGBMRegressor
+from packaging import version
 from sklearn import ensemble, linear_model, svm
 from sklearn.base import TransformerMixin
 from sklearn.datasets import (fetch_20newsgroups, fetch_california_housing,
@@ -62,6 +64,18 @@ def get_mimic_method(surrogate_model):
         raise Exception("Unsupported surrogate model")
 
 
+def get_ohe_params(sparse=False, use_simple_features_combiner=False):
+    # for older scikit-learn versions use sparse, for newer sparse_output:
+    if version.parse(sklearn.__version__) < version.parse('1.2'):
+        ohe_params = {"sparse": sparse}
+    else:
+        ohe_params = {"sparse_output": sparse}
+    version_above_1_3 = version.parse(sklearn.__version__) >= version.parse('1.3')
+    if version_above_1_3 and use_simple_features_combiner:
+        ohe_params["feature_name_combiner"] = _simple_ohe_callable
+    return ohe_params
+
+
 def create_binary_sparse_newsgroups_data():
     categories = ['alt.atheism', 'soc.religion.christian']
     newsgroups_train = fetch_20newsgroups(subset='train', categories=categories)
@@ -668,3 +682,7 @@ def _common_model_generator(feature_number, output_length=1):
     model.add(Dense(output_length, activation='relu', input_shape=(32,)))
     model.add(Dropout(0.5))
     return model
+
+
+def _simple_ohe_callable(input_feature, category):
+    return str(category)
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -29,7 +29,7 @@ def pytest_itemcollected(item):
         item.add_marker(pytest.mark.domain(["explain", "model"]))
 
 
-@pytest.fixture()
+@pytest.fixture
 def _clean_dir():
     old_cwd = os.getcwd()
     new_path = tempfile.mkdtemp()
diff --git a/tests/test_explain_model.py b/tests/test_explain_model.py
@@ -19,7 +19,7 @@
                           create_sklearn_random_forest_classifier,
                           create_sklearn_random_forest_regressor,
                           create_sklearn_svm_classifier, create_tf_model,
-                          create_xgboost_classifier,
+                          create_xgboost_classifier, get_ohe_params,
                           wrap_classifier_without_proba)
 from constants import DatasetConstants, owner_email_tools_and_ux
 from datasets import retrieve_dataset
@@ -638,6 +638,7 @@ def conv(X):
         many_to_many_transformer = FunctionTransformer(lambda x: np.hstack(
             (conv(np.prod(x, axis=1)).reshape(-1, 1), conv(np.prod(x, axis=1)**2).reshape(-1, 1))
         ))
+        ohe_params = get_ohe_params()
         transformations = ColumnTransformer([
             ("age_fare_1", Pipeline(steps=[
                 ('imputer', SimpleImputer(strategy='median')),
@@ -647,8 +648,8 @@ def conv(X):
             ("age_fare_3", many_to_many_transformer, ["age", "fare"]),
             ("embarked", Pipeline(steps=[
                 ("imputer", SimpleImputer(strategy='constant', fill_value='missing')),
-                ("encoder", OneHotEncoder(sparse=False))]), ["embarked"]),
-            ("sex_pclass", OneHotEncoder(sparse=False), ["sex", "pclass"])
+                ("encoder", OneHotEncoder(**ohe_params))]), ["embarked"]),
+            ("sex_pclass", OneHotEncoder(**ohe_params), ["sex", "pclass"])
         ])
         clf = Pipeline(steps=[('preprocessor', transformations),
                               ('classifier', LogisticRegression(solver='lbfgs'))])
diff --git a/tests/test_explanation_dashboard.py b/tests/test_explanation_dashboard.py
@@ -3,7 +3,7 @@
 import pytest
 from common_utils import (create_cancer_data, create_cancer_data_booleans,
                           create_lightgbm_classifier,
-                          create_sklearn_svm_classifier)
+                          create_sklearn_svm_classifier, get_ohe_params)
 from constants import owner_email_tools_and_ux
 from datasets import retrieve_dataset
 from interpret import show
@@ -35,10 +35,11 @@ def test_raw_timestamp_explanation(self, mimic_explainer):
         dt_cols = df_X.select_dtypes(include=[np.datetime64]).columns.tolist()
         numeric_cols = df_X.select_dtypes(include=[np.number]).columns.tolist()
         transforms_list = []
+        ohe_params = get_ohe_params()
         for str_col in str_cols:
             transforms_list.append((str_col, Pipeline(steps=[
                 ('imputer', SimpleImputer(strategy='most_frequent')),
-                ('ohe', OneHotEncoder(sparse=False))
+                ('ohe', OneHotEncoder(**ohe_params))
                 ]), [str_col]
             ))
         for numeric_col in numeric_cols:
diff --git a/tests/test_mimic_explainer.py b/tests/test_mimic_explainer.py
@@ -17,7 +17,7 @@
                           create_cancer_data, create_iris_data,
                           create_lightgbm_regressor,
                           create_pytorch_single_output_classifier,
-                          create_timeseries_data)
+                          create_timeseries_data, get_ohe_params)
 from constants import ModelType, owner_email_tools_and_ux
 from datasets import retrieve_dataset
 from interpret_community.common.constants import ModelTask, ShapValuesOutput
@@ -292,6 +292,7 @@ def _validate_model_serialization(self, model, x_train, x_test, mimic_explainer)
                                       de_global_explanation.global_importance_values)
         assert global_explanation.method == LIGHTGBM_METHOD
 
+    @pytest.mark.skip(reason="Requires ml-wrappers upgrade to latest scikit-learn")
     def test_explain_model_categorical(self, verify_mimic_regressor):
         for idx, verifier in enumerate(verify_mimic_regressor):
             verify_same_shape = idx == LGBM_MODEL_IDX
@@ -466,9 +467,10 @@ def test_explain_model_string_classes(self, mimic_explainer):
             ('num_imputer', SimpleImputer(strategy='median')),
             ('num_scaler', StandardScaler())
         ])
+        ohe_params = get_ohe_params()
         cat_pipe = Pipeline([
             ('cat_imputer', SimpleImputer(strategy='constant', fill_value='?')),
-            ('cat_encoder', OneHotEncoder(handle_unknown='ignore', sparse=False))
+            ('cat_encoder', OneHotEncoder(handle_unknown='ignore', **ohe_params))
         ])
         feat_pipe = ColumnTransformer([
             ('num_pipe', num_pipe, pipe_cfg['num_cols']),
diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py
@@ -38,7 +38,7 @@ def output_notebook_path(notebookname):
     return "./tests/{0}.output.ipynb".format(notebookname)
 
 
-@pytest.mark.notebooks()
+@pytest.mark.notebooks
 def test_explain_binary_classification_local():
     notebookname = 'explain-binary-classification-local'
     input_notebook = input_notebook_path(notebookname)
@@ -51,7 +51,7 @@ def test_explain_binary_classification_local():
     assert 'worst area' in nb.scraps.data_dict[SORTED_LOCAL_IMPORTANCE_NAMES]
 
 
-@pytest.mark.notebooks()
+@pytest.mark.notebooks
 def test_explain_regression_local():
     notebookname = 'explain-regression-local'
     input_notebook = input_notebook_path(notebookname)
@@ -64,7 +64,7 @@ def test_explain_regression_local():
     assert 'Latitude' in nb.scraps.data_dict[SORTED_LOCAL_IMPORTANCE_NAMES]
 
 
-@pytest.mark.notebooks()
+@pytest.mark.notebooks
 def test_explain_regression_mimic_explainer():
     notebookname = 'explain-regression-mimic-explainer'
     input_notebook = input_notebook_path(notebookname)
@@ -77,7 +77,7 @@ def test_explain_regression_mimic_explainer():
     assert 'Latitude' in nb.scraps.data_dict[SORTED_LOCAL_IMPORTANCE_NAMES]
 
 
-@pytest.mark.notebooks()
+@pytest.mark.notebooks
 def test_advanced_feature_transformations_explain_local():
     notebookname = 'advanced-feature-transformations-explain-local'
     input_notebook = input_notebook_path(notebookname)
@@ -90,7 +90,7 @@ def test_advanced_feature_transformations_explain_local():
     assert 'embarked' in nb.scraps.data_dict[SORTED_LOCAL_IMPORTANCE_NAMES][0]
 
 
-@pytest.mark.notebooks()
+@pytest.mark.notebooks
 def test_explain_multiclass_classification_local():
     notebookname = 'explain-multiclass-classification-local'
     input_notebook = input_notebook_path(notebookname)
@@ -103,7 +103,7 @@ def test_explain_multiclass_classification_local():
     assert 'petal width (cm)' in nb.scraps.data_dict[SORTED_LOCAL_IMPORTANCE_NAMES]
 
 
-@pytest.mark.notebooks()
+@pytest.mark.notebooks
 def test_simple_feature_transformations_explain_local():
     notebookname = 'simple-feature-transformations-explain-local'
     input_notebook = input_notebook_path(notebookname)
@@ -116,7 +116,7 @@ def test_simple_feature_transformations_explain_local():
     assert 'TotalWorkingYears' in nb.scraps.data_dict[SORTED_LOCAL_IMPORTANCE_NAMES][0]
 
 
-@pytest.mark.notebooks()
+@pytest.mark.notebooks
 def test_captum_integration_example():
     notebookname = 'captum-integration-example'
     input_notebook = input_notebook_path(notebookname)
