Merge pull request #64 from smurching/spark-2.2-patch

[#63] Remove usage of deprecated scikit-learn API in GridSearchCV

Author: thunterdb

python/README.md

@@ -65,7 +65,10 @@ More extensive documentation (generated with Sphinx) is available in the `python
 ## Changelog
 
 - 2015-12-10 First public release (0.1)
-- 2016-08-16 Minor release:
-   1. the official Spark target is Spark 0.2
+- 2016-08-16 Minor release (0.2.0):
+   1. the official Spark target is Spark 2.0
    2. support for keyed models
+- 2017-09-14 Minor release (0.2.2):
+   1. The official Spark target is Spark >= 2.1
+
 

python/requirements.txt

@@ -1,2 +1,2 @@
 # This file should list any python package dependencies.
-scikit-learn==0.18.1
+scikit-learn>=0.18.1, <=0.19

python/setup.py

@@ -19,7 +19,7 @@
     "Programming Language :: Python",
     "Topic :: Scientific/Engineering",
 ]
-INSTALL_REQUIRES = ["scikit-learn >= 0.18.1"]
+INSTALL_REQUIRES = ["scikit-learn >=0.18.1, <= 0.19"]
 
 # Project root
 ROOT = os.path.abspath(os.getcwd() + "/")

python/spark_sklearn/grid_search.py

@@ -2,9 +2,6 @@
 Class for parallelizing GridSearchCV jobs in scikit-learn
 """
 
-import sys
-
-from itertools import product
 from collections import defaultdict, Sized
 from functools import partial
 import warnings
@@ -115,21 +112,23 @@ class GridSearchCV(BaseSearchCV):
     Examples
     --------
     >>> from sklearn import svm, datasets
-    >>> from sklearn.model_selection import GridSearchCV
+    >>> from spark_sklearn.grid_search import GridSearchCV
+    >>> from spark_sklearn.util import createLocalSparkSession
+    >>> sc = createLocalSparkSession().sparkContext
     >>> iris = datasets.load_iris()
     >>> parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
     >>> svr = svm.SVC()
-    >>> clf = GridSearchCV(svr, parameters)
+    >>> clf = GridSearchCV(sc, svr, parameters)
     >>> clf.fit(iris.data, iris.target)
     ...                             # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
     GridSearchCV(cv=None, error_score=...,
            estimator=SVC(C=1.0, cache_size=..., class_weight=..., coef0=...,
-                         decision_function_shape=None, degree=..., gamma=...,
+                         decision_function_shape=..., degree=..., gamma=...,
                          kernel='rbf', max_iter=-1, probability=False,
                          random_state=None, shrinking=True, tol=...,
                          verbose=False),
            fit_params={}, iid=..., n_jobs=1,
-           param_grid=..., pre_dispatch=..., refit=..., return_train_score=...,
+           param_grid=..., pre_dispatch=..., refit=...,
            scoring=..., verbose=...)
     >>> sorted(clf.cv_results_.keys())
     ...                             # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
@@ -242,9 +241,12 @@ def __init__(self, sc, estimator, param_grid, scoring=None, fit_params=None,
                  n_jobs=1, iid=True, refit=True, cv=None, verbose=0,
                  pre_dispatch='2*n_jobs', error_score='raise', return_train_score=True):
         super(GridSearchCV, self).__init__(
-            estimator=estimator, scoring=scoring, fit_params=fit_params, n_jobs=n_jobs, iid=iid,
+            estimator=estimator, scoring=scoring, n_jobs=n_jobs, iid=iid,
             refit=refit, cv=cv, verbose=verbose, pre_dispatch=pre_dispatch, error_score=error_score,
             return_train_score=return_train_score)
+
+        self.fit_params = fit_params if fit_params is not None else {}
+
         self.sc = sc
         self.param_grid = param_grid
 

python/spark_sklearn/tests/test_grid_search_1.py

@@ -32,6 +32,10 @@ def do_test_expected(*kwargs):
     return do_test_expected
 
 def _add_to_module():
+    # NOTE: This doesn't actually run scikit-learn tests against SPGridSearchWrapper
+    # for scikit-learn >= 0.18, since the scikit-learn tests (in sklearn.model_selection.tests) use
+    # sklearn.model_selection.GridSearchCV (not sklearn.grid_search.GridSearchCV)
+    # TODO: Get scikit-learn tests to pass with spark-sklearn GridSearch implementation
     SKGridSearchCV = sklearn.grid_search.GridSearchCV
     sklearn.grid_search.GridSearchCV = SPGridSearchWrapper
     sklearn.grid_search.GridSearchCV_original = SKGridSearchCV