diff --git a/.ci/scripts/select_sklearn_tests.py b/.ci/scripts/select_sklearn_tests.py index 868e74cee3..63b491f27d 100644 --- a/.ci/scripts/select_sklearn_tests.py +++ b/.ci/scripts/select_sklearn_tests.py @@ -37,20 +37,20 @@ def parse_tests_tree(entry, prefix=""): # Reduced sklearn tests suite covering all patched functions in the shortest running time tests_map = { - "cluster/tests": ["test_dbscan.py", "test_k_means.py"], - "covariance/tests": "test_covariance.py", - "decomposition/tests": ["test_pca.py", "test_incremental_pca.py"], - "ensemble/tests": "test_forest.py", - "linear_model/tests": [ - "test_base.py", - "test_coordinate_descent.py", - "test_logistic.py", - "test_ridge.py", - ], - "manifold/tests": "test_t_sne.py", - "metrics/tests": ["test_pairwise.py", "test_ranking.py"], - "model_selection/tests": ["test_split.py", "test_validation.py"], - "neighbors/tests": ["test_lof.py", "test_neighbors.py", "test_neighbors_pipeline.py"], + # "cluster/tests": ["test_dbscan.py", "test_k_means.py"], + # "covariance/tests": "test_covariance.py", + # "decomposition/tests": ["test_pca.py", "test_incremental_pca.py"], + # "ensemble/tests": "test_forest.py", + # "linear_model/tests": [ + # "test_base.py", + # "test_coordinate_descent.py", + # "test_logistic.py", + # "test_ridge.py", + # ], + # "manifold/tests": "test_t_sne.py", + # "metrics/tests": ["test_pairwise.py", "test_ranking.py"], + # "model_selection/tests": ["test_split.py", "test_validation.py"], + # "neighbors/tests": ["test_lof.py", "test_neighbors.py", "test_neighbors_pipeline.py"], "svm/tests": ["test_sparse.py", "test_svm.py"], "tests": "test_dummy.py", } diff --git a/onedal/svm/__init__.py b/onedal/svm/__init__.py index 6bcf140a4a..edff432c69 100644 --- a/onedal/svm/__init__.py +++ b/onedal/svm/__init__.py @@ -14,6 +14,6 @@ # limitations under the License. # ============================================================================== -from .svm import SVC, SVR, NuSVC, NuSVR, SVMtype +from .svm import SVC, SVR, NuSVC, NuSVR -__all__ = ["SVC", "SVR", "NuSVC", "NuSVR", "SVMtype"] +__all__ = ["SVC", "SVR", "NuSVC", "NuSVR"] diff --git a/onedal/svm/svm.py b/onedal/svm/svm.py index 5fd38a2252..8d284b66e7 100644 --- a/onedal/svm/svm.py +++ b/onedal/svm/svm.py @@ -15,57 +15,36 @@ # ============================================================================== from abc import ABCMeta, abstractmethod -from enum import Enum import numpy as np from scipy import sparse as sp from onedal._device_offload import supports_queue from onedal.common._backend import bind_default_backend -from onedal.utils import _sycl_queue_manager as QM from ..common._estimator_checks import _check_is_fitted -from ..common._mixin import ClassifierMixin, RegressorMixin from ..datatypes import from_table, to_table -from ..utils.validation import ( - _check_array, - _check_n_features, - _check_X_y, - _column_or_1d, - _validate_targets, -) - - -class SVMtype(Enum): - c_svc = 0 - epsilon_svr = 1 - nu_svc = 2 - nu_svr = 3 +from ..utils.validation import _is_csr class BaseSVM(metaclass=ABCMeta): - @abstractmethod + def __init__( self, - C, + C, # Depending on the child class, C, nu, and/or epsilon are not used nu, epsilon, kernel="rbf", *, - degree, - gamma, - coef0, - tol, - shrinking, - cache_size, - max_iter, - tau, - class_weight, - decision_function_shape, - break_ties, - algorithm, - svm_type=None, - **kwargs, + degree=3, + gamma=None, + coef0=0.0, + tol=1e-3, + shrinking=True, + cache_size=200.0, + max_iter=-1, + tau=1e-12, + algorithm="thunder", ): self.C = C self.nu = nu @@ -79,11 +58,8 @@ def __init__( self.cache_size = cache_size self.max_iter = max_iter self.tau = tau - self.class_weight = class_weight - self.decision_function_shape = decision_function_shape - self.break_ties = break_ties self.algorithm = algorithm - self.svm_type = svm_type + self._onedal_model = None @abstractmethod def train(self, *args, **kwargs): ... @@ -91,116 +67,63 @@ def train(self, *args, **kwargs): ... @abstractmethod def infer(self, *args, **kwargs): ... - def _validate_targets(self, y, dtype): - self.class_weight_ = None - self.classes_ = None - return _column_or_1d(y, warn=True).astype(dtype, copy=False) - - def _get_onedal_params(self, data): + def _get_onedal_params(self, X): max_iter = 10000 if self.max_iter == -1 else self.max_iter # TODO: remove this workaround # when oneDAL SVM starts support of 'n_iterations' result - self.n_iter_ = 1 if max_iter < 1 else max_iter - class_count = 0 if self.classes_ is None else len(self.classes_) + self.n_iter_ = max(1, max_iter) + # if gamma is not given as a value, use sklearn's "auto" + gamma = 1 / X.shape[1] if self.gamma is None else self.gamma return { - "fptype": data.dtype, - "method": self.algorithm, - "kernel": self.kernel, + "fptype": X.dtype, "c": self.C, "nu": self.nu, "epsilon": self.epsilon, - "class_count": class_count, - "accuracy_threshold": self.tol, - "max_iteration_count": int(max_iter), - "scale": self._scale_, - "sigma": self._sigma_, - "shift": self.coef0, + "kernel": self.kernel, "degree": self.degree, - "tau": self.tau, + "shift": self.coef0 if self.kernel != "linear" else 0.0, + "scale": gamma if self.kernel != "linear" else 1.0, + "sigma": np.sqrt(0.5 / gamma) if self.kernel != "linear" else 1.0, + "accuracy_threshold": self.tol, "shrinking": self.shrinking, "cache_size": self.cache_size, + "max_iteration_count": int(max_iter), + "tau": self.tau, + "method": self.algorithm, + "class_count": self.class_count_, } - def _fit(self, X, y, sample_weight): - if hasattr(self, "decision_function_shape"): - if self.decision_function_shape not in ("ovr", "ovo", None): - raise ValueError( - f"decision_function_shape must be either 'ovr' or 'ovo', " - f"got {self.decision_function_shape}." - ) - - X, y = _check_X_y( - X, - y, - dtype=[np.float64, np.float32], - force_all_finite=True, - accept_sparse="csr", - ) - y = self._validate_targets(y, X.dtype) - if sample_weight is not None and len(sample_weight) > 0: - sample_weight = _check_array( - sample_weight, - accept_sparse=False, - ensure_2d=False, - dtype=X.dtype, - order="C", - ) - elif self.class_weight is not None: - sample_weight = np.ones(X.shape[0], dtype=X.dtype) - - if sample_weight is not None: - if self.class_weight_ is not None: - for i, v in enumerate(self.class_weight_): - sample_weight[y == i] *= v - data = (X, y, sample_weight) - else: - data = (X, y) - self._sparse = sp.issparse(X) + @supports_queue + def fit(self, X, y, sample_weight=None, class_count=0, queue=None): + # oneDAL expects that the user has a priori knowledge of the y data + # and has placed them in a oneDAL-acceptable format, most important + # for this is the number of classes. + self.class_count_ = class_count - if self.kernel == "linear": - self._scale_, self._sigma_ = 1.0, 1.0 - self.coef0 = 0.0 - else: - if isinstance(self.gamma, str): - if self.gamma == "scale": - if sp.issparse(X): - # var = E[X^2] - E[X]^2 - X_sc = (X.multiply(X)).mean() - (X.mean()) ** 2 - else: - X_sc = X.var() - _gamma = 1.0 / (X.shape[1] * X_sc) if X_sc != 0 else 1.0 - elif self.gamma == "auto": - _gamma = 1.0 / X.shape[1] - else: - raise ValueError( - f"When 'gamma' is a string, it should be either 'scale' or 'auto'. Got '{self.gamma}' instead." - ) - else: - _gamma = self.gamma - self._scale_, self._sigma_ = _gamma, np.sqrt(0.5 / _gamma) + data = (X, y) if sample_weight is None else (X, y, sample_weight) - data = to_table(*data, queue=QM.get_global_queue()) - params = self._get_onedal_params(data[0]) - result = self.train(params, *data) + self._sparse = sp.issparse(X) + + data_t = to_table(*data, queue=queue) + params = self._get_onedal_params(data_t[0]) + result = self.train(params, *data_t) if self._sparse: self.dual_coef_ = sp.csr_matrix(from_table(result.coeffs).T) self.support_vectors_ = sp.csr_matrix(from_table(result.support_vectors)) else: - self.dual_coef_ = from_table(result.coeffs).T - self.support_vectors_ = from_table(result.support_vectors) + self.dual_coef_ = from_table(result.coeffs, like=X).T + self.support_vectors_ = from_table(result.support_vectors, like=X) + + self.intercept_ = from_table(result.biases, like=X) - self.intercept_ = from_table(result.biases).ravel() - self.support_ = from_table(result.support_indices).ravel().astype("int") - self.n_features_in_ = X.shape[1] - self.shape_fit_ = X.shape + if len(self.intercept_.shape) > 1: + self.intercept_ = self.intercept_[:, 0] - if getattr(self, "classes_", None) is not None: - indices = y.take(self.support_, axis=0) - self._n_support = np.array( - [np.sum(indices == i) for i, _ in enumerate(self.classes_)] - ) - self._gamma = self._scale_ + self.support_ = from_table(result.support_indices, like=X) + + if len(self.support_.shape) > 1: + self.support_ = self.support_[:, 0] self._onedal_model = result.model return self @@ -211,138 +134,41 @@ def _create_model(self): m.support_vectors = to_table(self.support_vectors_) m.coeffs = to_table(self.dual_coef_.T) m.biases = to_table(self.intercept_) - - if self.svm_type is SVMtype.c_svc or self.svm_type is SVMtype.nu_svc: - m.first_class_response, m.second_class_response = 0, 1 return m - def _predict(self, X): + @supports_queue + def _infer(self, X, queue=None): _check_is_fitted(self) - if self.break_ties and self.decision_function_shape == "ovo": - raise ValueError( - "break_ties must be False when " "decision_function_shape is 'ovo'" - ) - - if isinstance(self, ClassifierMixin): - sv = self.support_vectors_ - if not self._sparse and sv.size > 0 and self._n_support.sum() != sv.shape[0]: - raise ValueError( - "The internal representation " - f"of {self.__class__.__name__} was altered" - ) - - if ( - self.break_ties - and self.decision_function_shape == "ovr" - and len(self.classes_) > 2 - ): - y = np.argmax(self.decision_function(X), axis=1) - else: - X = _check_array( - X, - dtype=[np.float64, np.float32], - force_all_finite=True, - accept_sparse="csr", - ) - _check_n_features(self, X, False) - - if self._sparse and not sp.isspmatrix(X): - X = sp.csr_matrix(X) - if self._sparse: - X.sort_indices() - if sp.issparse(X) and not self._sparse and not callable(self.kernel): - raise ValueError( - "cannot use sparse input in %r trained on dense data" - % type(self).__name__ - ) - - X = to_table(X, queue=QM.get_global_queue()) - params = self._get_onedal_params(X) - - if hasattr(self, "_onedal_model"): - model = self._onedal_model + if self._sparse: + if not _is_csr(X): + X = sp.csr_array(X) if hasattr(sp, "csr_array") else sp.csr_matrix(X) else: - model = self._create_model(module) - result = self.infer(params, model, X) - y = from_table(result.responses) - return y - - def _ovr_decision_function(self, predictions, confidences, n_classes): - n_samples = predictions.shape[0] - votes = np.zeros((n_samples, n_classes)) - sum_of_confidences = np.zeros((n_samples, n_classes)) - - k = 0 - for i in range(n_classes): - for j in range(i + 1, n_classes): - sum_of_confidences[:, i] -= confidences[:, k] - sum_of_confidences[:, j] += confidences[:, k] - votes[predictions[:, k] == 0, i] += 1 - votes[predictions[:, k] == 1, j] += 1 - k += 1 - - transformed_confidences = sum_of_confidences / ( - 3 * (np.abs(sum_of_confidences) + 1) - ) - return votes + transformed_confidences - - def _decision_function(self, X): - _check_is_fitted(self) - X = _check_array( - X, dtype=[np.float64, np.float32], force_all_finite=True, accept_sparse="csr" - ) - _check_n_features(self, X, False) + X.sort_indices() - if self._sparse and not sp.isspmatrix(X): - X = sp.csr_matrix(X) - if self._sparse: - X.sort_indices() - - if sp.issparse(X) and not self._sparse and not callable(self.kernel): - raise ValueError( - "cannot use sparse input in %r trained on dense data" - % type(self).__name__ - ) - - if isinstance(self, ClassifierMixin): - sv = self.support_vectors_ - if not self._sparse and sv.size > 0 and self._n_support.sum() != sv.shape[0]: - raise ValueError( - "The internal representation " - f"of {self.__class__.__name__} was altered" - ) - - X = to_table(X, queue=QM.get_global_queue()) + X = to_table(X, queue=queue) params = self._get_onedal_params(X) - if hasattr(self, "_onedal_model"): - model = self._onedal_model - else: - model = self._create_model(module) - result = self.infer(params, model, X) - decision_function = from_table(result.decision_function) + if self._onedal_model is None: + self._onedal_model = self._create_model() - if len(self.classes_) == 2: - decision_function = decision_function.ravel() + return self.infer(params, self._onedal_model, X) - if self.decision_function_shape == "ovr" and len(self.classes_) > 2: - decision_function = self._ovr_decision_function( - decision_function < 0, -decision_function, len(self.classes_) - ) - return decision_function + def predict(self, X, queue=None): + return from_table(self._infer(X, queue=queue).responses, like=X)[:, 0] -class SVR(RegressorMixin, BaseSVM): +class SVR(BaseSVM): def __init__( self, C=1.0, + nu=None, # not used epsilon=0.1, kernel="rbf", *, degree=3, - gamma="scale", + gamma=None, coef0=0.0, tol=1e-3, shrinking=True, @@ -350,11 +176,11 @@ def __init__( max_iter=-1, tau=1e-12, algorithm="thunder", - **kwargs, ): + super().__init__( C=C, - nu=0.5, + nu=nu, epsilon=epsilon, kernel=kernel, degree=degree, @@ -365,12 +191,8 @@ def __init__( cache_size=cache_size, max_iter=max_iter, tau=tau, - class_weight=None, - decision_function_shape=None, - break_ties=False, algorithm=algorithm, ) - self.svm_type = SVMtype.epsilon_svr @bind_default_backend("svm.regression") def train(self, *args, **kwargs): ... @@ -381,41 +203,36 @@ def infer(self, *args, **kwargs): ... @bind_default_backend("svm.regression") def model(self): ... - @supports_queue - def fit(self, X, y, sample_weight=None, queue=None): - return self._fit(X, y, sample_weight) - - @supports_queue - def predict(self, X, queue=None): - y = self._predict(X) - return y.ravel() + def _get_onedal_params(self, X): + params = super()._get_onedal_params(X) + # The nu parameter is not set + params.pop("nu") + return params -class SVC(ClassifierMixin, BaseSVM): +class SVC(BaseSVM): def __init__( self, C=1.0, + nu=None, # not used + epsilon=None, # not used kernel="rbf", *, degree=3, - gamma="scale", + gamma=None, coef0=0.0, tol=1e-3, shrinking=True, cache_size=200.0, max_iter=-1, tau=1e-12, - class_weight=None, - decision_function_shape="ovr", - break_ties=False, algorithm="thunder", - **kwargs, ): super().__init__( C=C, - nu=0.5, - epsilon=0.0, + nu=nu, + epsilon=epsilon, kernel=kernel, degree=degree, gamma=gamma, @@ -425,12 +242,20 @@ def __init__( cache_size=cache_size, max_iter=max_iter, tau=tau, - class_weight=class_weight, - decision_function_shape=decision_function_shape, - break_ties=break_ties, algorithm=algorithm, ) - self.svm_type = SVMtype.c_svc + + def _create_model(self): + m = super()._create_model() + m.first_class_response, m.second_class_response = 0, 1 + return m + + def _get_onedal_params(self, X): + params = super()._get_onedal_params(X) + # The nu and epsilon parameter are not used + params.pop("nu") + params.pop("epsilon") + return params @bind_default_backend("svm.classification") def train(self, *args, **kwargs): ... @@ -441,38 +266,21 @@ def infer(self, *args, **kwargs): ... @bind_default_backend("svm.classification") def model(self): ... - def _validate_targets(self, y, dtype): - y, self.class_weight_, self.classes_ = _validate_targets( - y, self.class_weight, dtype - ) - return y - - @supports_queue - def fit(self, X, y, sample_weight=None, queue=None): - return self._fit(X, y, sample_weight) - - @supports_queue - def predict(self, X, queue=None): - y = self._predict(X) - if len(self.classes_) == 2: - y = y.ravel() - return self.classes_.take(np.asarray(y, dtype=np.intp)).ravel() - - @supports_queue def decision_function(self, X, queue=None): - return self._decision_function(X) + return from_table(self._infer(X, queue=queue).decision_function, like=X) -class NuSVR(RegressorMixin, BaseSVM): +class NuSVR(BaseSVM): def __init__( self, nu=0.5, C=1.0, + epsilon=None, # not used kernel="rbf", *, degree=3, - gamma="scale", + gamma=None, coef0=0.0, tol=1e-3, shrinking=True, @@ -480,12 +288,11 @@ def __init__( max_iter=-1, tau=1e-12, algorithm="thunder", - **kwargs, ): super().__init__( C=C, nu=nu, - epsilon=0.0, + epsilon=epsilon, kernel=kernel, degree=degree, gamma=gamma, @@ -495,12 +302,14 @@ def __init__( cache_size=cache_size, max_iter=max_iter, tau=tau, - class_weight=None, - decision_function_shape=None, - break_ties=False, algorithm=algorithm, ) - self.svm_type = SVMtype.nu_svr + + def _get_onedal_params(self, X): + params = super()._get_onedal_params(X) + # The epsilon parameter is not used + params.pop("epsilon") + return params @bind_default_backend("svm.nu_regression") def train(self, *args, **kwargs): ... @@ -511,40 +320,30 @@ def infer(self, *args, **kwargs): ... @bind_default_backend("svm.nu_regression") def model(self): ... - @supports_queue - def fit(self, X, y, sample_weight=None, queue=None): - return self._fit(X, y, sample_weight) - @supports_queue - def predict(self, X, queue=None): - return self._predict(X).ravel() - - -class NuSVC(ClassifierMixin, BaseSVM): +class NuSVC(BaseSVM): def __init__( self, + C=None, # not used nu=0.5, + epsilon=None, # not used kernel="rbf", *, degree=3, - gamma="scale", + gamma=None, coef0=0.0, tol=1e-3, shrinking=True, cache_size=200.0, max_iter=-1, tau=1e-12, - class_weight=None, - decision_function_shape="ovr", - break_ties=False, algorithm="thunder", - **kwargs, ): super().__init__( - C=1.0, + C=C, nu=nu, - epsilon=0.0, + epsilon=epsilon, kernel=kernel, degree=degree, gamma=gamma, @@ -554,12 +353,20 @@ def __init__( cache_size=cache_size, max_iter=max_iter, tau=tau, - class_weight=class_weight, - decision_function_shape=decision_function_shape, - break_ties=break_ties, algorithm=algorithm, ) - self.svm_type = SVMtype.nu_svc + + def _create_model(self): + m = super()._create_model() + m.first_class_response, m.second_class_response = 0, 1 + return m + + def _get_onedal_params(self, X): + params = super()._get_onedal_params(X) + # The C and epsilon parameters are not used + params.pop("c") + params.pop("epsilon") + return params @bind_default_backend("svm.nu_classification") def train(self, *args, **kwargs): ... @@ -570,23 +377,5 @@ def infer(self, *args, **kwargs): ... @bind_default_backend("svm.nu_classification") def model(self): ... - def _validate_targets(self, y, dtype): - y, self.class_weight_, self.classes_ = _validate_targets( - y, self.class_weight, dtype - ) - return y - - @supports_queue - def fit(self, X, y, sample_weight=None, queue=None): - return self._fit(X, y, sample_weight) - - @supports_queue - def predict(self, X, queue=None): - y = self._predict(X) - if len(self.classes_) == 2: - y = y.ravel() - return self.classes_.take(np.asarray(y, dtype=np.intp)).ravel() - - @supports_queue def decision_function(self, X, queue=None): - return self._decision_function(X) + return from_table(self._infer(X, queue=queue).decision_function, like=X) diff --git a/onedal/svm/tests/test_csr_svm.py b/onedal/svm/tests/test_csr_svm.py index d7da6d404c..4432b5b07d 100644 --- a/onedal/svm/tests/test_csr_svm.py +++ b/onedal/svm/tests/test_csr_svm.py @@ -18,55 +18,64 @@ import pytest from numpy.testing import assert_array_almost_equal, assert_array_equal from scipy import sparse as sp -from sklearn import datasets -from sklearn.datasets import make_classification from onedal.common._mixin import ClassifierMixin -from onedal.svm import SVC, SVR +from onedal.svm import SVC from onedal.tests.utils._device_selection import ( get_queues, pass_if_not_implemented_for_gpu, ) +from onedal.utils import _sycl_queue_manager as QM def check_svm_model_equal( queue, dense_svm, sparse_svm, X_train, y_train, X_test, decimal=6 ): - dense_svm.fit(X_train.toarray(), y_train, queue=queue) - if sp.issparse(X_test): - X_test_dense = X_test.toarray() + if dense_svm.__class__.__module__.startswith("onedal"): + params = {"class_count": len(np.unique(y_train))} else: - X_test_dense = X_test - sparse_svm.fit(X_train, y_train, queue=queue) - assert sp.issparse(sparse_svm.support_vectors_) - assert sp.issparse(sparse_svm.dual_coef_) - assert_array_almost_equal( - dense_svm.support_vectors_, sparse_svm.support_vectors_.toarray(), decimal - ) - assert_array_almost_equal( - dense_svm.dual_coef_, sparse_svm.dual_coef_.toarray(), decimal - ) - assert_array_almost_equal(dense_svm.support_, sparse_svm.support_) - assert_array_almost_equal( - dense_svm.predict(X_test_dense, queue=queue), - sparse_svm.predict(X_test, queue=queue), - ) + params = {} + + with QM.manage_global_queue(queue): + + dense_svm.fit(X_train.toarray(), y_train, **params) + if sp.issparse(X_test): + X_test_dense = X_test.toarray() + else: + X_test_dense = X_test + sparse_svm.fit(X_train, y_train, **params) + + assert sp.issparse(sparse_svm.support_vectors_) + assert sp.issparse(sparse_svm.dual_coef_) - if isinstance(dense_svm, ClassifierMixin) and isinstance(sparse_svm, ClassifierMixin): assert_array_almost_equal( - dense_svm.decision_function(X_test_dense, queue=queue), - sparse_svm.decision_function(X_test, queue=queue), - decimal, + dense_svm.support_vectors_, sparse_svm.support_vectors_.toarray(), decimal + ) + assert_array_almost_equal( + dense_svm.dual_coef_, sparse_svm.dual_coef_.toarray(), decimal + ) + assert_array_almost_equal(dense_svm.support_, sparse_svm.support_) + assert_array_almost_equal( + dense_svm.predict(X_test_dense), sparse_svm.predict(X_test) ) + if isinstance(dense_svm, ClassifierMixin) and isinstance( + sparse_svm, ClassifierMixin + ): + assert_array_almost_equal( + dense_svm.decision_function(X_test_dense), + sparse_svm.decision_function(X_test), + decimal, + ) + def _test_simple_dataset(queue, kernel): - X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]) - sparse_X = sp.lil_matrix(X) - Y = [1, 1, 1, 2, 2, 2] + X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]], dtype=np.float64) + sparse_X = sp.csr_matrix(X) + Y = np.array([1, 1, 1, 2, 2, 2], dtype=np.float64) - X2 = np.array([[-1, -1], [2, 2], [3, 2]]) - sparse_X2 = sp.dok_matrix(X2) + X2 = np.array([[-1, -1], [2, 2], [3, 2]], dtype=np.float64) + sparse_X2 = sp.csr_matrix(X2) dataset = sparse_X, Y, sparse_X2 clf0 = SVC(kernel=kernel, gamma=1) @@ -92,82 +101,6 @@ def test_simple_dataset(queue, kernel): _test_simple_dataset(queue, kernel) -def _test_binary_dataset(queue, kernel): - X, y = make_classification(n_samples=80, n_features=20, n_classes=2, random_state=0) - sparse_X = sp.csr_matrix(X) - - dataset = sparse_X, y, sparse_X - clf0 = SVC(kernel=kernel) - clf1 = SVC(kernel=kernel) - check_svm_model_equal(queue, clf0, clf1, *dataset) - - -@pass_if_not_implemented_for_gpu(reason="not implemented") -@pytest.mark.parametrize( - "queue", - get_queues("cpu") - + [ - pytest.param( - get_queues("gpu"), - marks=pytest.mark.xfail( - reason=( - "raises UnknownError for linear and rbf, " - "Unimplemented error with inconsistent error message " - "for poly and sigmoid" - ) - ), - ) - ], -) -@pytest.mark.parametrize("kernel", ["linear", "rbf", "poly", "sigmoid"]) -def test_binary_dataset(queue, kernel): - _test_binary_dataset(queue, kernel) - - -def _test_iris(queue, kernel): - iris = datasets.load_iris() - rng = np.random.RandomState(0) - perm = rng.permutation(iris.target.size) - iris.data = iris.data[perm] - iris.target = iris.target[perm] - sparse_iris_data = sp.csr_matrix(iris.data) - - dataset = sparse_iris_data, iris.target, sparse_iris_data - - clf0 = SVC(kernel=kernel) - clf1 = SVC(kernel=kernel) - check_svm_model_equal(queue, clf0, clf1, *dataset, decimal=2) - - -@pass_if_not_implemented_for_gpu(reason="not implemented") -@pytest.mark.parametrize("queue", get_queues()) -@pytest.mark.parametrize("kernel", ["linear", "rbf", "poly", "sigmoid"]) -def test_iris(queue, kernel): - if kernel == "rbf": - pytest.skip("RBF CSR SVM test failing in 2025.0.") - _test_iris(queue, kernel) - - -def _test_diabetes(queue, kernel): - diabetes = datasets.load_diabetes() - - sparse_diabetes_data = sp.csr_matrix(diabetes.data) - dataset = sparse_diabetes_data, diabetes.target, sparse_diabetes_data - - clf0 = SVR(kernel=kernel, C=0.1) - clf1 = SVR(kernel=kernel, C=0.1) - check_svm_model_equal(queue, clf0, clf1, *dataset) - - -@pass_if_not_implemented_for_gpu(reason="not implemented") -@pytest.mark.parametrize("queue", get_queues()) -@pytest.mark.parametrize("kernel", ["linear", "rbf", "poly", "sigmoid"]) -def test_diabetes(queue, kernel): - if kernel == "sigmoid": - pytest.skip("Sparse sigmoid kernel function is buggy.") - _test_diabetes(queue, kernel) - - @pass_if_not_implemented_for_gpu(reason="csr svm is not implemented") @pytest.mark.xfail(reason="Failed test. Need investigate") @pytest.mark.parametrize("queue", get_queues()) @@ -344,9 +277,9 @@ def test_sparse_realdata(queue): 3.0, ] ) - - clf = SVC(kernel="linear").fit(X.toarray(), y, queue=queue) - sp_clf = SVC(kernel="linear").fit(X, y, queue=queue) + class_count = len(np.unique(y)) + clf = SVC(kernel="linear").fit(X.toarray(), y, class_count=class_count, queue=queue) + sp_clf = SVC(kernel="linear").fit(X, y, class_count=class_count, queue=queue) assert_array_equal(clf.support_vectors_, sp_clf.support_vectors_.toarray()) assert_array_equal(clf.dual_coef_, sp_clf.dual_coef_.toarray()) diff --git a/onedal/svm/tests/test_nusvc.py b/onedal/svm/tests/test_nusvc.py index 29e8d2272f..bcfa1011d9 100644 --- a/onedal/svm/tests/test_nusvc.py +++ b/onedal/svm/tests/test_nusvc.py @@ -19,6 +19,7 @@ from numpy.testing import assert_array_almost_equal, assert_array_equal from sklearn import datasets from sklearn.datasets import make_blobs +from sklearn.metrics import accuracy_score from sklearn.metrics.pairwise import rbf_kernel from sklearn.model_selection import train_test_split from sklearn.svm import NuSVC as SklearnNuSVC @@ -32,16 +33,16 @@ def _test_libsvm_parameters(queue, array_constr, dtype): X = array_constr([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]], dtype=dtype) - y = array_constr([1, 1, 1, 2, 2, 2], dtype=dtype) + y = array_constr([0, 0, 0, 1, 1, 1], dtype=dtype) - clf = NuSVC(kernel="linear").fit(X, y, queue=queue) + clf = NuSVC(kernel="linear").fit(X, y, class_count=2, queue=queue) assert_array_almost_equal( clf.dual_coef_, [[-0.04761905, -0.0952381, 0.0952381, 0.04761905]] ) assert_array_equal(clf.support_, [0, 1, 3, 4]) - assert_array_equal(clf.support_vectors_, X[clf.support_]) + assert_array_equal(clf.support_vectors_, X[clf.support_.astype(int)]) assert_array_equal(clf.intercept_, [0.0]) - assert_array_equal(clf.predict(X, queue=queue), y) + assert_array_equal(clf.predict(X, queue=queue).ravel(), y) @pass_if_not_implemented_for_gpu(reason="not implemented") @@ -52,73 +53,64 @@ def test_libsvm_parameters(queue, array_constr, dtype): _test_libsvm_parameters(queue, array_constr, dtype) -@pass_if_not_implemented_for_gpu(reason="not implemented") -@pytest.mark.parametrize("queue", get_queues()) -def test_class_weight(queue): - X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]) - y = np.array([1, 1, 1, 2, 2, 2]) - - clf = NuSVC(class_weight={1: 0.1}) - clf.fit(X, y, queue=queue) - assert_array_almost_equal(clf.predict(X, queue=queue), [2] * 6) - - @pass_if_not_implemented_for_gpu(reason="not implemented") @pytest.mark.parametrize("queue", get_queues()) def test_sample_weight(queue): - X = np.array([[-2, 0], [-1, -1], [0, -2], [0, 2], [1, 1], [2, 2]]) - y = np.array([1, 1, 1, 2, 2, 2]) + X = np.array([[-2, 0], [-1, -1], [0, -2], [0, 2], [1, 1], [2, 2]], dtype=np.float64) + y = np.array([1, 1, 1, 2, 2, 2], dtype=np.float64) clf = NuSVC(kernel="linear") - clf.fit(X, y, sample_weight=[1] * 6, queue=queue) + clf.fit(X, y, sample_weight=np.array([1.0] * 6), class_count=2, queue=queue) assert_array_almost_equal(clf.intercept_, [0.0]) @pass_if_not_implemented_for_gpu(reason="not implemented") @pytest.mark.parametrize("queue", get_queues()) def test_decision_function(queue): - X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] - Y = [1, 1, 1, 2, 2, 2] + X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]], dtype=np.float32) + Y = np.array([1, 1, 1, 2, 2, 2], dtype=np.float32) - clf = NuSVC(kernel="rbf", gamma=1, decision_function_shape="ovo") - clf.fit(X, Y, queue=queue) + clf = NuSVC(kernel="rbf", gamma=1) + clf.fit(X, Y, class_count=2, queue=queue) rbfs = rbf_kernel(X, clf.support_vectors_, gamma=clf.gamma) dec = np.dot(rbfs, clf.dual_coef_.T) + clf.intercept_ - assert_array_almost_equal(dec.ravel(), clf.decision_function(X, queue=queue)) + assert_array_almost_equal(dec.ravel(), clf.decision_function(X, queue=queue).ravel()) @pass_if_not_implemented_for_gpu(reason="not implemented") @pytest.mark.parametrize("queue", get_queues()) def test_iris(queue): iris = datasets.load_iris() - clf = NuSVC(kernel="linear").fit(iris.data, iris.target, queue=queue) - assert clf.score(iris.data, iris.target, queue=queue) > 0.9 - assert_array_equal(clf.classes_, np.sort(clf.classes_)) + class_count = len(np.unique(iris.target)) + clf = NuSVC(kernel="linear").fit( + iris.data, iris.target, class_count=class_count, queue=queue + ) + assert accuracy_score(iris.target, clf.predict(iris.data, queue=queue)) > 0.9 @pass_if_not_implemented_for_gpu(reason="not implemented") @pytest.mark.parametrize("queue", get_queues()) def test_decision_function_shape(queue): X, y = make_blobs(n_samples=80, centers=5, random_state=0) + class_count = len(np.unique(y)) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) - # check shape of ovo_decition_function=True - clf = NuSVC(kernel="linear", decision_function_shape="ovo").fit( - X_train, y_train, queue=queue + clf = NuSVC(kernel="linear").fit( + X_train, y_train, class_count=class_count, queue=queue ) dec = clf.decision_function(X_train, queue=queue) assert dec.shape == (len(X_train), 10) - # with pytest.raises(ValueError, match="must be either 'ovr' or 'ovo'"): - # SVC(decision_function_shape='bad').fit(X_train, y_train) - @pass_if_not_implemented_for_gpu(reason="not implemented") @pytest.mark.parametrize("queue", get_queues()) def test_pickle(queue): iris = datasets.load_iris() - clf = NuSVC(kernel="linear").fit(iris.data, iris.target, queue=queue) + class_count = len(np.unique(iris.target)) + clf = NuSVC(kernel="linear").fit( + iris.data, iris.target, class_count=class_count, queue=queue + ) expected = clf.decision_function(iris.data, queue=queue) import pickle @@ -133,12 +125,19 @@ def test_pickle(queue): def _test_cancer_rbf_compare_with_sklearn(queue, nu, gamma): cancer = datasets.load_breast_cancer() - - clf = NuSVC(kernel="rbf", gamma=gamma, nu=nu) - clf.fit(cancer.data, cancer.target, queue=queue) - result = clf.score(cancer.data, cancer.target, queue=queue) - - clf = SklearnNuSVC(kernel="rbf", gamma=gamma, nu=nu) + class_count = len(np.unique(cancer.target)) + if gamma == "auto": + _gamma = 1.0 / cancer.data.shape[1] + elif gamma == "scale": + _gamma = 1.0 / (cancer.data.shape[1] * cancer.data.var()) + else: + _gamma = gamma + + clf = NuSVC(kernel="rbf", gamma=_gamma, nu=nu) + clf.fit(cancer.data, cancer.target, class_count=class_count, queue=queue) + result = accuracy_score(cancer.target, clf.predict(cancer.data, queue=queue)) + + clf = SklearnNuSVC(kernel="rbf", gamma=_gamma, nu=nu) clf.fit(cancer.data, cancer.target) expected = clf.score(cancer.data, cancer.target) @@ -156,10 +155,11 @@ def test_cancer_rbf_compare_with_sklearn(queue, nu, gamma): def _test_cancer_linear_compare_with_sklearn(queue, nu): cancer = datasets.load_breast_cancer() + class_count = len(np.unique(cancer.target)) clf = NuSVC(kernel="linear", nu=nu) - clf.fit(cancer.data, cancer.target, queue=queue) - result = clf.score(cancer.data, cancer.target, queue=queue) + clf.fit(cancer.data, cancer.target, class_count=class_count, queue=queue) + result = accuracy_score(cancer.target, clf.predict(cancer.data, queue=queue)) clf = SklearnNuSVC(kernel="linear", nu=nu) clf.fit(cancer.data, cancer.target) @@ -178,12 +178,14 @@ def test_cancer_linear_compare_with_sklearn(queue, nu): def _test_cancer_poly_compare_with_sklearn(queue, params): cancer = datasets.load_breast_cancer() - - clf = NuSVC(kernel="poly", **params) - clf.fit(cancer.data, cancer.target, queue=queue) - result = clf.score(cancer.data, cancer.target, queue=queue) - - clf = SklearnNuSVC(kernel="poly", **params) + class_count = len(np.unique(cancer.target)) + # gamma="scale" + _gamma = 1.0 / (cancer.data.shape[1] * cancer.data.var()) + clf = NuSVC(kernel="poly", gamma=_gamma, **params) + clf.fit(cancer.data, cancer.target, class_count=class_count, queue=queue) + result = accuracy_score(cancer.target, clf.predict(cancer.data, queue=queue)) + + clf = SklearnNuSVC(kernel="poly", gamma=_gamma, **params) clf.fit(cancer.data, cancer.target) expected = clf.score(cancer.data, cancer.target) @@ -196,8 +198,8 @@ def _test_cancer_poly_compare_with_sklearn(queue, params): @pytest.mark.parametrize( "params", [ - {"degree": 2, "coef0": 0.1, "gamma": "scale", "nu": 0.25}, - {"degree": 3, "coef0": 0.0, "gamma": "scale", "nu": 0.5}, + {"degree": 2, "coef0": 0.1, "nu": 0.25}, + {"degree": 3, "coef0": 0.0, "nu": 0.5}, ], ) def test_cancer_poly_compare_with_sklearn(queue, params): diff --git a/onedal/svm/tests/test_nusvr.py b/onedal/svm/tests/test_nusvr.py index 6bcc04e9f4..f24f9ef0d9 100644 --- a/onedal/svm/tests/test_nusvr.py +++ b/onedal/svm/tests/test_nusvr.py @@ -18,6 +18,7 @@ import pytest from numpy.testing import assert_allclose, assert_array_almost_equal, assert_array_equal from sklearn import datasets +from sklearn.metrics import r2_score from sklearn.metrics.pairwise import rbf_kernel from sklearn.svm import NuSVR as SklearnNuSVR @@ -36,7 +37,7 @@ def test_diabetes_simple(queue): diabetes = datasets.load_diabetes() clf = NuSVR(kernel="linear", C=10.0) clf.fit(diabetes.data, diabetes.target, queue=queue) - assert clf.score(diabetes.data, diabetes.target, queue=queue) > 0.02 + assert r2_score(diabetes.target, clf.predict(diabetes.data, queue=queue)) > 0.02 @pass_if_not_implemented_for_gpu(reason="not implemented for GPU") @@ -89,9 +90,11 @@ def test_predict(queue): def _test_diabetes_compare_with_sklearn(queue, kernel): diabetes = datasets.load_diabetes() - clf_onedal = NuSVR(kernel=kernel, nu=0.25, C=10.0) + gamma = 1.0 / (diabetes.data.shape[1] * diabetes.data.var()) + # set gamma to value that would occur when gamma="scale" + clf_onedal = NuSVR(kernel=kernel, nu=0.25, C=10.0, gamma=gamma) clf_onedal.fit(diabetes.data, diabetes.target, queue=queue) - result = clf_onedal.score(diabetes.data, diabetes.target, queue=queue) + result = r2_score(diabetes.target, clf_onedal.predict(diabetes.data, queue=queue)) clf_sklearn = SklearnNuSVR(kernel=kernel, nu=0.25, C=10.0) clf_sklearn.fit(diabetes.data, diabetes.target) @@ -116,12 +119,18 @@ def test_diabetes_compare_with_sklearn(queue, kernel): def _test_synth_rbf_compare_with_sklearn(queue, C, nu, gamma): x, y = datasets.make_regression(**synth_params) - - clf = NuSVR(kernel="rbf", gamma=gamma, C=C, nu=nu) + if gamma == "auto": + _gamma = 1.0 / x.shape[1] + elif gamma == "scale": + _gamma = 1.0 / (x.shape[1] * x.var()) + else: + _gamma = gamma + + clf = NuSVR(kernel="rbf", gamma=_gamma, C=C, nu=nu) clf.fit(x, y, queue=queue) - result = clf.score(x, y, queue=queue) + result = r2_score(y, clf.predict(x, queue=queue)) - clf = SklearnNuSVR(kernel="rbf", gamma=gamma, C=C, nu=nu) + clf = SklearnNuSVR(kernel="rbf", gamma=_gamma, C=C, nu=nu) clf.fit(x, y) expected = clf.score(x, y) @@ -141,11 +150,14 @@ def test_synth_rbf_compare_with_sklearn(queue, C, nu, gamma): def _test_synth_linear_compare_with_sklearn(queue, C, nu): x, y = datasets.make_regression(**synth_params) - clf = NuSVR(kernel="linear", C=C, nu=nu) + # gamma is set separately + gamma = 1.0 / x.shape[1] + + clf = NuSVR(kernel="linear", C=C, nu=nu, gamma=gamma) clf.fit(x, y, queue=queue) - result = clf.score(x, y, queue=queue) + result = r2_score(y, clf.predict(x, queue=queue)) - clf = SklearnNuSVR(kernel="linear", C=C, nu=nu) + clf = SklearnNuSVR(kernel="linear", C=C, nu=nu, gamma=gamma) clf.fit(x, y) expected = clf.score(x, y) @@ -165,10 +177,11 @@ def test_synth_linear_compare_with_sklearn(queue, C, nu): def _test_synth_poly_compare_with_sklearn(queue, params): x, y = datasets.make_regression(**synth_params) + params["gamma"] = 1 / (x.shape[1] * x.var()) clf = NuSVR(kernel="poly", **params) clf.fit(x, y, queue=queue) - result = clf.score(x, y, queue=queue) + result = r2_score(y, clf.predict(x, queue=queue)) clf = SklearnNuSVR(kernel="poly", **params) clf.fit(x, y) @@ -183,8 +196,8 @@ def _test_synth_poly_compare_with_sklearn(queue, params): @pytest.mark.parametrize( "params", [ - {"degree": 2, "coef0": 0.1, "gamma": "scale", "C": 100, "nu": 0.25}, - {"degree": 3, "coef0": 0.0, "gamma": "scale", "C": 1000, "nu": 0.75}, + {"degree": 2, "coef0": 0.1, "C": 100, "nu": 0.25}, + {"degree": 3, "coef0": 0.0, "C": 1000, "nu": 0.75}, ], ) def test_synth_poly_compare_with_sklearn(queue, params): @@ -196,7 +209,7 @@ def test_synth_poly_compare_with_sklearn(queue, params): def test_pickle(queue): diabetes = datasets.load_diabetes() - clf = NuSVR(kernel="rbf", C=10.0) + clf = NuSVR(kernel="linear", C=10.0) clf.fit(diabetes.data, diabetes.target, queue=queue) expected = clf.predict(diabetes.data, queue=queue) diff --git a/onedal/svm/tests/test_svc.py b/onedal/svm/tests/test_svc.py index f97d01c091..a115b5e381 100644 --- a/onedal/svm/tests/test_svc.py +++ b/onedal/svm/tests/test_svc.py @@ -24,10 +24,10 @@ import numpy as np import pytest -import sklearn.utils.estimator_checks from numpy.testing import assert_array_almost_equal, assert_array_equal from sklearn import datasets from sklearn.datasets import make_blobs +from sklearn.metrics import accuracy_score from sklearn.metrics.pairwise import rbf_kernel from sklearn.model_selection import train_test_split @@ -40,14 +40,14 @@ def _test_libsvm_parameters(queue, array_constr, dtype): X = array_constr([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]], dtype=dtype) - y = array_constr([1, 1, 1, 2, 2, 2], dtype=dtype) + y = array_constr([0, 0, 0, 1, 1, 1], dtype=dtype) - clf = SVC(kernel="linear").fit(X, y, queue=queue) + clf = SVC(kernel="linear").fit(X, y, class_count=2, queue=queue) assert_array_equal(clf.dual_coef_, [[-0.25, 0.25]]) assert_array_equal(clf.support_, [1, 3]) assert_array_equal(clf.support_vectors_, (X[1], X[3])) assert_array_equal(clf.intercept_, [0.0]) - assert_array_equal(clf.predict(X), y) + assert_array_equal(clf.predict(X).ravel(), y) @pytest.mark.parametrize("queue", get_queues()) @@ -59,37 +59,15 @@ def test_libsvm_parameters(queue, array_constr, dtype): _test_libsvm_parameters(queue, array_constr, dtype) -@pass_if_not_implemented_for_gpu(reason="class weights are not implemented") -@pytest.mark.parametrize( - "queue", - get_queues("cpu") - + [ - pytest.param( - get_queues("gpu"), - marks=pytest.mark.xfail( - reason="class weights are not implemented but the error is not raised" - ), - ) - ], -) -def test_class_weight(queue): - X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]) - y = np.array([1, 1, 1, 2, 2, 2]) - - clf = SVC(class_weight={1: 0.1}) - clf.fit(X, y, queue=queue) - assert_array_almost_equal(clf.predict(X, queue=queue), [2] * 6) - - @pytest.mark.parametrize("queue", get_queues()) def test_sample_weight(queue): if queue and queue.sycl_device.is_gpu: pytest.skip("Sporadic failures on GPU sycl_queue.") - X = np.array([[-2, 0], [-1, -1], [0, -2], [0, 2], [1, 1], [2, 2]]) - y = np.array([1, 1, 1, 2, 2, 2]) + X = np.array([[-2, 0], [-1, -1], [0, -2], [0, 2], [1, 1], [2, 2]], dtype=np.float64) + y = np.array([1, 1, 1, 2, 2, 2], dtype=np.float64) clf = SVC(kernel="linear") - clf.fit(X, y, sample_weight=[1] * 6, queue=queue) + clf.fit(X, y, sample_weight=np.array([1.0] * 6), class_count=2, queue=queue) assert_array_almost_equal(clf.intercept_, [0.0]) @@ -98,45 +76,46 @@ def test_decision_function(queue): X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]], dtype=np.float32) Y = np.array([1, 1, 1, 2, 2, 2], dtype=np.float32) - clf = SVC(kernel="rbf", gamma=1, decision_function_shape="ovo") - clf.fit(X, Y, queue=queue) + clf = SVC(kernel="rbf", gamma=1) + clf.fit(X, Y, class_count=2, queue=queue) rbfs = rbf_kernel(X, clf.support_vectors_, gamma=clf.gamma) dec = np.dot(rbfs, clf.dual_coef_.T) + clf.intercept_ - assert_array_almost_equal(dec.ravel(), clf.decision_function(X, queue=queue)) + assert_array_almost_equal(dec.ravel(), clf.decision_function(X, queue=queue).ravel()) @pass_if_not_implemented_for_gpu(reason="not implemented") @pytest.mark.parametrize("queue", get_queues()) def test_iris(queue): iris = datasets.load_iris() - clf = SVC(kernel="linear").fit(iris.data, iris.target, queue=queue) - assert clf.score(iris.data, iris.target, queue=queue) > 0.9 - assert_array_equal(clf.classes_, np.sort(clf.classes_)) + class_count = len(np.unique(iris.target)) + clf = SVC(kernel="linear").fit( + iris.data, iris.target, class_count=class_count, queue=queue + ) + assert accuracy_score(iris.target, clf.predict(iris.data, queue=queue)) > 0.9 @pass_if_not_implemented_for_gpu(reason="not implemented") @pytest.mark.parametrize("queue", get_queues()) def test_decision_function_shape(queue): X, y = make_blobs(n_samples=80, centers=5, random_state=0) - X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + X_train, _, y_train, _ = train_test_split(X, y, random_state=0) + class_count = len(np.unique(y_train)) # check shape of ovo_decition_function=True - clf = SVC(kernel="linear", decision_function_shape="ovo").fit( - X_train, y_train, queue=queue - ) + clf = SVC(kernel="linear").fit(X_train, y_train, class_count=class_count, queue=queue) dec = clf.decision_function(X_train, queue=queue) assert dec.shape == (len(X_train), 10) - with pytest.raises(ValueError, match="must be either 'ovr' or 'ovo'"): - SVC(decision_function_shape="bad").fit(X_train, y_train, queue=queue) - @pass_if_not_implemented_for_gpu(reason="not implemented") @pytest.mark.parametrize("queue", get_queues()) def test_pickle(queue): iris = datasets.load_iris() - clf = SVC(kernel="linear").fit(iris.data, iris.target, queue=queue) + class_count = len(np.unique(iris.target)) + clf = SVC(kernel="linear").fit( + iris.data, iris.target, class_count=class_count, queue=queue + ) expected = clf.decision_function(iris.data, queue=queue) import pickle @@ -164,13 +143,11 @@ def test_pickle(queue): ) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_svc_sigmoid(queue, dtype): - X_train = np.array( - [[-1, 2], [0, 0], [2, -1], [+1, +1], [+1, +2], [+2, +1]], dtype=dtype - ) + X_train = np.array([[-1, 2], [0, 0], [2, -1], [1, 1], [1, 2], [2, 1]], dtype=dtype) X_test = np.array([[0, 2], [0.5, 0.5], [0.3, 0.1], [2, 0], [-1, -1]], dtype=dtype) - y_train = np.array([1, 1, 1, 2, 2, 2], dtype=dtype) - svc = SVC(kernel="sigmoid").fit(X_train, y_train, queue=queue) + y_train = np.array([0, 0, 0, 1, 1, 1], dtype=dtype) + svc = SVC(kernel="sigmoid").fit(X_train, y_train, class_count=2, queue=queue) assert_array_equal(svc.dual_coef_, [[-1, -1, -1, 1, 1, 1]]) assert_array_equal(svc.support_, [0, 1, 2, 3, 4, 5]) - assert_array_equal(svc.predict(X_test, queue=queue), [2, 2, 1, 2, 1]) + assert_array_equal(svc.predict(X_test, queue=queue).ravel(), [1, 1, 0, 1, 0]) diff --git a/onedal/svm/tests/test_svr.py b/onedal/svm/tests/test_svr.py index 8432fb09b3..c5e7de2ff3 100644 --- a/onedal/svm/tests/test_svr.py +++ b/onedal/svm/tests/test_svr.py @@ -18,6 +18,7 @@ import pytest from numpy.testing import assert_allclose, assert_array_almost_equal, assert_array_equal from sklearn import datasets +from sklearn.metrics import r2_score from sklearn.metrics.pairwise import rbf_kernel from sklearn.svm import SVR as SklearnSVR @@ -51,7 +52,7 @@ def test_diabetes_simple(queue): diabetes = datasets.load_diabetes() clf = SVR(kernel="linear", C=10.0) clf.fit(diabetes.data, diabetes.target, queue=queue) - assert clf.score(diabetes.data, diabetes.target, queue=queue) > 0.02 + assert r2_score(diabetes.target, clf.predict(diabetes.data, queue=queue)) > 0.02 @pass_if_not_implemented_for_gpu(reason="Regression SVM is not implemented for GPU") @@ -106,7 +107,7 @@ def _test_diabetes_compare_with_sklearn(queue, kernel): diabetes = datasets.load_diabetes() clf_onedal = SVR(kernel=kernel, C=10.0, gamma=2) clf_onedal.fit(diabetes.data, diabetes.target, queue=queue) - result = clf_onedal.score(diabetes.data, diabetes.target, queue=queue) + result = r2_score(diabetes.target, clf_onedal.predict(diabetes.data, queue=queue)) clf_sklearn = SklearnSVR(kernel=kernel, C=10.0, gamma=2) clf_sklearn.fit(diabetes.data, diabetes.target) @@ -131,11 +132,18 @@ def test_diabetes_compare_with_sklearn(queue, kernel): def _test_synth_rbf_compare_with_sklearn(queue, C, gamma): x, y = datasets.make_regression(**synth_params) - clf = SVR(kernel="rbf", gamma=gamma, C=C) + if gamma == "auto": + _gamma = 1.0 / x.shape[1] + elif gamma == "scale": + _gamma = 1.0 / (x.shape[1] * x.var()) + else: + _gamma = gamma + + clf = SVR(kernel="rbf", gamma=_gamma, C=C) clf.fit(x, y, queue=queue) - result = clf.score(x, y, queue=queue) + result = r2_score(y, clf.predict(x, queue=queue)) - clf = SklearnSVR(kernel="rbf", gamma=gamma, C=C) + clf = SklearnSVR(kernel="rbf", gamma=_gamma, C=C) clf.fit(x, y) expected = clf.score(x, y) @@ -155,7 +163,7 @@ def _test_synth_linear_compare_with_sklearn(queue, C): x, y = datasets.make_regression(**synth_params) clf = SVR(kernel="linear", C=C) clf.fit(x, y, queue=queue) - result = clf.score(x, y, queue=queue) + result = r2_score(y, clf.predict(x, queue=queue)) clf = SklearnSVR(kernel="linear", C=C) clf.fit(x, y) @@ -176,9 +184,15 @@ def test_synth_linear_compare_with_sklearn(queue, C): def _test_synth_poly_compare_with_sklearn(queue, params): x, y = datasets.make_regression(**synth_params) + if params["gamma"] == "auto": + params["gamma"] = 1.0 / x.shape[1] + + elif params["gamma"] == "scale": + params["gamma"] = 1.0 / (x.shape[1] * x.var()) + clf = SVR(kernel="poly", **params) clf.fit(x, y, queue=queue) - result = clf.score(x, y, queue=queue) + result = r2_score(y, clf.predict(x, queue=queue)) clf = SklearnSVR(kernel="poly", **params) clf.fit(x, y) @@ -206,22 +220,24 @@ def test_synth_poly_compare_with_sklearn(queue, params): def test_sided_sample_weight(queue): clf = SVR(C=1e-2, kernel="linear") - X = [[-2, 0], [-1, -1], [0, -2], [0, 2], [1, 1], [2, 0]] - Y = [1, 1, 1, 2, 2, 2] + X = np.array([[-2, 0], [-1, -1], [0, -2], [0, 2], [1, 1], [2, 0]], dtype=np.float64) + Y = np.array([1, 1, 1, 2, 2, 2], dtype=np.float64) - sample_weight = [10.0, 0.1, 0.1, 0.1, 0.1, 10] + X_pred = np.array([[-1.0, 1.0]], dtype=np.float64) + + sample_weight = np.array([10.0, 0.1, 0.1, 0.1, 0.1, 10], dtype=np.float64) clf.fit(X, Y, sample_weight=sample_weight, queue=queue) - y_pred = clf.predict([[-1.0, 1.0]], queue=queue) + y_pred = clf.predict(X_pred, queue=queue) assert y_pred < 1.5 - sample_weight = [1.0, 0.1, 10.0, 10.0, 0.1, 0.1] + sample_weight = np.array([1.0, 0.1, 10.0, 10.0, 0.1, 0.1], dtype=np.float64) clf.fit(X, Y, sample_weight=sample_weight, queue=queue) - y_pred = clf.predict([[-1.0, 1.0]], queue=queue) + y_pred = clf.predict(X_pred, queue=queue) assert y_pred > 1.5 - sample_weight = [1] * 6 + sample_weight = np.array([1] * 6, dtype=np.float64) clf.fit(X, Y, sample_weight=sample_weight, queue=queue) - y_pred = clf.predict([[-1.0, 1.0]], queue=queue) + y_pred = clf.predict(X_pred, queue=queue) assert y_pred == pytest.approx(1.5) @@ -229,7 +245,7 @@ def test_sided_sample_weight(queue): @pytest.mark.parametrize("queue", get_queues()) def test_pickle(queue): diabetes = datasets.load_diabetes() - clf = SVR(kernel="rbf", C=10.0) + clf = SVR(kernel="linear", C=10.0) clf.fit(diabetes.data, diabetes.target, queue=queue) expected = clf.predict(diabetes.data, queue=queue) diff --git a/sklearnex/_utils.py b/sklearnex/_utils.py index aeed28c442..d294f6a225 100755 --- a/sklearnex/_utils.py +++ b/sklearnex/_utils.py @@ -19,14 +19,11 @@ import re import sys import warnings -from abc import ABC - -import sklearn from daal4py.sklearn._utils import ( PatchingConditionsChain as daal4py_PatchingConditionsChain, ) -from daal4py.sklearn._utils import daal_check_version, sklearn_check_version +from daal4py.sklearn._utils import sklearn_check_version from onedal.common.hyperparameters import ( get_hyperparameters as onedal_get_hyperparameters, ) @@ -120,10 +117,6 @@ def get_patch_message(s, queue=None, transferred_to_host=True): return message -def get_sklearnex_version(rule): - return daal_check_version(rule) - - def register_hyperparameters(hyperparameters_map): """Decorator for hyperparameters support in estimator class. diff --git a/sklearnex/svm/__init__.py b/sklearnex/svm/__init__.py index cf43ad4e37..8f673f8848 100755 --- a/sklearnex/svm/__init__.py +++ b/sklearnex/svm/__init__.py @@ -14,13 +14,10 @@ # limitations under the License. # ============================================================================== -from .._utils import get_sklearnex_version +from daal4py.sklearn._utils import daal_check_version -if get_sklearnex_version((2021, "P", 300)): - from .nusvc import NuSVC - from .nusvr import NuSVR - from .svc import SVC - from .svr import SVR +if daal_check_version((2021, "P", 300)): + from ._classes import SVC, SVR, NuSVC, NuSVR __all__ = ["SVR", "SVC", "NuSVC", "NuSVR"] else: diff --git a/sklearnex/svm/_base.py b/sklearnex/svm/_base.py new file mode 100644 index 0000000000..b5647dd475 --- /dev/null +++ b/sklearnex/svm/_base.py @@ -0,0 +1,673 @@ +# ============================================================================== +# Copyright Contributors to the oneDAL Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import warnings +from functools import wraps +from numbers import Real + +import numpy as np +from scipy import sparse as sp +from sklearn.base import RegressorMixin +from sklearn.calibration import CalibratedClassifierCV +from sklearn.exceptions import NotFittedError +from sklearn.metrics import accuracy_score, r2_score +from sklearn.svm._base import BaseLibSVM as _sklearn_BaseLibSVM +from sklearn.svm._base import BaseSVC as _sklearn_BaseSVC +from sklearn.utils.metaestimators import available_if +from sklearn.utils.multiclass import check_classification_targets +from sklearn.utils.validation import check_is_fitted, column_or_1d + +from daal4py.sklearn._utils import sklearn_check_version + +from .._config import config_context, get_config +from .._device_offload import dispatch, wrap_output_data +from .._utils import PatchingConditionsChain +from ..base import oneDALEstimator +from ..utils._array_api import get_namespace +from ..utils.class_weight import _compute_class_weight +from ..utils.validation import _check_sample_weight, validate_data + +if sklearn_check_version("1.6"): + from sklearn.calibration import _fit_calibrator + from sklearn.frozen import FrozenEstimator + from sklearn.utils import indexable + from sklearn.utils._response import _get_response_values + from sklearn.utils.validation import check_is_fitted + + def _prefit_CalibratedClassifierCV_fit(self, X, y, **fit_params): + # This is a stop-gap solution where the cv='prefit' of CalibratedClassifierCV + # was removed and the single fold solution needs to be maintained. Discussion + # of the mathematical and performance implications of this choice can be found + # here: https://github.com/uxlfoundation/scikit-learn-intelex/pull/1879 + # This is distilled from the sklearn CalibratedClassifierCV for sklearn <1.8 for + # use in sklearn > 1.8 to maintain performance. + xp, _ = get_namespace(X, y) + check_classification_targets(y) + X, y = indexable(X, y) + + estimator = self._get_estimator() + + self.calibrated_classifiers_ = [] + check_is_fitted(self.estimator, attributes=["classes_"]) + self.classes_ = self.estimator.classes_ + + predictions, _ = _get_response_values( + estimator, + X, + response_method=["decision_function", "predict_proba"], + ) + if predictions.ndim == 1: + # Reshape binary output from `(n_samples,)` to `(n_samples, 1)` + predictions = xp.reshape(predictions, (-1, 1)) + + calibrated_classifier = _fit_calibrator( + estimator, + predictions, + y, + self.classes_, + self.method, + ) + self.calibrated_classifiers_.append(calibrated_classifier) + + first_clf = self.calibrated_classifiers_[0].estimator + if hasattr(first_clf, "n_features_in_"): + self.n_features_in_ = first_clf.n_features_in_ + if hasattr(first_clf, "feature_names_in_"): + self.feature_names_in_ = first_clf.feature_names_in_ + return self + + +class BaseSVM(oneDALEstimator): + + _onedal_factory = None + + @property + def _dual_coef_(self): + return self._dualcoef_ + + @_dual_coef_.setter + def _dual_coef_(self, value): + self._dualcoef_ = value + if hasattr(self, "_onedal_estimator"): + self._onedal_estimator.dual_coef_ = value + self._onedal_estimator._onedal_model = None + + @_dual_coef_.deleter + def _dual_coef_(self): + del self._dualcoef_ + + @property + def intercept_(self): + return self._icept_ + + @intercept_.setter + def intercept_(self, value): + self._icept_ = value + if hasattr(self, "_onedal_estimator"): + self._onedal_estimator.intercept_ = value + self._onedal_estimator._onedal_model = None + + @intercept_.deleter + def intercept_(self): + del self._icept_ + + def _onedal_gpu_supported(self, method_name, *data): + class_name = self.__class__.__name__ + patching_status = PatchingConditionsChain( + f"sklearn.svm.{class_name}.{method_name}" + ) + patching_status.and_conditions([(False, "GPU offloading is not supported.")]) + return patching_status + + def _onedal_cpu_supported(self, method_name, *data): + class_name = self.__class__.__name__ + patching_status = PatchingConditionsChain( + f"sklearn.svm.{class_name}.{method_name}" + ) + if method_name == "fit": + patching_status.and_conditions( + [ + ( + self.kernel in ["linear", "rbf", "poly", "sigmoid"], + f'Kernel is "{self.kernel}" while ' + '"linear", "rbf", "poly" and "sigmoid" are only supported.', + ) + ] + ) + return patching_status + elif method_name in self._n_jobs_supported_onedal_methods: + # _n_jobs_supported_onedal_methods is different for classifiers vs regressors + # and can be easily used to find which methods have underlying direct onedal + # support. + patching_status.and_conditions( + [(hasattr(self, "_onedal_estimator"), "oneDAL model was not trained.")] + ) + return patching_status + raise RuntimeError(f"Unknown method {method_name} in {class_name}") + + def _svm_sample_weight_check(self, sample_weight, y, xp): + # This is purely for sklearn conformance. SVM algos in SVM raise unique errors. + if xp.all(sample_weight <= 0): + raise ValueError("Invalid input - all samples have zero or negative weights.") + + def _compute_gamma_sigma(self, X): + # only run extended conversion if kernel is not linear + # set to a value = 1.0, so gamma will always be passed to + # the onedal estimator as a float type. This replicates functionality + # directly out of scikit-learn to enable various variable gamma values. + if self.kernel == "linear": + return 1.0 + + if isinstance(self.gamma, str): + if self.gamma == "scale": + if sp.issparse(X): + # var = E[X^2] - E[X]^2 + X_sc = (X.multiply(X)).mean() - (X.mean()) ** 2 + else: + xp, _ = get_namespace(X) + X_sc = xp.var(X) + _gamma = 1.0 / (X.shape[1] * float(X_sc)) if X_sc != 0 else 1.0 + elif self.gamma == "auto": + _gamma = 1.0 / X.shape[1] + else: + raise ValueError( + "When 'gamma' is a string, it should be either 'scale' or " + "'auto'. Got '{}' instead.".format(self.gamma) + ) + else: + if sklearn_check_version("1.1") and not sklearn_check_version("1.2"): + if isinstance(self.gamma, Real): + if self.gamma <= 0: + msg = ( + f"gamma value must be > 0; {self.gamma!r} is invalid. Use" + " a positive number or use 'auto' to set gamma to a" + " value of 1 / n_features." + ) + raise ValueError(msg) + _gamma = self.gamma + else: + msg = ( + "The gamma value should be set to 'scale', 'auto' or a" + f" positive float value. {self.gamma!r} is not a valid option" + ) + raise ValueError(msg) + else: + _gamma = self.gamma + return _gamma + + def _onedal_predict(self, X, queue=None, xp=None): + if xp is None: + xp, _ = get_namespace(X) + + X = validate_data( + self, + X, + dtype=[xp.float64, xp.float32], + accept_sparse="csr", + reset=False, + ) + + return self._onedal_estimator.predict(X, queue=queue) + + +class BaseSVC(BaseSVM): + + # overwrite _validate_targets for array API support + def _validate_targets(self, y): + xp, is_array_api_compliant = get_namespace(y) + + if not is_array_api_compliant: + y = super()._validate_targets(y) + else: + # _validate_targets equivalent: + y_ = column_or_1d(y, warn=True) + check_classification_targets(y) + cls, y = xp.unique_inverse(y_) + self.class_weight_ = _compute_class_weight( + self.class_weight, classes=cls, y=y_ + ) + if cls.shape[0] < 2: + raise ValueError( + "The number of classes has to be greater than one; got %d class" + % len(cls) + ) + + self.classes_ = cls + return y + + def _onedal_fit(self, X, y, sample_weight=None, queue=None): + if not sklearn_check_version("1.2"): + if self.decision_function_shape not in ("ovr", "ovo", None): + raise ValueError( + f"decision_function_shape must be either 'ovr' or 'ovo', " + f"got {self.decision_function_shape}." + ) + + xp, _ = get_namespace(X, y, sample_weight) + + X, y = validate_data( + self, + X, + y, + dtype=[xp.float64, xp.float32], + accept_sparse="csr", + ) + + y = self._validate_targets(y) + + if self.class_weight is not None or sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X) + # oneDAL only accepts sample_weights, apply class_weight directly + if self.class_weight is not None: + for i, v in enumerate(self.class_weight_): + sample_weight[y == i] *= v + + self._svm_sample_weight_check(sample_weight, y, xp) + + onedal_params = { + "C": self.C, + "nu": self.nu, + "kernel": self.kernel, + "degree": self.degree, + "gamma": self._compute_gamma_sigma(X), + "coef0": self.coef0, + "tol": self.tol, + "shrinking": self.shrinking, + "max_iter": self.max_iter, + "cache_size": self.cache_size, + } + + self._onedal_estimator = self._onedal_factory(**onedal_params) + self._onedal_estimator.fit( + X, y, sample_weight, class_count=self.classes_.shape[0], queue=queue + ) + + if self.probability: + self._fit_proba( + X, + y, + sample_weight=sample_weight, + queue=queue, + ) + + self._save_attributes(X, y, xp=xp) + + def _fit_proba(self, X, y, sample_weight=None, queue=None): + # TODO: rewrite this method when probabilities output is implemented in oneDAL + + # LibSVM uses the random seed to control cross-validation for probability generation + # CalibratedClassifierCV with "prefit" does not use an RNG nor a seed. This may + # impact users without their knowledge, so display a warning. + if self.random_state is not None: + warnings.warn( + "random_state does not influence oneDAL SVM results", + RuntimeWarning, + ) + + params = self.get_params() + params["probability"] = False + params["decision_function_shape"] = "ovr" + clf_base = self.__class__(**params) + + # We use stock metaestimators below, so the only way + # to pass a queue is using config_context. + cfg = get_config() + cfg["target_offload"] = queue + with config_context(**cfg): + clf_base.fit(X, y) + + # Forced use of FrozenEstimator starting in sklearn 1.6 + if sklearn_check_version("1.6"): + clf_base = FrozenEstimator(clf_base) + + self.clf_prob = CalibratedClassifierCV( + clf_base, + ensemble=False, + method="sigmoid", + ) + # see custom stopgap solution defined above + _prefit_CalibratedClassifierCV_fit( + self.clf_prob, X, y, sample_weight=sample_weight + ) + else: + + self.clf_prob = CalibratedClassifierCV( + clf_base, + ensemble=False, + cv="prefit", + method="sigmoid", + ).fit(X, y, sample_weight=sample_weight) + + def _save_attributes(self, X, y, xp=np): + # This function requires array API adaptation. + self.support_vectors_ = self._onedal_estimator.support_vectors_ + + self.dual_coef_ = self._onedal_estimator.dual_coef_ + self.support_ = xp.asarray(self._onedal_estimator.support_, dtype=xp.int32) + + self._icept_ = self._onedal_estimator.intercept_ + self._sparse = False + self.fit_status_ = 0 + self.shape_fit_ = X.shape + + self._gamma = self._onedal_estimator.gamma + length = (self.classes_.shape[0] ** 2 - self.classes_.shape[0]) // 2 + + if self.probability: + # Parameter learned in Platt scaling, exposed as probA_ and probB_ + # via the sklearn SVM estimator + self._probA = xp.zeros(length) + self._probB = xp.zeros(length) + else: + self._probA = xp.empty(0) + self._probB = xp.empty(0) + + self._dualcoef_ = self.dual_coef_ + + indices = xp.take(y, self.support_, axis=0) + self._n_support = xp.zeros_like(self.classes_, dtype=xp.int32) + for i in range(self.classes_.shape[0]): + self._n_support[i] = xp.sum( + xp.asarray(indices == i, dtype=xp.int32), dtype=xp.int32 + ) + + if sklearn_check_version("1.1"): + self.n_iter_ = xp.full((length,), self._onedal_estimator.n_iter_) + + def _onedal_predict(self, X, queue=None): + sv = self.support_vectors_ + + xp, _ = get_namespace(X) + + # sklearn conformance >1.0, with array API conversion + # https://github.com/scikit-learn/scikit-learn/pull/21336 + if not self._sparse and sv.size > 0 and xp.sum(self._n_support) != sv.shape[0]: + raise ValueError( + "The internal representation " f"of {self.__class__.__name__} was altered" + ) + + if self.break_ties and self.decision_function_shape == "ovo": + raise ValueError( + "break_ties must be False when " "decision_function_shape is 'ovo'" + ) + + if ( + self.break_ties + and self.decision_function_shape == "ovr" + and self.classes_.shape[0] > 2 + ): + res = xp.argmax(self._onedal_decision_function(X, queue=queue), axis=1) + else: + res = super()._onedal_predict(X, queue=queue, xp=xp) + + # the extensive reshaping here comes from the previous implementation, and + # should be sorted out, as this is inefficient and likely can be reduced + res = xp.asarray(res, dtype=xp.int32) + if self.classes_.shape[0] == 2: + res = xp.reshape(res, (-1,)) + + return xp.reshape(xp.take(xp.asarray(self.classes_), res), (-1,)) + + def _onedal_ovr_decision_function(self, decision_function, n_classes, xp=None): + # This function is legacy from the original implementation and needs + # to be refactored. + + predictions = xp.asarray(decision_function < 0, dtype=decision_function.dtype) + confidences = -decision_function + + if xp is None: + xp, _ = get_namespace(decision_function) + # use `zeros_like` to support correct device allocation while still + # supporting numpy < 1.26 + votes = xp.full_like(decision_function[:, :n_classes], n_classes) + sum_of_confidences = xp.zeros_like(votes) + + # This is extraordinarily bad, as its doing strided access behind + # two python for loops. Its the main math converting an ovo to ovr. + k = 0 + for i in range(n_classes): + votes[:, i] -= i + 1 + for j in range(i + 1, n_classes): + sum_of_confidences[:, i] -= confidences[:, k] + sum_of_confidences[:, j] += confidences[:, k] + votes[:, i] -= predictions[:, k] + votes[:, j] += predictions[:, k] + k += 1 + + transformed_confidences = sum_of_confidences / ( + 3 * (xp.abs(sum_of_confidences) + 1) + ) + return votes + transformed_confidences + + def _onedal_decision_function(self, X, queue=None): + xp, _ = get_namespace(X) + + X = validate_data( + self, + X, + dtype=[xp.float64, xp.float32], + accept_sparse="csr", + reset=False, + ) + + sv = self.support_vectors_ + if not self._sparse and sv.size > 0 and xp.sum(self._n_support) != sv.shape[0]: + raise ValueError( + "The internal representation " f"of {self.__class__.__name__} was altered" + ) + + decision_function = self._onedal_estimator.decision_function(X, queue=queue) + + lencls = self.classes_.shape[0] + if lencls == 2: + decision_function = xp.reshape(decision_function, (-1,)) + elif lencls > 2 and self.decision_function_shape == "ovr": + decision_function = self._onedal_ovr_decision_function( + decision_function, lencls, xp + ) + + return decision_function + + def _onedal_predict_proba(self, X, queue=None): + if not hasattr(self, "clf_prob"): + raise NotFittedError( + "predict_proba is not available when fitted with probability=False" + ) + + # We use stock metaestimators below, so the only way + # to pass a queue is using config_context. + cfg = get_config() + cfg["target_offload"] = queue + with config_context(**cfg): + return self.clf_prob.predict_proba(X) + + def _onedal_score(self, X, y, sample_weight=None, queue=None): + return accuracy_score( + y, self._onedal_predict(X, queue=queue), sample_weight=sample_weight + ) + + @wrap_output_data + def predict(self, X): + check_is_fitted(self) + return dispatch( + self, + "predict", + { + "onedal": self.__class__._onedal_predict, + "sklearn": _sklearn_BaseSVC.predict, + }, + X, + ) + + @wrap_output_data + def score(self, X, y, sample_weight=None): + check_is_fitted(self) + return dispatch( + self, + "score", + { + "onedal": self.__class__._onedal_score, + "sklearn": _sklearn_BaseSVC.score, + }, + X, + y, + sample_weight=sample_weight, + ) + + @wrap_output_data + def decision_function(self, X): + check_is_fitted(self) + return dispatch( + self, + "decision_function", + { + "onedal": self.__class__._onedal_decision_function, + "sklearn": _sklearn_BaseSVC.decision_function, + }, + X, + ) + + @available_if(_sklearn_BaseSVC._check_proba) + @wraps(_sklearn_BaseSVC.predict_proba, assigned=["__doc__"]) + def predict_proba(self, X): + check_is_fitted(self) + return self._predict_proba(X) + + @available_if(_sklearn_BaseSVC._check_proba) + @wraps(_sklearn_BaseSVC.predict_log_proba, assigned=["__doc__"]) + def predict_log_proba(self, X): + xp, _ = get_namespace(X) + + return xp.log(self.predict_proba(X)) + + @wrap_output_data + def _predict_proba(self, X): + return dispatch( + self, + "_predict_proba", + { + "onedal": self.__class__._onedal_predict_proba, + "sklearn": _sklearn_BaseSVC.predict_proba, + }, + X, + ) + + predict.__doc__ = _sklearn_BaseSVC.predict.__doc__ + decision_function.__doc__ = _sklearn_BaseSVC.decision_function.__doc__ + score.__doc__ = _sklearn_BaseSVC.score.__doc__ + + +class BaseSVR(BaseSVM): + + # overwrite _validate_targets for array API support + def _validate_targets(self, y): + xp, is_array_api_compliant = get_namespace(y) + + if not is_array_api_compliant: + return super()._validate_targets(y) + + return xp.astype(column_or_1d(y, warn=True), xp.float64, copy=False) + + def _onedal_fit(self, X, y, sample_weight=None, queue=None): + xp, is_ = get_namespace(X, y, sample_weight) + + X, y = validate_data( + self, + X, + y, + dtype=[xp.float64, xp.float32], + accept_sparse="csr", + ) + + y = self._validate_targets(y) + + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X) + self._svm_sample_weight_check(sample_weight, y, xp) + + onedal_params = { + "C": self.C, + "nu": self.nu, + "epsilon": self.epsilon, + "kernel": self.kernel, + "degree": self.degree, + "gamma": self._compute_gamma_sigma(X), + "coef0": self.coef0, + "tol": self.tol, + "shrinking": self.shrinking, + "cache_size": self.cache_size, + "max_iter": self.max_iter, + } + + self._onedal_estimator = self._onedal_factory(**onedal_params) + self._onedal_estimator.fit(X, y, sample_weight, queue=queue) + self._save_attributes(X, xp=xp) + + def _save_attributes(self, X, xp=None): + self.support_vectors_ = self._onedal_estimator.support_vectors_ + self.fit_status_ = 0 + self.dual_coef_ = self._onedal_estimator.dual_coef_ + self.shape_fit_ = X.shape + self.support_ = xp.asarray(self._onedal_estimator.support_, dtype=xp.int32) + + self._icept_ = self._onedal_estimator.intercept_ + self._n_support = xp.asarray([self.support_vectors_.shape[0]], dtype=xp.int32) + + self._sparse = False + self._gamma = self._onedal_estimator.gamma + self._probA = None + self._probB = None + + if sklearn_check_version("1.1"): + self.n_iter_ = self._onedal_estimator.n_iter_ + + self._dualcoef_ = self.dual_coef_ + + def _onedal_score(self, X, y, sample_weight=None, queue=None): + return r2_score( + y, self._onedal_predict(X, queue=queue), sample_weight=sample_weight + ) + + @wrap_output_data + def predict(self, X): + check_is_fitted(self) + return dispatch( + self, + "predict", + { + "onedal": self.__class__._onedal_predict, + "sklearn": _sklearn_BaseLibSVM.predict, + }, + X, + ) + + @wrap_output_data + def score(self, X, y, sample_weight=None): + check_is_fitted(self) + return dispatch( + self, + "score", + { + "onedal": self.__class__._onedal_score, + "sklearn": RegressorMixin.score, + }, + X, + y, + sample_weight=sample_weight, + ) + + predict.__doc__ = _sklearn_BaseLibSVM.predict.__doc__ + score.__doc__ = RegressorMixin.score.__doc__ diff --git a/sklearnex/svm/_classes.py b/sklearnex/svm/_classes.py new file mode 100644 index 0000000000..682b442630 --- /dev/null +++ b/sklearnex/svm/_classes.py @@ -0,0 +1,404 @@ +# ============================================================================== +# Copyright Contributors to the oneDAL Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import numpy as np +from scipy import sparse as sp +from sklearn.svm import SVC as _sklearn_SVC +from sklearn.svm import SVR as _sklearn_SVR +from sklearn.svm import NuSVC as _sklearn_NuSVC +from sklearn.svm import NuSVR as _sklearn_NuSVR +from sklearn.utils.validation import _deprecate_positional_args + +from daal4py.sklearn._n_jobs_support import control_n_jobs +from daal4py.sklearn._utils import sklearn_check_version +from onedal.svm import SVC as onedal_SVC +from onedal.svm import SVR as onedal_SVR +from onedal.svm import NuSVC as onedal_NuSVC +from onedal.svm import NuSVR as onedal_NuSVR + +from .._device_offload import dispatch +from .._utils import PatchingConditionsChain +from ..utils._array_api import enable_array_api +from ._base import BaseSVC, BaseSVR + +# array API support limited to sklearn 1.5 for regressors due to an incorrect array API +# implementation of `accuracy_score`. Classifiers limited by `_compute_class_weight` to +# sklearn 1.6. + + +@enable_array_api("1.6") +@control_n_jobs( + decorated_methods=["fit", "predict", "_predict_proba", "decision_function", "score"] +) +class SVC(BaseSVC, _sklearn_SVC): + __doc__ = _sklearn_SVC.__doc__ + _onedal_factory = onedal_SVC + + if sklearn_check_version("1.2"): + _parameter_constraints: dict = {**_sklearn_SVC._parameter_constraints} + + @_deprecate_positional_args + def __init__( + self, + *, + C=1.0, + kernel="rbf", + degree=3, + gamma="scale", + coef0=0.0, + shrinking=True, + probability=False, + tol=1e-3, + cache_size=200, + class_weight=None, + verbose=False, + max_iter=-1, + decision_function_shape="ovr", + break_ties=False, + random_state=None, + ): + super().__init__( + C=C, + kernel=kernel, + degree=degree, + gamma=gamma, + coef0=coef0, + shrinking=shrinking, + probability=probability, + tol=tol, + cache_size=cache_size, + class_weight=class_weight, + verbose=verbose, + max_iter=max_iter, + decision_function_shape=decision_function_shape, + break_ties=break_ties, + random_state=random_state, + ) + + def fit(self, X, y, sample_weight=None): + if sklearn_check_version("1.2"): + self._validate_params() + elif self.C <= 0: + # else if added to correct issues with + # sklearn tests: + # svm/tests/test_sparse.py::test_error + # svm/tests/test_svm.py::test_bad_input + # for sklearn versions < 1.2 (i.e. without + # validate_params parameter checking) + # Without this, a segmentation fault with + # Windows fatal exception: access violation + # occurs + raise ValueError("C <= 0") + dispatch( + self, + "fit", + { + "onedal": self.__class__._onedal_fit, + "sklearn": _sklearn_SVC.fit, + }, + X, + y, + sample_weight=sample_weight, + ) + + return self + + def _onedal_gpu_supported(self, method_name, *data): + class_name = self.__class__.__name__ + patching_status = PatchingConditionsChain( + f"sklearn.svm.{class_name}.{method_name}" + ) + if len(data) > 1: + self._class_count = len(np.unique(data[1])) + self._is_sparse = sp.issparse(data[0]) + conditions = [ + ( + self.kernel in ["linear", "rbf"], + f'Kernel is "{self.kernel}" while ' + '"linear" and "rbf" are only supported on GPU.', + ), + (self.class_weight is None, "Class weight is not supported on GPU."), + (not self._is_sparse, "Sparse input is not supported on GPU."), + (self._class_count == 2, "Multiclassification is not supported on GPU."), + ] + if method_name == "fit": + patching_status.and_conditions(conditions) + return patching_status + if method_name in ["predict", "predict_proba", "decision_function", "score"]: + conditions.append( + (hasattr(self, "_onedal_estimator"), "oneDAL model was not trained") + ) + patching_status.and_conditions(conditions) + return patching_status + raise RuntimeError(f"Unknown method {method_name} in {class_name}") + + def _svm_sample_weight_check(self, sample_weight, y, xp): + # This provides SVM estimator differentiation with respect to sample_weight errors + super()._svm_sample_weight_check(sample_weight, y, xp) + # y is an index type vector (integer), where the variance == 0 shows + # that is is constant (i.e) single class. y[sample_weight > 0] should + # never be empty due to the previous check. + if xp.any(sample_weight <= 0) and xp.var(y[sample_weight > 0]) == 0: + raise ValueError( + "Invalid input - all samples with positive weights " + "belong to the same class" + if sklearn_check_version("1.2") + else "Invalid input - all samples with positive weights " + "have the same label." + ) + + fit.__doc__ = _sklearn_SVC.fit.__doc__ + + +@enable_array_api("1.6") +@control_n_jobs( + decorated_methods=["fit", "predict", "_predict_proba", "decision_function", "score"] +) +class NuSVC(BaseSVC, _sklearn_NuSVC): + __doc__ = _sklearn_NuSVC.__doc__ + _onedal_factory = onedal_NuSVC + + if sklearn_check_version("1.2"): + _parameter_constraints: dict = {**_sklearn_NuSVC._parameter_constraints} + + @_deprecate_positional_args + def __init__( + self, + *, + nu=0.5, + kernel="rbf", + degree=3, + gamma="scale", + coef0=0.0, + shrinking=True, + probability=False, + tol=1e-3, + cache_size=200, + class_weight=None, + verbose=False, + max_iter=-1, + decision_function_shape="ovr", + break_ties=False, + random_state=None, + ): + super().__init__( + nu=nu, + kernel=kernel, + degree=degree, + gamma=gamma, + coef0=coef0, + shrinking=shrinking, + probability=probability, + tol=tol, + cache_size=cache_size, + class_weight=class_weight, + verbose=verbose, + max_iter=max_iter, + decision_function_shape=decision_function_shape, + break_ties=break_ties, + random_state=random_state, + ) + + def fit(self, X, y, sample_weight=None): + if sklearn_check_version("1.2"): + self._validate_params() + elif self.nu <= 0 or self.nu > 1: + # else if added to correct issues with + # sklearn tests: + # svm/tests/test_sparse.py::test_error + # svm/tests/test_svm.py::test_bad_input + # for sklearn versions < 1.2 (i.e. without + # validate_params parameter checking) + # Without this, a segmentation fault with + # Windows fatal exception: access violation + # occurs + raise ValueError("nu <= 0 or nu > 1") + dispatch( + self, + "fit", + { + "onedal": self.__class__._onedal_fit, + "sklearn": _sklearn_NuSVC.fit, + }, + X, + y, + sample_weight=sample_weight, + ) + + return self + + def _svm_sample_weight_check(self, sample_weight, y, xp): + # This provides SVM-specific sample_weight conformance checks + if xp.all(sample_weight <= 0): + raise ValueError("negative dimensions are not allowed") + + # y is an index type vector (integer), where the variance == 0 shows + # that is is constant (i.e) single class. y[sample_weight > 0] should + # never be empty due to the previous check. + + # taken from previous implementation, can be improved (try to remove for loops). + weight_per_class = [ + xp.sum(sample_weight[y == class_label]) + for class_label in range(int(xp.max(y)) + 1) + ] + + for i in range(len(weight_per_class)): + for j in range(i + 1, len(weight_per_class)): + if self.nu * (weight_per_class[i] + weight_per_class[j]) / 2 > min( + weight_per_class[i], weight_per_class[j] + ): + raise ValueError("specified nu is infeasible") + + fit.__doc__ = _sklearn_NuSVC.fit.__doc__ + + +@enable_array_api("1.5") +@control_n_jobs(decorated_methods=["fit", "predict", "score"]) +class SVR(BaseSVR, _sklearn_SVR): + __doc__ = _sklearn_SVR.__doc__ + _onedal_factory = onedal_SVR + + if sklearn_check_version("1.2"): + _parameter_constraints: dict = {**_sklearn_SVR._parameter_constraints} + + @_deprecate_positional_args + def __init__( + self, + *, + kernel="rbf", + degree=3, + gamma="scale", + coef0=0.0, + tol=1e-3, + C=1.0, + epsilon=0.1, + shrinking=True, + cache_size=200, + verbose=False, + max_iter=-1, + ): + super().__init__( + kernel=kernel, + degree=degree, + gamma=gamma, + coef0=coef0, + tol=tol, + C=C, + epsilon=epsilon, + shrinking=shrinking, + cache_size=cache_size, + verbose=verbose, + max_iter=max_iter, + ) + + def fit(self, X, y, sample_weight=None): + if sklearn_check_version("1.2"): + self._validate_params() + elif self.C <= 0: + # else if added to correct issues with + # sklearn tests: + # svm/tests/test_sparse.py::test_error + # svm/tests/test_svm.py::test_bad_input + # for sklearn versions < 1.2 (i.e. without + # validate_params parameter checking) + # Without this, a segmentation fault with + # Windows fatal exception: access violation + # occurs + raise ValueError("C <= 0") + dispatch( + self, + "fit", + { + "onedal": self.__class__._onedal_fit, + "sklearn": _sklearn_SVR.fit, + }, + X, + y, + sample_weight=sample_weight, + ) + + return self + + fit.__doc__ = _sklearn_SVR.fit.__doc__ + + +@enable_array_api("1.5") +@control_n_jobs(decorated_methods=["fit", "predict", "score"]) +class NuSVR(BaseSVR, _sklearn_NuSVR): + __doc__ = _sklearn_NuSVR.__doc__ + _onedal_factory = onedal_NuSVR + + if sklearn_check_version("1.2"): + _parameter_constraints: dict = {**_sklearn_NuSVR._parameter_constraints} + + @_deprecate_positional_args + def __init__( + self, + *, + nu=0.5, + C=1.0, + kernel="rbf", + degree=3, + gamma="scale", + coef0=0.0, + shrinking=True, + tol=1e-3, + cache_size=200, + verbose=False, + max_iter=-1, + ): + super().__init__( + kernel=kernel, + degree=degree, + gamma=gamma, + coef0=coef0, + tol=tol, + C=C, + nu=nu, + shrinking=shrinking, + cache_size=cache_size, + verbose=verbose, + max_iter=max_iter, + ) + + def fit(self, X, y, sample_weight=None): + if sklearn_check_version("1.2"): + self._validate_params() + elif self.nu <= 0 or self.nu > 1: + # else if added to correct issues with + # sklearn tests: + # svm/tests/test_sparse.py::test_error + # svm/tests/test_svm.py::test_bad_input + # for sklearn versions < 1.2 (i.e. without + # validate_params parameter checking) + # Without this, a segmentation fault with + # Windows fatal exception: access violation + # occurs + raise ValueError("nu <= 0 or nu > 1") + dispatch( + self, + "fit", + { + "onedal": self.__class__._onedal_fit, + "sklearn": _sklearn_NuSVR.fit, + }, + X, + y, + sample_weight=sample_weight, + ) + return self + + fit.__doc__ = _sklearn_NuSVR.fit.__doc__ diff --git a/sklearnex/svm/_common.py b/sklearnex/svm/_common.py deleted file mode 100644 index adcc7032bb..0000000000 --- a/sklearnex/svm/_common.py +++ /dev/null @@ -1,389 +0,0 @@ -# ============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -import warnings -from numbers import Number, Real - -import numpy as np -from scipy import sparse as sp -from sklearn.base import ClassifierMixin -from sklearn.calibration import CalibratedClassifierCV -from sklearn.metrics import r2_score -from sklearn.preprocessing import LabelEncoder - -from daal4py.sklearn._utils import sklearn_check_version -from daal4py.sklearn.utils.validation import get_requires_y_tag -from onedal.utils.validation import _check_array, _column_or_1d - -from .._config import config_context, get_config -from .._utils import PatchingConditionsChain -from ..base import oneDALEstimator -from ..utils.validation import validate_data - -if sklearn_check_version("1.6"): - from sklearn.calibration import _fit_calibrator - from sklearn.frozen import FrozenEstimator - from sklearn.utils import indexable - from sklearn.utils._response import _get_response_values - from sklearn.utils.multiclass import check_classification_targets - from sklearn.utils.validation import check_is_fitted - - def _prefit_CalibratedClassifierCV_fit(self, X, y, **fit_params): - # This is a stop-gap solution where the cv='prefit' of CalibratedClassifierCV - # was removed and the single fold solution needs to be maintained. Discussion - # of the mathematical and performance implications of this choice can be found - # here: https://github.com/uxlfoundation/scikit-learn-intelex/pull/1879 - # This is distilled from the sklearn CalibratedClassifierCV for sklearn <1.8 for - # use in sklearn > 1.8 to maintain performance. - check_classification_targets(y) - X, y = indexable(X, y) - - estimator = self._get_estimator() - - self.calibrated_classifiers_ = [] - check_is_fitted(self.estimator, attributes=["classes_"]) - self.classes_ = self.estimator.classes_ - - predictions, _ = _get_response_values( - estimator, - X, - response_method=["decision_function", "predict_proba"], - ) - if predictions.ndim == 1: - # Reshape binary output from `(n_samples,)` to `(n_samples, 1)` - predictions = predictions.reshape(-1, 1) - - calibrated_classifier = _fit_calibrator( - estimator, - predictions, - y, - self.classes_, - self.method, - ) - self.calibrated_classifiers_.append(calibrated_classifier) - - first_clf = self.calibrated_classifiers_[0].estimator - if hasattr(first_clf, "n_features_in_"): - self.n_features_in_ = first_clf.n_features_in_ - if hasattr(first_clf, "feature_names_in_"): - self.feature_names_in_ = first_clf.feature_names_in_ - return self - - -class BaseSVM(oneDALEstimator): - - @property - def _dual_coef_(self): - return self._dualcoef_ - - @_dual_coef_.setter - def _dual_coef_(self, value): - self._dualcoef_ = value - if hasattr(self, "_onedal_estimator"): - self._onedal_estimator.dual_coef_ = value - if hasattr(self._onedal_estimator, "_onedal_model"): - del self._onedal_estimator._onedal_model - - @_dual_coef_.deleter - def _dual_coef_(self): - del self._dualcoef_ - - @property - def intercept_(self): - return self._icept_ - - @intercept_.setter - def intercept_(self, value): - self._icept_ = value - if hasattr(self, "_onedal_estimator"): - self._onedal_estimator.intercept_ = value - if hasattr(self._onedal_estimator, "_onedal_model"): - del self._onedal_estimator._onedal_model - - @intercept_.deleter - def intercept_(self): - del self._icept_ - - def _onedal_gpu_supported(self, method_name, *data): - patching_status = PatchingConditionsChain(f"sklearn.{method_name}") - patching_status.and_conditions([(False, "GPU offloading is not supported.")]) - return patching_status - - def _onedal_cpu_supported(self, method_name, *data): - class_name = self.__class__.__name__ - patching_status = PatchingConditionsChain( - f"sklearn.svm.{class_name}.{method_name}" - ) - if method_name == "fit": - patching_status.and_conditions( - [ - ( - self.kernel in ["linear", "rbf", "poly", "sigmoid"], - f'Kernel is "{self.kernel}" while ' - '"linear", "rbf", "poly" and "sigmoid" are only supported.', - ) - ] - ) - return patching_status - inference_methods = ( - ["predict", "score"] - if class_name.endswith("R") - else ["predict", "predict_proba", "decision_function", "score"] - ) - if method_name in inference_methods: - patching_status.and_conditions( - [(hasattr(self, "_onedal_estimator"), "oneDAL model was not trained.")] - ) - return patching_status - raise RuntimeError(f"Unknown method {method_name} in {class_name}") - - def _compute_gamma_sigma(self, X): - # only run extended conversion if kernel is not linear - # set to a value = 1.0, so gamma will always be passed to - # the onedal estimator as a float type - if self.kernel == "linear": - return 1.0 - - if isinstance(self.gamma, str): - if self.gamma == "scale": - if sp.issparse(X): - # var = E[X^2] - E[X]^2 - X_sc = (X.multiply(X)).mean() - (X.mean()) ** 2 - else: - X_sc = X.var() - _gamma = 1.0 / (X.shape[1] * X_sc) if X_sc != 0 else 1.0 - elif self.gamma == "auto": - _gamma = 1.0 / X.shape[1] - else: - raise ValueError( - "When 'gamma' is a string, it should be either 'scale' or " - "'auto'. Got '{}' instead.".format(self.gamma) - ) - else: - if sklearn_check_version("1.1") and not sklearn_check_version("1.2"): - if isinstance(self.gamma, Real): - if self.gamma <= 0: - msg = ( - f"gamma value must be > 0; {self.gamma!r} is invalid. Use" - " a positive number or use 'auto' to set gamma to a" - " value of 1 / n_features." - ) - raise ValueError(msg) - _gamma = self.gamma - else: - msg = ( - "The gamma value should be set to 'scale', 'auto' or a" - f" positive float value. {self.gamma!r} is not a valid option" - ) - raise ValueError(msg) - else: - _gamma = self.gamma - return _gamma - - def _onedal_fit_checks(self, X, y, sample_weight=None): - if hasattr(self, "decision_function_shape"): - if self.decision_function_shape not in ("ovr", "ovo", None): - raise ValueError( - f"decision_function_shape must be either 'ovr' or 'ovo', " - f"got {self.decision_function_shape}." - ) - - if y is None: - if get_requires_y_tag(self): - raise ValueError( - f"This {self.__class__.__name__} estimator " - f"requires y to be passed, but the target y is None." - ) - # finite check occurs in onedal estimator - X, y = validate_data( - self, - X, - y, - dtype=[np.float64, np.float32], - ensure_all_finite=False, - accept_sparse="csr", - ) - y = self._validate_targets(y) - sample_weight = self._get_sample_weight(X, y, sample_weight) - return X, y, sample_weight - - def _get_sample_weight(self, X, y, sample_weight): - n_samples = X.shape[0] - dtype = X.dtype - if n_samples == 1: - raise ValueError("n_samples=1") - - sample_weight = np.ascontiguousarray( - [] if sample_weight is None else sample_weight, dtype=np.float64 - ) - - sample_weight_count = sample_weight.shape[0] - if sample_weight_count != 0 and sample_weight_count != n_samples: - raise ValueError( - "sample_weight and X have incompatible shapes: " - "%r vs %r\n" - "Note: Sparse matrices cannot be indexed w/" - "boolean masks (use `indices=True` in CV)." - % (len(sample_weight), X.shape) - ) - - if sample_weight_count == 0: - if not isinstance(self, ClassifierMixin) or self.class_weight_ is None: - return None - sample_weight = np.ones(n_samples, dtype=dtype) - elif isinstance(sample_weight, Number): - sample_weight = np.full(n_samples, sample_weight, dtype=dtype) - else: - sample_weight = _check_array( - sample_weight, - accept_sparse=False, - ensure_2d=False, - dtype=dtype, - order="C", - ) - if sample_weight.ndim != 1: - raise ValueError("Sample weights must be 1D array or scalar") - - if sample_weight.shape != (n_samples,): - raise ValueError( - "sample_weight.shape == {}, expected {}!".format( - sample_weight.shape, (n_samples,) - ) - ) - - if np.all(sample_weight <= 0): - if "nusvc" in self.__module__: - raise ValueError("negative dimensions are not allowed") - else: - raise ValueError( - "Invalid input - all samples have zero or negative weights." - ) - - return sample_weight - - -class BaseSVC(BaseSVM): - def _compute_balanced_class_weight(self, y): - y_ = _column_or_1d(y) - classes, _ = np.unique(y_, return_inverse=True) - - le = LabelEncoder() - y_ind = le.fit_transform(y_) - if not np.isin(classes, le.classes_).all(): - raise ValueError("classes should have valid labels that are in y") - - recip_freq = len(y_) / (len(le.classes_) * np.bincount(y_ind).astype(np.float64)) - return recip_freq[le.transform(classes)] - - def _fit_proba(self, X, y, sample_weight=None, queue=None): - # TODO: rewrite this method when probabilities output is implemented in oneDAL - - # LibSVM uses the random seed to control cross-validation for probability generation - # CalibratedClassifierCV with "prefit" does not use an RNG nor a seed. This may - # impact users without their knowledge, so display a warning. - if self.random_state is not None: - warnings.warn( - "random_state does not influence oneDAL SVM results", - RuntimeWarning, - ) - - params = self.get_params() - params["probability"] = False - params["decision_function_shape"] = "ovr" - clf_base = self.__class__(**params) - - # We use stock metaestimators below, so the only way - # to pass a queue is using config_context. - cfg = get_config() - cfg["target_offload"] = queue - with config_context(**cfg): - clf_base.fit(X, y) - - # Forced use of FrozenEstimator starting in sklearn 1.6 - if sklearn_check_version("1.6"): - clf_base = FrozenEstimator(clf_base) - - self.clf_prob = CalibratedClassifierCV( - clf_base, - ensemble=False, - method="sigmoid", - ) - # see custom stopgap solution defined above - _prefit_CalibratedClassifierCV_fit(self.clf_prob, X, y) - else: - - self.clf_prob = CalibratedClassifierCV( - clf_base, - ensemble=False, - cv="prefit", - method="sigmoid", - ).fit(X, y) - - def _save_attributes(self): - self.support_vectors_ = self._onedal_estimator.support_vectors_ - self.n_features_in_ = self._onedal_estimator.n_features_in_ - self.fit_status_ = 0 - self.dual_coef_ = self._onedal_estimator.dual_coef_ - self.shape_fit_ = self._onedal_estimator.class_weight_ - self.classes_ = self._onedal_estimator.classes_ - if isinstance(self, ClassifierMixin) or not sklearn_check_version("1.2"): - self.class_weight_ = self._onedal_estimator.class_weight_ - self.support_ = self._onedal_estimator.support_ - - self._icept_ = self._onedal_estimator.intercept_ - self._n_support = self._onedal_estimator._n_support - self._sparse = False - self._gamma = self._onedal_estimator._gamma - if self.probability: - length = int(len(self.classes_) * (len(self.classes_) - 1) / 2) - self._probA = np.zeros(length) - self._probB = np.zeros(length) - else: - self._probA = np.empty(0) - self._probB = np.empty(0) - - self._dualcoef_ = self.dual_coef_ - - if sklearn_check_version("1.1"): - length = int(len(self.classes_) * (len(self.classes_) - 1) / 2) - self.n_iter_ = np.full((length,), self._onedal_estimator.n_iter_) - - -class BaseSVR(BaseSVM): - def _save_attributes(self): - self.support_vectors_ = self._onedal_estimator.support_vectors_ - self.n_features_in_ = self._onedal_estimator.n_features_in_ - self.fit_status_ = 0 - self.dual_coef_ = self._onedal_estimator.dual_coef_ - self.shape_fit_ = self._onedal_estimator.shape_fit_ - self.support_ = self._onedal_estimator.support_ - - self._icept_ = self._onedal_estimator.intercept_ - self._n_support = [self.support_vectors_.shape[0]] - self._sparse = False - self._gamma = self._onedal_estimator._gamma - self._probA = None - self._probB = None - - if sklearn_check_version("1.1"): - self.n_iter_ = self._onedal_estimator.n_iter_ - - self._dualcoef_ = self.dual_coef_ - - def _onedal_score(self, X, y, sample_weight=None, queue=None): - return r2_score( - y, self._onedal_predict(X, queue=queue), sample_weight=sample_weight - ) diff --git a/sklearnex/svm/nusvc.py b/sklearnex/svm/nusvc.py deleted file mode 100644 index 6065af3123..0000000000 --- a/sklearnex/svm/nusvc.py +++ /dev/null @@ -1,278 +0,0 @@ -# ============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from functools import wraps - -import numpy as np -from sklearn.exceptions import NotFittedError -from sklearn.metrics import accuracy_score -from sklearn.svm import NuSVC as _sklearn_NuSVC -from sklearn.utils.metaestimators import available_if -from sklearn.utils.validation import ( - _deprecate_positional_args, - check_array, - check_is_fitted, -) - -from daal4py.sklearn._n_jobs_support import control_n_jobs -from daal4py.sklearn._utils import sklearn_check_version -from onedal.svm import NuSVC as onedal_NuSVC - -from .._device_offload import dispatch, wrap_output_data -from ..utils._array_api import get_namespace -from ..utils.validation import validate_data -from ._common import BaseSVC - - -@control_n_jobs( - decorated_methods=["fit", "predict", "_predict_proba", "decision_function", "score"] -) -class NuSVC(BaseSVC, _sklearn_NuSVC): - __doc__ = _sklearn_NuSVC.__doc__ - - if sklearn_check_version("1.2"): - _parameter_constraints: dict = {**_sklearn_NuSVC._parameter_constraints} - - @_deprecate_positional_args - def __init__( - self, - *, - nu=0.5, - kernel="rbf", - degree=3, - gamma="scale", - coef0=0.0, - shrinking=True, - probability=False, - tol=1e-3, - cache_size=200, - class_weight=None, - verbose=False, - max_iter=-1, - decision_function_shape="ovr", - break_ties=False, - random_state=None, - ): - super().__init__( - nu=nu, - kernel=kernel, - degree=degree, - gamma=gamma, - coef0=coef0, - shrinking=shrinking, - probability=probability, - tol=tol, - cache_size=cache_size, - class_weight=class_weight, - verbose=verbose, - max_iter=max_iter, - decision_function_shape=decision_function_shape, - break_ties=break_ties, - random_state=random_state, - ) - - def fit(self, X, y, sample_weight=None): - if sklearn_check_version("1.2"): - self._validate_params() - elif self.nu <= 0 or self.nu > 1: - # else if added to correct issues with - # sklearn tests: - # svm/tests/test_sparse.py::test_error - # svm/tests/test_svm.py::test_bad_input - # for sklearn versions < 1.2 (i.e. without - # validate_params parameter checking) - # Without this, a segmentation fault with - # Windows fatal exception: access violation - # occurs - raise ValueError("nu <= 0 or nu > 1") - dispatch( - self, - "fit", - { - "onedal": self.__class__._onedal_fit, - "sklearn": _sklearn_NuSVC.fit, - }, - X, - y, - sample_weight=sample_weight, - ) - - return self - - @wrap_output_data - def predict(self, X): - check_is_fitted(self) - return dispatch( - self, - "predict", - { - "onedal": self.__class__._onedal_predict, - "sklearn": _sklearn_NuSVC.predict, - }, - X, - ) - - @wrap_output_data - def score(self, X, y, sample_weight=None): - check_is_fitted(self) - return dispatch( - self, - "score", - { - "onedal": self.__class__._onedal_score, - "sklearn": _sklearn_NuSVC.score, - }, - X, - y, - sample_weight=sample_weight, - ) - - @available_if(_sklearn_NuSVC._check_proba) - @wraps(_sklearn_NuSVC.predict_proba, assigned=["__doc__"]) - def predict_proba(self, X): - check_is_fitted(self) - return self._predict_proba(X) - - @available_if(_sklearn_NuSVC._check_proba) - @wraps(_sklearn_NuSVC.predict_log_proba, assigned=["__doc__"]) - def predict_log_proba(self, X): - xp, _ = get_namespace(X) - - return xp.log(self.predict_proba(X)) - - @wrap_output_data - def _predict_proba(self, X): - return dispatch( - self, - "predict_proba", - { - "onedal": self.__class__._onedal_predict_proba, - "sklearn": _sklearn_NuSVC.predict_proba, - }, - X, - ) - - @wrap_output_data - def decision_function(self, X): - check_is_fitted(self) - return dispatch( - self, - "decision_function", - { - "onedal": self.__class__._onedal_decision_function, - "sklearn": _sklearn_NuSVC.decision_function, - }, - X, - ) - - decision_function.__doc__ = _sklearn_NuSVC.decision_function.__doc__ - - def _get_sample_weight(self, X, y, sample_weight=None): - sample_weight = super()._get_sample_weight(X, y, sample_weight) - if sample_weight is None: - return sample_weight - - weight_per_class = [ - np.sum(sample_weight[y == class_label]) for class_label in np.unique(y) - ] - - for i in range(len(weight_per_class)): - for j in range(i + 1, len(weight_per_class)): - if self.nu * (weight_per_class[i] + weight_per_class[j]) / 2 > min( - weight_per_class[i], weight_per_class[j] - ): - raise ValueError("specified nu is infeasible") - - return sample_weight - - def _onedal_fit(self, X, y, sample_weight=None, queue=None): - X, _, weights = self._onedal_fit_checks(X, y, sample_weight) - onedal_params = { - "nu": self.nu, - "kernel": self.kernel, - "degree": self.degree, - "gamma": self._compute_gamma_sigma(X), - "coef0": self.coef0, - "tol": self.tol, - "shrinking": self.shrinking, - "cache_size": self.cache_size, - "max_iter": self.max_iter, - "class_weight": self.class_weight, - "break_ties": self.break_ties, - "decision_function_shape": self.decision_function_shape, - } - - self._onedal_estimator = onedal_NuSVC(**onedal_params) - self._onedal_estimator.fit(X, y, weights, queue=queue) - - if self.probability: - self._fit_proba( - X, - y, - sample_weight=sample_weight, - queue=queue, - ) - - self._save_attributes() - - def _onedal_predict(self, X, queue=None): - validate_data( - self, - X, - dtype=[np.float64, np.float32], - ensure_all_finite=False, - ensure_2d=False, - accept_sparse="csr", - reset=False, - ) - - return self._onedal_estimator.predict(X, queue=queue) - - def _onedal_predict_proba(self, X, queue=None): - if getattr(self, "clf_prob", None) is None: - raise NotFittedError( - "predict_proba is not available when fitted with probability=False" - ) - from .._config import config_context, get_config - - # We use stock metaestimators below, so the only way - # to pass a queue is using config_context. - cfg = get_config() - cfg["target_offload"] = queue - with config_context(**cfg): - return self.clf_prob.predict_proba(X) - - def _onedal_decision_function(self, X, queue=None): - validate_data( - self, - X, - dtype=[np.float64, np.float32], - ensure_all_finite=False, - accept_sparse="csr", - reset=False, - ) - - return self._onedal_estimator.decision_function(X, queue=queue) - - def _onedal_score(self, X, y, sample_weight=None, queue=None): - return accuracy_score( - y, self._onedal_predict(X, queue=queue), sample_weight=sample_weight - ) - - fit.__doc__ = _sklearn_NuSVC.fit.__doc__ - predict.__doc__ = _sklearn_NuSVC.predict.__doc__ - decision_function.__doc__ = _sklearn_NuSVC.decision_function.__doc__ - score.__doc__ = _sklearn_NuSVC.score.__doc__ diff --git a/sklearnex/svm/nusvr.py b/sklearnex/svm/nusvr.py deleted file mode 100644 index ac50f25865..0000000000 --- a/sklearnex/svm/nusvr.py +++ /dev/null @@ -1,158 +0,0 @@ -# ============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -import numpy as np -from sklearn.svm import NuSVR as _sklearn_NuSVR -from sklearn.utils.validation import ( - _deprecate_positional_args, - check_array, - check_is_fitted, -) - -from daal4py.sklearn._n_jobs_support import control_n_jobs -from daal4py.sklearn._utils import sklearn_check_version -from onedal.svm import NuSVR as onedal_NuSVR - -from .._device_offload import dispatch, wrap_output_data -from ..utils.validation import validate_data -from ._common import BaseSVR - - -@control_n_jobs(decorated_methods=["fit", "predict", "score"]) -class NuSVR(BaseSVR, _sklearn_NuSVR): - __doc__ = _sklearn_NuSVR.__doc__ - - if sklearn_check_version("1.2"): - _parameter_constraints: dict = {**_sklearn_NuSVR._parameter_constraints} - - @_deprecate_positional_args - def __init__( - self, - *, - nu=0.5, - C=1.0, - kernel="rbf", - degree=3, - gamma="scale", - coef0=0.0, - shrinking=True, - tol=1e-3, - cache_size=200, - verbose=False, - max_iter=-1, - ): - super().__init__( - kernel=kernel, - degree=degree, - gamma=gamma, - coef0=coef0, - tol=tol, - C=C, - nu=nu, - shrinking=shrinking, - cache_size=cache_size, - verbose=verbose, - max_iter=max_iter, - ) - - def fit(self, X, y, sample_weight=None): - if sklearn_check_version("1.2"): - self._validate_params() - elif self.nu <= 0 or self.nu > 1: - # else if added to correct issues with - # sklearn tests: - # svm/tests/test_sparse.py::test_error - # svm/tests/test_svm.py::test_bad_input - # for sklearn versions < 1.2 (i.e. without - # validate_params parameter checking) - # Without this, a segmentation fault with - # Windows fatal exception: access violation - # occurs - raise ValueError("nu <= 0 or nu > 1") - dispatch( - self, - "fit", - { - "onedal": self.__class__._onedal_fit, - "sklearn": _sklearn_NuSVR.fit, - }, - X, - y, - sample_weight=sample_weight, - ) - return self - - @wrap_output_data - def predict(self, X): - check_is_fitted(self) - return dispatch( - self, - "predict", - { - "onedal": self.__class__._onedal_predict, - "sklearn": _sklearn_NuSVR.predict, - }, - X, - ) - - @wrap_output_data - def score(self, X, y, sample_weight=None): - check_is_fitted(self) - return dispatch( - self, - "score", - { - "onedal": self.__class__._onedal_score, - "sklearn": _sklearn_NuSVR.score, - }, - X, - y, - sample_weight=sample_weight, - ) - - def _onedal_fit(self, X, y, sample_weight=None, queue=None): - X, _, sample_weight = self._onedal_fit_checks(X, y, sample_weight) - onedal_params = { - "C": self.C, - "nu": self.nu, - "kernel": self.kernel, - "degree": self.degree, - "gamma": self._compute_gamma_sigma(X), - "coef0": self.coef0, - "tol": self.tol, - "shrinking": self.shrinking, - "cache_size": self.cache_size, - "max_iter": self.max_iter, - } - - self._onedal_estimator = onedal_NuSVR(**onedal_params) - self._onedal_estimator.fit(X, y, sample_weight, queue=queue) - self._save_attributes() - - def _onedal_predict(self, X, queue=None): - X = validate_data( - self, - X, - dtype=[np.float64, np.float32], - ensure_all_finite=False, - accept_sparse="csr", - reset=False, - ) - return self._onedal_estimator.predict(X, queue=queue) - - fit.__doc__ = _sklearn_NuSVR.fit.__doc__ - predict.__doc__ = _sklearn_NuSVR.predict.__doc__ - score.__doc__ = _sklearn_NuSVR.score.__doc__ diff --git a/sklearnex/svm/svc.py b/sklearnex/svm/svc.py deleted file mode 100644 index d0f8cc6127..0000000000 --- a/sklearnex/svm/svc.py +++ /dev/null @@ -1,306 +0,0 @@ -# ============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from functools import wraps - -import numpy as np -from scipy import sparse as sp -from sklearn.exceptions import NotFittedError -from sklearn.metrics import accuracy_score -from sklearn.svm import SVC as _sklearn_SVC -from sklearn.utils.metaestimators import available_if -from sklearn.utils.validation import ( - _deprecate_positional_args, - check_array, - check_is_fitted, -) - -from daal4py.sklearn._n_jobs_support import control_n_jobs -from daal4py.sklearn._utils import sklearn_check_version -from onedal.svm import SVC as onedal_SVC - -from .._device_offload import dispatch, wrap_output_data -from .._utils import PatchingConditionsChain -from ..utils._array_api import get_namespace -from ..utils.validation import validate_data -from ._common import BaseSVC - - -@control_n_jobs( - decorated_methods=["fit", "predict", "_predict_proba", "decision_function", "score"] -) -class SVC(BaseSVC, _sklearn_SVC): - __doc__ = _sklearn_SVC.__doc__ - - if sklearn_check_version("1.2"): - _parameter_constraints: dict = {**_sklearn_SVC._parameter_constraints} - - @_deprecate_positional_args - def __init__( - self, - *, - C=1.0, - kernel="rbf", - degree=3, - gamma="scale", - coef0=0.0, - shrinking=True, - probability=False, - tol=1e-3, - cache_size=200, - class_weight=None, - verbose=False, - max_iter=-1, - decision_function_shape="ovr", - break_ties=False, - random_state=None, - ): - super().__init__( - C=C, - kernel=kernel, - degree=degree, - gamma=gamma, - coef0=coef0, - shrinking=shrinking, - probability=probability, - tol=tol, - cache_size=cache_size, - class_weight=class_weight, - verbose=verbose, - max_iter=max_iter, - decision_function_shape=decision_function_shape, - break_ties=break_ties, - random_state=random_state, - ) - - def fit(self, X, y, sample_weight=None): - if sklearn_check_version("1.2"): - self._validate_params() - elif self.C <= 0: - # else if added to correct issues with - # sklearn tests: - # svm/tests/test_sparse.py::test_error - # svm/tests/test_svm.py::test_bad_input - # for sklearn versions < 1.2 (i.e. without - # validate_params parameter checking) - # Without this, a segmentation fault with - # Windows fatal exception: access violation - # occurs - raise ValueError("C <= 0") - dispatch( - self, - "fit", - { - "onedal": self.__class__._onedal_fit, - "sklearn": _sklearn_SVC.fit, - }, - X, - y, - sample_weight=sample_weight, - ) - - return self - - @wrap_output_data - def predict(self, X): - check_is_fitted(self) - return dispatch( - self, - "predict", - { - "onedal": self.__class__._onedal_predict, - "sklearn": _sklearn_SVC.predict, - }, - X, - ) - - @wrap_output_data - def score(self, X, y, sample_weight=None): - check_is_fitted(self) - return dispatch( - self, - "score", - { - "onedal": self.__class__._onedal_score, - "sklearn": _sklearn_SVC.score, - }, - X, - y, - sample_weight=sample_weight, - ) - - @available_if(_sklearn_SVC._check_proba) - @wraps(_sklearn_SVC.predict_proba, assigned=["__doc__"]) - def predict_proba(self, X): - check_is_fitted(self) - return self._predict_proba(X) - - @available_if(_sklearn_SVC._check_proba) - @wraps(_sklearn_SVC.predict_log_proba, assigned=["__doc__"]) - def predict_log_proba(self, X): - xp, _ = get_namespace(X) - - return xp.log(self.predict_proba(X)) - - @wrap_output_data - def _predict_proba(self, X): - return dispatch( - self, - "predict_proba", - { - "onedal": self.__class__._onedal_predict_proba, - "sklearn": _sklearn_SVC.predict_proba, - }, - X, - ) - - @wrap_output_data - def decision_function(self, X): - check_is_fitted(self) - return dispatch( - self, - "decision_function", - { - "onedal": self.__class__._onedal_decision_function, - "sklearn": _sklearn_SVC.decision_function, - }, - X, - ) - - decision_function.__doc__ = _sklearn_SVC.decision_function.__doc__ - - def _onedal_gpu_supported(self, method_name, *data): - class_name = self.__class__.__name__ - patching_status = PatchingConditionsChain( - f"sklearn.svm.{class_name}.{method_name}" - ) - if len(data) > 1: - self._class_count = len(np.unique(data[1])) - self._is_sparse = sp.issparse(data[0]) - conditions = [ - ( - self.kernel in ["linear", "rbf"], - f'Kernel is "{self.kernel}" while ' - '"linear" and "rbf" are only supported on GPU.', - ), - (self.class_weight is None, "Class weight is not supported on GPU."), - (not self._is_sparse, "Sparse input is not supported on GPU."), - (self._class_count == 2, "Multiclassification is not supported on GPU."), - ] - if method_name == "fit": - patching_status.and_conditions(conditions) - return patching_status - if method_name in ["predict", "predict_proba", "decision_function", "score"]: - conditions.append( - (hasattr(self, "_onedal_estimator"), "oneDAL model was not trained") - ) - patching_status.and_conditions(conditions) - return patching_status - raise RuntimeError(f"Unknown method {method_name} in {class_name}") - - def _get_sample_weight(self, X, y, sample_weight=None): - sample_weight = super()._get_sample_weight(X, y, sample_weight) - if sample_weight is None: - return sample_weight - - if np.any(sample_weight <= 0) and len(np.unique(y[sample_weight > 0])) != len( - self.classes_ - ): - raise ValueError( - "Invalid input - all samples with positive weights " - "belong to the same class" - if sklearn_check_version("1.2") - else "Invalid input - all samples with positive weights " - "have the same label." - ) - return sample_weight - - def _onedal_fit(self, X, y, sample_weight=None, queue=None): - X, _, weights = self._onedal_fit_checks(X, y, sample_weight) - onedal_params = { - "C": self.C, - "kernel": self.kernel, - "degree": self.degree, - "gamma": self._compute_gamma_sigma(X), - "coef0": self.coef0, - "tol": self.tol, - "shrinking": self.shrinking, - "cache_size": self.cache_size, - "max_iter": self.max_iter, - "class_weight": self.class_weight, - "break_ties": self.break_ties, - "decision_function_shape": self.decision_function_shape, - } - - self._onedal_estimator = onedal_SVC(**onedal_params) - self._onedal_estimator.fit(X, y, weights, queue=queue) - - if self.probability: - self._fit_proba( - X, - y, - sample_weight=sample_weight, - queue=queue, - ) - - self._save_attributes() - - def _onedal_predict(self, X, queue=None): - X = validate_data( - self, - X, - dtype=[np.float64, np.float32], - ensure_all_finite=False, - ensure_2d=False, - accept_sparse="csr", - reset=False, - ) - return self._onedal_estimator.predict(X, queue=queue) - - def _onedal_predict_proba(self, X, queue=None): - if getattr(self, "clf_prob", None) is None: - raise NotFittedError( - "predict_proba is not available when fitted with probability=False" - ) - from .._config import config_context, get_config - - # We use stock metaestimators below, so the only way - # to pass a queue is using config_context. - cfg = get_config() - cfg["target_offload"] = queue - with config_context(**cfg): - return self.clf_prob.predict_proba(X) - - def _onedal_decision_function(self, X, queue=None): - X = validate_data( - self, - X, - dtype=[np.float64, np.float32], - ensure_all_finite=False, - accept_sparse="csr", - reset=False, - ) - return self._onedal_estimator.decision_function(X, queue=queue) - - def _onedal_score(self, X, y, sample_weight=None, queue=None): - return accuracy_score( - y, self._onedal_predict(X, queue=queue), sample_weight=sample_weight - ) - - fit.__doc__ = _sklearn_SVC.fit.__doc__ - predict.__doc__ = _sklearn_SVC.predict.__doc__ - decision_function.__doc__ = _sklearn_SVC.decision_function.__doc__ - score.__doc__ = _sklearn_SVC.score.__doc__ diff --git a/sklearnex/svm/svr.py b/sklearnex/svm/svr.py deleted file mode 100644 index d0c96e366a..0000000000 --- a/sklearnex/svm/svr.py +++ /dev/null @@ -1,155 +0,0 @@ -# ============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -import numpy as np -from sklearn.svm import SVR as _sklearn_SVR -from sklearn.utils.validation import _deprecate_positional_args, check_is_fitted - -from daal4py.sklearn._n_jobs_support import control_n_jobs -from daal4py.sklearn._utils import sklearn_check_version -from onedal.svm import SVR as onedal_SVR - -from .._device_offload import dispatch, wrap_output_data -from ..utils.validation import validate_data -from ._common import BaseSVR - - -@control_n_jobs(decorated_methods=["fit", "predict", "score"]) -class SVR(BaseSVR, _sklearn_SVR): - __doc__ = _sklearn_SVR.__doc__ - - if sklearn_check_version("1.2"): - _parameter_constraints: dict = {**_sklearn_SVR._parameter_constraints} - - @_deprecate_positional_args - def __init__( - self, - *, - kernel="rbf", - degree=3, - gamma="scale", - coef0=0.0, - tol=1e-3, - C=1.0, - epsilon=0.1, - shrinking=True, - cache_size=200, - verbose=False, - max_iter=-1, - ): - super().__init__( - kernel=kernel, - degree=degree, - gamma=gamma, - coef0=coef0, - tol=tol, - C=C, - epsilon=epsilon, - shrinking=shrinking, - cache_size=cache_size, - verbose=verbose, - max_iter=max_iter, - ) - - def fit(self, X, y, sample_weight=None): - if sklearn_check_version("1.2"): - self._validate_params() - elif self.C <= 0: - # else if added to correct issues with - # sklearn tests: - # svm/tests/test_sparse.py::test_error - # svm/tests/test_svm.py::test_bad_input - # for sklearn versions < 1.2 (i.e. without - # validate_params parameter checking) - # Without this, a segmentation fault with - # Windows fatal exception: access violation - # occurs - raise ValueError("C <= 0") - dispatch( - self, - "fit", - { - "onedal": self.__class__._onedal_fit, - "sklearn": _sklearn_SVR.fit, - }, - X, - y, - sample_weight=sample_weight, - ) - - return self - - @wrap_output_data - def predict(self, X): - check_is_fitted(self) - return dispatch( - self, - "predict", - { - "onedal": self.__class__._onedal_predict, - "sklearn": _sklearn_SVR.predict, - }, - X, - ) - - @wrap_output_data - def score(self, X, y, sample_weight=None): - check_is_fitted(self) - return dispatch( - self, - "score", - { - "onedal": self.__class__._onedal_score, - "sklearn": _sklearn_SVR.score, - }, - X, - y, - sample_weight=sample_weight, - ) - - def _onedal_fit(self, X, y, sample_weight=None, queue=None): - X, _, sample_weight = self._onedal_fit_checks(X, y, sample_weight) - onedal_params = { - "C": self.C, - "epsilon": self.epsilon, - "kernel": self.kernel, - "degree": self.degree, - "gamma": self._compute_gamma_sigma(X), - "coef0": self.coef0, - "tol": self.tol, - "shrinking": self.shrinking, - "cache_size": self.cache_size, - "max_iter": self.max_iter, - } - - self._onedal_estimator = onedal_SVR(**onedal_params) - self._onedal_estimator.fit(X, y, sample_weight, queue=queue) - self._save_attributes() - - def _onedal_predict(self, X, queue=None): - X = validate_data( - self, - X, - dtype=[np.float64, np.float32], - ensure_all_finite=False, - accept_sparse="csr", - reset=False, - ) - return self._onedal_estimator.predict(X, queue=queue) - - fit.__doc__ = _sklearn_SVR.fit.__doc__ - predict.__doc__ = _sklearn_SVR.predict.__doc__ - score.__doc__ = _sklearn_SVR.score.__doc__ diff --git a/sklearnex/svm/tests/test_svm.py b/sklearnex/svm/tests/test_svm.py index 57cb6bdc13..b1144fdcd3 100755 --- a/sklearnex/svm/tests/test_svm.py +++ b/sklearnex/svm/tests/test_svm.py @@ -16,7 +16,12 @@ import numpy as np import pytest -from numpy.testing import assert_allclose +import scipy.sparse as sp +from numpy.testing import assert_allclose, assert_array_almost_equal +from sklearn.datasets import load_diabetes, load_iris, make_classification + +from onedal.svm.tests.test_csr_svm import check_svm_model_equal +from sklearnex import config_context try: from scipy.sparse import csr_array as csr_class @@ -28,6 +33,10 @@ _convert_to_dataframe, get_dataframes_and_queues, ) +from onedal.tests.utils._device_selection import ( + get_queues, + pass_if_not_implemented_for_gpu, +) @pytest.mark.parametrize("dataframe,queue", get_dataframes_and_queues()) @@ -98,6 +107,80 @@ def test_sklearnex_import_nusvr(dataframe, queue): assert_allclose(_as_numpy(svc.support_), [1, 2, 3, 5]) +@pass_if_not_implemented_for_gpu(reason="csr svm is not implemented") +@pytest.mark.parametrize( + "queue", + get_queues("cpu") + + [ + pytest.param( + get_queues("gpu"), + marks=pytest.mark.xfail( + reason="raises UnknownError for linear and rbf, " + "Unimplemented error with inconsistent error message " + "for poly and sigmoid" + ), + ) + ], +) +@pytest.mark.parametrize("kernel", ["linear", "rbf", "poly", "sigmoid"]) +def test_binary_dataset(queue, kernel): + from sklearnex import config_context + from sklearnex.svm import SVC + + X, y = make_classification(n_samples=80, n_features=20, n_classes=2, random_state=0) + sparse_X = sp.csr_matrix(X) + + dataset = sparse_X, y, sparse_X + with config_context(target_offload=queue): + clf0 = SVC(kernel=kernel) + clf1 = SVC(kernel=kernel) + check_svm_model_equal(queue, clf0, clf1, *dataset) + + +@pass_if_not_implemented_for_gpu(reason="csr svm is not implemented") +@pytest.mark.parametrize("queue", get_queues()) +@pytest.mark.parametrize("kernel", ["linear", "rbf", "poly", "sigmoid"]) +def test_iris(queue, kernel): + from sklearnex import config_context + from sklearnex.svm import SVC + + if kernel == "rbf": + pytest.skip("RBF CSR SVM test failing in 2025.0.") + iris = load_iris() + rng = np.random.RandomState(0) + perm = rng.permutation(iris.target.size) + iris.data = iris.data[perm] + iris.target = iris.target[perm] + sparse_iris_data = sp.csr_matrix(iris.data) + + dataset = sparse_iris_data, iris.target, sparse_iris_data + + with config_context(target_offload=queue): + clf0 = SVC(kernel=kernel) + clf1 = SVC(kernel=kernel) + check_svm_model_equal(queue, clf0, clf1, *dataset, decimal=2) + + +@pass_if_not_implemented_for_gpu(reason="csr svm is not implemented") +@pytest.mark.parametrize("queue", get_queues()) +@pytest.mark.parametrize("kernel", ["linear", "rbf", "poly", "sigmoid"]) +def test_diabetes(queue, kernel): + from sklearnex import config_context + from sklearnex.svm import SVR + + if kernel == "sigmoid": + pytest.skip("Sparse sigmoid kernel function is buggy.") + diabetes = load_diabetes() + + sparse_diabetes_data = sp.csr_matrix(diabetes.data) + dataset = sparse_diabetes_data, diabetes.target, sparse_diabetes_data + + with config_context(target_offload=queue): + clf0 = SVR(kernel=kernel, C=0.1) + clf1 = SVR(kernel=kernel, C=0.1) + check_svm_model_equal(queue, clf0, clf1, *dataset) + + # https://github.com/uxlfoundation/scikit-learn-intelex/issues/1880 def test_works_with_unsorted_indices(): from sklearnex.svm import SVC @@ -122,3 +205,31 @@ def test_works_with_unsorted_indices(): pred_single.reshape(-1), pred_multi.reshape(-1), ) + + +@pass_if_not_implemented_for_gpu(reason="class weights are not implemented") +@pytest.mark.parametrize( + "queue", + get_queues("cpu") + + [ + pytest.param( + get_queues("gpu"), + marks=pytest.mark.xfail( + reason="class weights are not implemented but the error is not raised" + ), + ) + ], +) +def test_class_weight(queue): + from sklearnex.svm import SVC, NuSVC + + for estimator in [SVC, NuSVC]: + X = np.array( + [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]], dtype=np.float64 + ) + y = np.array([0, 0, 0, 1, 1, 1], dtype=np.float64) + + clf = estimator(class_weight={0: 0.1}) + with config_context(target_offload=queue): + clf.fit(X, y) + assert_array_almost_equal(clf.predict(X).ravel(), [1] * 6) diff --git a/sklearnex/tests/test_common.py b/sklearnex/tests/test_common.py index d01597344d..188d840f02 100644 --- a/sklearnex/tests/test_common.py +++ b/sklearnex/tests/test_common.py @@ -47,8 +47,7 @@ "_config.py", "_device_offload.py", "test", - "svc.py", - "svm" + os.sep + "_common.py", + "svm" + os.sep + "_base.py", ] _DESIGN_RULE_VIOLATIONS = { diff --git a/sklearnex/utils/class_weight.py b/sklearnex/utils/class_weight.py index 2b46cf33d5..df5911b23d 100644 --- a/sklearnex/utils/class_weight.py +++ b/sklearnex/utils/class_weight.py @@ -49,7 +49,9 @@ def _compute_class_weight(class_weight, *, classes, y, sample_weight=None): raise ValueError("classes should include all valid labels that can be in y") if class_weight is None or len(class_weight) == 0: # uniform class weights - weight = xp.ones((classes.shape[0],), dtype=xp.float64, device=classes.device) + weight = xp.ones( + (classes.shape[0],), dtype=xp.float64, device=getattr(classes, "device", None) + ) elif class_weight == "balanced": if not sklearn_check_version("1.6"): raise RuntimeError( @@ -70,7 +72,9 @@ def _compute_class_weight(class_weight, *, classes, y, sample_weight=None): # then core logic of bincount is replicated: # https://github.com/numpy/numpy/blob/main/numpy/_core/src/multiarray/compiled_base.c weighted_class_counts = xp.zeros( - (xp.max(y_ind) + 1,), dtype=sample_weight.dtype, device=y.device + (xp.max(y_ind) + 1,), + dtype=sample_weight.dtype, + device=getattr(y, "device", None), ) # use a more GPU-friendly summation approach for collecting weighted_class_counts @@ -84,7 +88,9 @@ def _compute_class_weight(class_weight, *, classes, y, sample_weight=None): weight = xp.take(recip_freq, le.transform(classes)) else: # user-defined dictionary - weight = xp.ones((classes.shape[0],), dtype=xp.float64, device=classes.device) + weight = xp.ones( + (classes.shape[0],), dtype=xp.float64, device=getattr(classes, "device", None) + ) unweighted_classes = [] for i, c in enumerate(classes): if (fc := float(c)) in class_weight: