Fix OOB scores in preview Random Forest; preview algorithms fixes (#1209)

Author: Alexsandruss

onedal/datatypes/_data_conversion.py

@@ -14,12 +14,14 @@
 # limitations under the License.
 # ===============================================================================
 
+import numpy as np
 import warnings
 import numpy as np
 
 from onedal import _is_dpc_backend
 from onedal import _backend
 from daal4py.sklearn._utils import make2d
+from onedal import _is_dpc_backend
 
 try:
     import dpctl

onedal/decomposition/pca.py

@@ -49,6 +49,10 @@ def fit(self, X, y, queue):
 
         policy = _get_policy(queue, X, y)
 
+        # TODO: investigate why np.ndarray with OWNDATA=FALSE flag
+        # fails to be converted to oneDAL table
+        if isinstance(X, np.ndarray) and not X.flags['OWNDATA']:
+            X = X.copy()
         X, y = _convert_to_supported(policy, X, y)
         params = self.get_onedal_params(X)
         cov_result = _backend.covariance.compute(

onedal/ensemble/forest.cpp

@@ -17,6 +17,7 @@
 #include "oneapi/dal/algo/decision_forest.hpp"
 
 #include "onedal/common.hpp"
+#include "onedal/version.hpp"
 
 namespace py = pybind11;
 
@@ -73,6 +74,16 @@ auto get_error_metric_mode(const py::dict& params) {
             result_mode |= error_metric_mode::out_of_bag_error;
         else if (modes[i] == "out_of_bag_error_per_observation")
             result_mode |= error_metric_mode::out_of_bag_error_per_observation;
+#if defined(ONEDAL_VERSION) && ONEDAL_VERSION >= 20230101
+        else if (modes[i] == "out_of_bag_error_accuracy")
+            result_mode |= error_metric_mode::out_of_bag_error_accuracy;
+        else if (modes[i] == "out_of_bag_error_r2")
+            result_mode |= error_metric_mode::out_of_bag_error_r2;
+        else if (modes[i] == "out_of_bag_error_decision_function")
+            result_mode |= error_metric_mode::out_of_bag_error_decision_function;
+        else if (modes[i] == "out_of_bag_error_prediction")
+            result_mode |= error_metric_mode::out_of_bag_error_prediction;
+#endif // defined(ONEDAL_VERSION) && ONEDAL_VERSION>=20230101
         else
             ONEDAL_PARAM_DISPATCH_THROW_INVALID_VALUE(mode);
     }
@@ -238,6 +249,12 @@ void init_train_result(py::module_& m) {
         .DEF_ONEDAL_PY_PROPERTY(model, result_t)
         .DEF_ONEDAL_PY_PROPERTY(oob_err, result_t)
         .DEF_ONEDAL_PY_PROPERTY(oob_err_per_observation, result_t)
+#if defined(ONEDAL_VERSION) && ONEDAL_VERSION >= 20230101
+        .DEF_ONEDAL_PY_PROPERTY(oob_err_accuracy, result_t)
+        .DEF_ONEDAL_PY_PROPERTY(oob_err_r2, result_t)
+        .DEF_ONEDAL_PY_PROPERTY(oob_err_decision_function, result_t)
+        .DEF_ONEDAL_PY_PROPERTY(oob_err_prediction, result_t)
+#endif // defined(ONEDAL_VERSION) && ONEDAL_VERSION>=20230101
         .DEF_ONEDAL_PY_PROPERTY(var_importance, result_t);
 }
 

onedal/ensemble/forest.py

@@ -302,7 +302,7 @@ def _validate_targets(self, y, dtype):
         return _column_or_1d(y, warn=True).astype(dtype, copy=False)
 
     def _get_sample_weight(self, X, y, sample_weight):
-        n_samples, _ = X.shape
+        n_samples = X.shape[0]
         dtype = X.dtype
         if n_samples == 1:
             raise ValueError("n_samples=1")
@@ -359,9 +359,14 @@ def _fit(self, X, y, sample_weight, module, queue):
         self._onedal_model = train_result.model
 
         if self.oob_score:
-            self.oob_score_ = from_table(train_result.oob_err)[0, 0]
-            self.oob_prediction_ = from_table(
-                train_result.oob_err_per_observation)
+            if self.is_classification:
+                self.oob_score_ = from_table(train_result.oob_err_accuracy)[0, 0]
+                self.oob_prediction_ = from_table(
+                    train_result.oob_err_decision_function)
+            else:
+                self.oob_score_ = from_table(train_result.oob_err_r2)[0, 0]
+                self.oob_prediction_ = from_table(
+                    train_result.oob_err_prediction).reshape(-1)
             if np.any(self.oob_prediction_ == 0):
                 warnings.warn(
                     "Some inputs do not have OOB scores. This probably means "

sklearnex/preview/decomposition/pca.py

@@ -28,6 +28,8 @@
 from sklearn.utils.validation import check_array
 from sklearn.base import BaseEstimator
 from sklearn.utils.validation import check_is_fitted
+if sklearn_check_version('1.1') and not sklearn_check_version('1.2'):
+    from sklearn.utils import check_scalar
 if sklearn_check_version('0.23'):
     from sklearn.decomposition._pca import _infer_dimension
 else:
@@ -38,6 +40,9 @@
 
 
 class PCA(sklearn_PCA):
+    if sklearn_check_version('1.2'):
+        _parameter_constraints: dict = {**sklearn_PCA._parameter_constraints}
+
     def __init__(
         self,
         n_components=None,
@@ -83,6 +88,15 @@ def _validate_n_components(self, n_components, n_samples,
                                  % (n_components, type(n_components)))
 
     def fit(self, X, y=None):
+        if sklearn_check_version('1.2'):
+            self._validate_params()
+        elif sklearn_check_version('1.1'):
+            check_scalar(
+                self.n_oversamples,
+                "n_oversamples",
+                min_val=1,
+                target_type=numbers.Integral,
+            )
         self._fit(X)
         return self
 
@@ -93,12 +107,12 @@ def _fit(self, X):
                 "TruncatedSVD for a possible alternative."
             )
 
-        X = _check_array(
-            X,
-            dtype=[np.float64, np.float32],
-            ensure_2d=True,
-            copy=False
-        )
+        if sklearn_check_version('0.23'):
+            X = self._validate_data(X, dtype=[np.float64, np.float32],
+                                    ensure_2d=True, copy=False)
+        else:
+            X = _check_array(X, dtype=[np.float64, np.float32],
+                             ensure_2d=True, copy=False)
 
         n_samples, n_features = X.shape
         n_sf_min = min(n_samples, n_features)

sklearnex/preview/ensemble/forest.py

@@ -39,7 +39,8 @@
 from sklearn.utils.validation import (
     check_is_fitted,
     check_consistent_length,
-    check_array)
+    check_array,
+    check_X_y)
 
 from onedal.datatypes import _check_array, _num_features, _num_samples
 
@@ -339,8 +340,6 @@ def _onedal_ready(self, X, y, sample_weight):
             self._validate_params()
         else:
             self._check_parameters()
-        if sample_weight is not None:
-            sample_weight = self.check_sample_weight(sample_weight, X)
 
         correct_sparsity = not sp.issparse(X)
         correct_ccp_alpha = self.ccp_alpha == 0.0
@@ -526,6 +525,8 @@ def _onedal_cpu_supported(self, method_name, *data):
             ready, X, y, sample_weight = self._onedal_ready(*data)
             if not ready:
                 return False
+            elif sp.issparse(X):
+                return False
             elif sp.issparse(y):
                 return False
             elif sp.issparse(sample_weight):
@@ -534,6 +535,8 @@ def _onedal_cpu_supported(self, method_name, *data):
                 return False
             elif self.warm_start:
                 return False
+            elif self.oob_score and not daal_check_version((2023, 'P', 101)):
+                return False
             elif not self.n_outputs_ == 1:
                 return False
             elif hasattr(self, 'estimators_'):
@@ -563,14 +566,20 @@ def _onedal_gpu_supported(self, method_name, *data):
             ready, X, y, sample_weight = self._onedal_ready(*data)
             if not ready:
                 return False
+            elif sp.issparse(X):
+                return False
             elif sp.issparse(y):
                 return False
+            elif sp.issparse(sample_weight):
+                return False
             elif not sample_weight:  # `sample_weight` is not supported.
                 return False
             elif not self.ccp_alpha == 0.0:
                 return False
             elif self.warm_start:
                 return False
+            elif self.oob_score:
+                return False
             elif not self.n_outputs_ == 1:
                 return False
             elif hasattr(self, 'estimators_'):
@@ -596,9 +605,33 @@ def _onedal_gpu_supported(self, method_name, *data):
             f'Unknown method {method_name} in {self.__class__.__name__}')
 
     def _onedal_fit(self, X, y, sample_weight=None, queue=None):
-        X, y = make2d(np.asarray(X)), make2d(np.asarray(y))
+        if sklearn_check_version('1.2'):
+            X, y = self._validate_data(
+                X, y, multi_output=False, accept_sparse=False,
+                dtype=[np.float64, np.float32]
+            )
+        else:
+            X, y = check_X_y(
+                X, y, accept_sparse=False, dtype=[np.float64, np.float32],
+                multi_output=False
+            )
 
-        y = check_array(y, ensure_2d=False)
+        if sample_weight is not None:
+            sample_weight = self.check_sample_weight(sample_weight, X)
+
+        y = np.atleast_1d(y)
+        if y.ndim == 2 and y.shape[1] == 1:
+            warnings.warn(
+                "A column-vector y was passed when a 1d array was"
+                " expected. Please change the shape of y to "
+                "(n_samples,), for example using ravel().",
+                DataConversionWarning,
+                stacklevel=2,
+            )
+        if y.ndim == 1:
+            # reshape is necessary to preserve the data contiguity against vs
+            # [:, np.newaxis] that does not.
+            y = np.reshape(y, (-1, 1))
 
         y, expanded_class_weight = self._validate_y_class_weight(y)
 
@@ -620,7 +653,7 @@ def _onedal_fit(self, X, y, sample_weight=None, queue=None):
                 "Training data only contain information about one class.")
 
         if self.oob_score:
-            err = 'out_of_bag_error|out_of_bag_error_per_observation'
+            err = 'out_of_bag_error_accuracy|out_of_bag_error_decision_function'
         else:
             err = 'none'
 
@@ -664,35 +697,35 @@ def _onedal_fit(self, X, y, sample_weight=None, queue=None):
         return self
 
     def _onedal_predict(self, X, queue=None):
+        X = check_array(X, dtype=[np.float32, np.float64])
+        check_is_fitted(self)
         if sklearn_check_version("1.0"):
             self._check_feature_names(X, reset=False)
-        X = check_array(
-            X,
-            accept_sparse=False,  # is not supported
-            dtype=[np.float64, np.float32]
-        )
 
         res = self._onedal_estimator.predict(X, queue=queue)
         return np.take(self.classes_,
                        res.ravel().astype(np.int64, casting='unsafe'))
 
     def _onedal_predict_proba(self, X, queue=None):
+        X = check_array(X, dtype=[np.float64, np.float32])
         check_is_fitted(self)
         if sklearn_check_version('0.23'):
             self._check_n_features(X, reset=False)
         if sklearn_check_version("1.0"):
             self._check_feature_names(X, reset=False)
-        X = check_array(
-            X,
-            accept_sparse=False,  # is not supported
-            dtype=[np.float64, np.float32]
-        )
         return self._onedal_estimator.predict_proba(X, queue=queue)
 
 
 class RandomForestRegressor(sklearn_RandomForestRegressor, BaseRandomForest):
     __doc__ = sklearn_RandomForestRegressor.__doc__
 
+    if sklearn_check_version('1.2'):
+        _parameter_constraints: dict = {
+            **sklearn_RandomForestRegressor._parameter_constraints,
+            "max_bins": [Interval(numbers.Integral, 2, None, closed="left")],
+            "min_bin_size": [Interval(numbers.Integral, 1, None, closed="left")]
+        }
+
     if sklearn_check_version('1.0'):
         def __init__(
                 self,
@@ -862,6 +895,8 @@ def _onedal_cpu_supported(self, method_name, *data):
                 return False
             elif self.warm_start:
                 return False
+            elif self.oob_score and not daal_check_version((2023, 'P', 101)):
+                return False
             elif not self.n_outputs_ == 1:
                 return False
             elif hasattr(self, 'estimators_'):
@@ -903,6 +938,8 @@ def _onedal_gpu_supported(self, method_name, *data):
                 return False
             elif self.warm_start:
                 return False
+            elif self.oob_score:
+                return False
             elif not self.n_outputs_ == 1:
                 return False
             elif hasattr(self, 'estimators_'):
@@ -949,7 +986,7 @@ def _onedal_fit(self, X, y, sample_weight=None, queue=None):
         rs_ = check_random_state(self.random_state)
 
         if self.oob_score:
-            err = 'out_of_bag_error|out_of_bag_error_per_observation'
+            err = 'out_of_bag_error_r2|out_of_bag_error_prediction'
         else:
             err = 'none'
 
@@ -986,11 +1023,7 @@ def _onedal_fit(self, X, y, sample_weight=None, queue=None):
     def _onedal_predict(self, X, queue=None):
         if sklearn_check_version("1.0"):
             self._check_feature_names(X, reset=False)
-        X = check_array(
-            X,
-            accept_sparse=False,
-            dtype=[np.float64, np.float32]
-        )
+        X = self._validate_X_predict(X)
         return self._onedal_estimator.predict(X, queue=queue)
 
     @wrap_output_data

sklearnex/preview/linear_model/linear.py

@@ -30,7 +30,7 @@
     if sklearn_check_version('1.0') and not sklearn_check_version('1.2'):
         from sklearn.linear_model._base import _deprecate_normalize
 
-    from sklearn.utils.validation import _deprecate_positional_args
+    from sklearn.utils.validation import _deprecate_positional_args, check_X_y
     from sklearn.exceptions import NotFittedError
     from scipy.sparse import issparse
 
@@ -233,6 +233,20 @@ def _initialize_onedal_estimator(self):
         def _onedal_fit(self, X, y, sample_weight, queue=None):
             assert sample_weight is None
 
+            check_params = {
+                'X': X,
+                'y': y,
+                'dtype': [np.float64, np.float32],
+                'accept_sparse': ['csr', 'csc', 'coo'],
+                'y_numeric': True,
+                'multi_output': True,
+                'force_all_finite': False
+            }
+            if sklearn_check_version('1.2'):
+                X, y = self._validate_data(**check_params)
+            else:
+                X, y = check_X_y(**check_params)
+
             if sklearn_check_version(
                     '1.0') and not sklearn_check_version('1.2'):
                 self._normalize = _deprecate_normalize(
@@ -247,6 +261,7 @@ def _onedal_fit(self, X, y, sample_weight, queue=None):
             self._save_attributes()
 
         def _onedal_predict(self, X, queue=None):
+            X = self._validate_data(X, accept_sparse=False, reset=False)
             if not hasattr(self, '_onedal_estimator'):
                 self._initialize_onedal_estimator()
                 self._onedal_estimator.coef_ = self.coef_