From 49b9eb5b5d65fc986a681c42c7789ab06d99896a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 6 Apr 2021 14:53:04 -0500 Subject: [PATCH 1/4] add MAPE to regression metrics (fixes #691) --- dask_ml/metrics/__init__.py | 1 + dask_ml/metrics/regression.py | 27 +++++++++++++++++++++++++++ tests/metrics/test_regression.py | 3 ++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/dask_ml/metrics/__init__.py b/dask_ml/metrics/__init__.py index 05349d10f..32b85f721 100644 --- a/dask_ml/metrics/__init__.py +++ b/dask_ml/metrics/__init__.py @@ -6,6 +6,7 @@ ) from .regression import ( # noqa mean_absolute_error, + mean_absolute_percentage_error, mean_squared_error, mean_squared_log_error, r2_score, diff --git a/dask_ml/metrics/regression.py b/dask_ml/metrics/regression.py index d1849ef88..77ba35bcb 100644 --- a/dask_ml/metrics/regression.py +++ b/dask_ml/metrics/regression.py @@ -81,6 +81,33 @@ def mean_absolute_error( return result +@derived_from(sklearn.metrics) +def mean_absolute_percentage_error( + y_true: ArrayLike, + y_pred: ArrayLike, + sample_weight: Optional[ArrayLike] = None, + multioutput: Optional[str] = "uniform_average", + compute: bool = True, +) -> ArrayLike: + _check_sample_weight(sample_weight) + epsilon = np.finfo(np.float64).eps + mape = abs(y_pred - y_true) / da.maximum(y_true, epsilon) + output_errors = mape.mean(axis=0) + + if isinstance(multioutput, str) or multioutput is None: + if multioutput == "raw_values": + if compute: + return output_errors.compute() + else: + return output_errors + else: + raise ValueError("Weighted 'multioutput' not supported.") + result = output_errors.mean() + if compute: + result = result.compute() + return result + + @derived_from(sklearn.metrics) def r2_score( y_true: ArrayLike, diff --git a/tests/metrics/test_regression.py b/tests/metrics/test_regression.py index dfdc5480c..5d21cfaef 100644 --- a/tests/metrics/test_regression.py +++ b/tests/metrics/test_regression.py @@ -7,12 +7,13 @@ import dask_ml.metrics -@pytest.fixture(params=["mean_squared_error", "mean_absolute_error", "r2_score"]) +@pytest.fixture(params=["mean_squared_error", "mean_absolute_error", "mean_absolute_percentage_error", "r2_score"]) def metric_pairs(request): """Pairs of (dask-ml, sklearn) regression metrics. * mean_squared_error * mean_absolute_error + * mean_absolute_percentage_error * r2_score """ return ( From 1142fcc93a95e2af72211410959afed14116121d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 9 Apr 2021 10:51:13 -0500 Subject: [PATCH 2/4] linting --- tests/metrics/test_regression.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/metrics/test_regression.py b/tests/metrics/test_regression.py index 5d21cfaef..1e06c32ec 100644 --- a/tests/metrics/test_regression.py +++ b/tests/metrics/test_regression.py @@ -7,7 +7,14 @@ import dask_ml.metrics -@pytest.fixture(params=["mean_squared_error", "mean_absolute_error", "mean_absolute_percentage_error", "r2_score"]) +@pytest.fixture( + params=[ + "mean_squared_error", + "mean_absolute_error", + "mean_absolute_percentage_error", + "r2_score", + ] +) def metric_pairs(request): """Pairs of (dask-ml, sklearn) regression metrics. From b05213b52941cb73db0e8ac6f375cd117cd5683b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 9 Apr 2021 11:46:38 -0500 Subject: [PATCH 3/4] fix compatibility with older scikit-learn --- dask_ml/metrics/regression.py | 38 +++++++++++++++++++++++++++++++- docs/source/modules/api.rst | 1 + tests/metrics/test_regression.py | 20 +++++++++++------ 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/dask_ml/metrics/regression.py b/dask_ml/metrics/regression.py index 77ba35bcb..7fcdde8fe 100644 --- a/dask_ml/metrics/regression.py +++ b/dask_ml/metrics/regression.py @@ -81,7 +81,6 @@ def mean_absolute_error( return result -@derived_from(sklearn.metrics) def mean_absolute_percentage_error( y_true: ArrayLike, y_pred: ArrayLike, @@ -89,6 +88,43 @@ def mean_absolute_percentage_error( multioutput: Optional[str] = "uniform_average", compute: bool = True, ) -> ArrayLike: + """Mean absolute percentage error regression loss. + + Note here that we do not represent the output as a percentage in range + [0, 100]. Instead, we represent it in range [0, 1/eps]. Read more in + https://scikit-learn.org/stable/modules/model_evaluation.html#mean-absolute-percentage-error + + Parameters + ---------- + y_true : array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + multioutput : {'raw_values', 'uniform_average'} or array-like + Defines aggregating of multiple output values. + Array-like value defines weights used to average errors. + If input is list then the shape must be (n_outputs,). + 'raw_values' : + Returns a full set of errors in case of multioutput input. + 'uniform_average' : + Errors of all outputs are averaged with uniform weight. + compute : bool + Whether to compute this result (default ``True``) + + Returns + ------- + loss : float or array-like of floats in the range [0, 1/eps] + If multioutput is 'raw_values', then mean absolute percentage error + is returned for each output separately. + If multioutput is 'uniform_average' or ``None``, then the + equally-weighted average of all output errors is returned. + MAPE output is non-negative floating point. The best value is 0.0. + But note the fact that bad predictions can lead to arbitarily large + MAPE values, especially if some y_true values are very close to zero. + Note that we return a large value instead of `inf` when y_true is zero. + """ _check_sample_weight(sample_weight) epsilon = np.finfo(np.float64).eps mape = abs(y_pred - y_true) / da.maximum(y_true, epsilon) diff --git a/docs/source/modules/api.rst b/docs/source/modules/api.rst index 601357ba6..7a2d4d06c 100644 --- a/docs/source/modules/api.rst +++ b/docs/source/modules/api.rst @@ -245,6 +245,7 @@ Regression Metrics :toctree: generated/ metrics.mean_absolute_error + metrics.mean_absolute_percentage_error metrics.mean_squared_error metrics.mean_squared_log_error metrics.r2_score diff --git a/tests/metrics/test_regression.py b/tests/metrics/test_regression.py index 1e06c32ec..a03c4ef41 100644 --- a/tests/metrics/test_regression.py +++ b/tests/metrics/test_regression.py @@ -5,22 +5,28 @@ import sklearn.metrics import dask_ml.metrics +from dask_ml._compat import SK_024 + +_METRICS_TO_TEST = [ + "mean_squared_error", + "mean_absolute_error", + "r2_score", +] + +# mean_absolute_percentage_error() was added in scikit-learn 0.24.0 +if SK_024: + _METRICS_TO_TEST.append("mean_absolute_percentage_error") @pytest.fixture( - params=[ - "mean_squared_error", - "mean_absolute_error", - "mean_absolute_percentage_error", - "r2_score", - ] + params=_METRICS_TO_TEST ) def metric_pairs(request): """Pairs of (dask-ml, sklearn) regression metrics. * mean_squared_error * mean_absolute_error - * mean_absolute_percentage_error + * mean_absolute_percentage_error (if scikit-learn >= 0.24.0) * r2_score """ return ( From 99415c756b58be08c46cb522d28525e5aec3c708 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 9 Apr 2021 12:00:14 -0500 Subject: [PATCH 4/4] linting --- tests/metrics/test_regression.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/metrics/test_regression.py b/tests/metrics/test_regression.py index a03c4ef41..475b6e31c 100644 --- a/tests/metrics/test_regression.py +++ b/tests/metrics/test_regression.py @@ -18,9 +18,7 @@ _METRICS_TO_TEST.append("mean_absolute_percentage_error") -@pytest.fixture( - params=_METRICS_TO_TEST -) +@pytest.fixture(params=_METRICS_TO_TEST) def metric_pairs(request): """Pairs of (dask-ml, sklearn) regression metrics.