Merge branch 'master' into include-crossval-for-cumulatedscore

Author: vincentblot28

HISTORY.rst

@@ -10,8 +10,7 @@ History
 * Add MNIST example for classification
 * Add cross-conformal for classification
 * Add `notebooks` folder containing notebooks used for generating documentation tutorials
-* Uniformize the use of matrix k_ and add an argument "ensemble" to method 
-  "predict" in regression.py
+* Uniformize the use of matrix k_ and add an argument "ensemble" to method "predict" in regression.py
 * Add replication of the Chen Xu's tutorial testing Jackknife+aB vs Jackknife+
 * Add Jackknife+-after-Bootstrap documentation
 * Improve scikit-learn pipelines compatibility

examples/classification/1-quickstart/plot_comp_methods_on_2d_dataset.py

@@ -53,7 +53,10 @@
 import matplotlib.pyplot as plt
 
 from mapie.classification import MapieClassifier
-from mapie.metrics import classification_coverage_score
+from mapie.metrics import (
+    classification_coverage_score,
+    classification_mean_width_score
+)
 from mapie._typing import ArrayLike
 
 
@@ -261,7 +264,7 @@ def plot_results(
         for i, _ in enumerate(alpha_)
     ]
     mean_width[method] = [
-        y_ps_mapie[method][:, :, i].sum(axis=1).mean()
+        classification_mean_width_score(y_ps_mapie[method][:, :, i])
         for i, _ in enumerate(alpha_)
     ]
 

examples/classification/2-advanced-analysis/plot_digits_classification.py

@@ -18,7 +18,10 @@
 import matplotlib.pyplot as plt
 
 from mapie.classification import MapieClassifier
-from mapie.metrics import classification_coverage_score
+from mapie.metrics import (
+    classification_coverage_score,
+    classification_mean_width_score
+)
 from mapie._typing import ArrayLike
 
 
@@ -206,15 +209,21 @@ def get_datasets(dataset: Any) -> Tuple[
 # the "real" coverage obtained on the test set.
 
 coverages1 = [
-    classification_coverage_score(y_test1, y_ps1[:, :, ia])
-    for ia, _ in enumerate(alpha)
+    classification_coverage_score(y_test1, y_ps1[:, :, i])
+    for i, _ in enumerate(alpha)
 ]
 coverages2 = [
-    classification_coverage_score(y_test2, y_ps2[:, :, ia])
-    for ia, _ in enumerate(alpha)
+    classification_coverage_score(y_test2, y_ps2[:, :, i])
+    for i, _ in enumerate(alpha)
+]
+widths1 = [
+    classification_mean_width_score(y_ps1[:, :, i])
+    for i, _ in enumerate(alpha)
+]
+widths2 = [
+    classification_mean_width_score(y_ps2[:, :, i])
+    for i, _ in enumerate(alpha)
 ]
-widths1 = [y_ps1[:, :, ia].sum(axis=1).mean() for ia, _ in enumerate(alpha)]
-widths2 = [y_ps2[:, :, ia].sum(axis=1).mean() for ia, _ in enumerate(alpha)]
 
 _, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 5))
 axes[0].set_xlabel("1 - alpha")

examples/regression/1-quickstart/plot_timeseries_example.py

@@ -30,13 +30,17 @@
 Jackknife+ and CV+ methods.
 """
 import pandas as pd
-from mapie.metrics import regression_coverage_score
-from mapie.regression import MapieRegressor
 from matplotlib import pylab as plt
 from scipy.stats import randint
 from sklearn.ensemble import RandomForestRegressor
 from sklearn.model_selection import RandomizedSearchCV, TimeSeriesSplit
 
+from mapie.metrics import (
+    regression_coverage_score,
+    regression_mean_width_score
+)
+from mapie.regression import MapieRegressor
+
 # Load input data and feature engineering
 demand_df = pd.read_csv(
     "../../data/demand_temperature.csv", parse_dates=True, index_col=0
@@ -86,7 +90,7 @@
 mapie.fit(X_train, y_train)
 y_pred, y_pis = mapie.predict(X_test, alpha=alpha)
 coverage = regression_coverage_score(y_test, y_pis[:, 0, 0], y_pis[:, 1, 0])
-width = (y_pis[:, 1, 0] - y_pis[:, 0, 0]).mean()
+width = regression_mean_width_score(y_pis[:, 0, 0], y_pis[:, 1, 0])
 
 # Print results
 print(

examples/regression/3-scientific-articles/plot_barber2020_simulations.py

@@ -34,7 +34,10 @@
 from sklearn.linear_model import LinearRegression
 from matplotlib import pyplot as plt
 
-from mapie.metrics import regression_coverage_score
+from mapie.metrics import (
+    regression_coverage_score,
+    regression_mean_width_score
+)
 from mapie.regression import MapieRegressor
 from mapie._typing import ArrayLike
 
@@ -116,12 +119,14 @@ def PIs_vs_dimensions(
                     **params
                 )
                 mapie.fit(X_train, y_train)
-                y_pred, y_pis = mapie.predict(X_test, alpha=alpha)
+                _, y_pis = mapie.predict(X_test, alpha=alpha)
                 coverage = regression_coverage_score(
                     y_test, y_pis[:, 0, 0], y_pis[:, 1, 0]
                 )
                 results[strategy][dimension]["coverage"][trial] = coverage
-                width_mean = (y_pis[:, 1, 0] - y_pis[:, 0, 0]).mean()
+                width_mean = regression_mean_width_score(
+                    y_pis[:, 0, 0], y_pis[:, 1, 0]
+                )
                 results[strategy][dimension]["width_mean"][trial] = width_mean
     return results
 

mapie/regression.py

@@ -675,5 +675,4 @@ def predict(
                 ).data
                 if ensemble:
                     y_pred = aggregate_all(self.agg_function, y_pred_multi)
-                np.stack([y_pred_low, y_pred_up], axis=1)
             return y_pred, np.stack([y_pred_low, y_pred_up], axis=1)