From 37c79dc794173b7fb6f26d716c435a10a016d0f0 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Tue, 22 Jul 2025 15:05:24 +0200 Subject: [PATCH 01/18] ENH: merge prototype with segmentation risk control code (#726) --- mapie/__init__.py | 2 + mapie/control_risk/ltt.py | 14 +-- mapie/control_risk/p_values.py | 28 +++--- mapie/risk_control_draft.py | 168 +++++++++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 17 deletions(-) create mode 100644 mapie/risk_control_draft.py diff --git a/mapie/__init__.py b/mapie/__init__.py index 5ec9939e1..35fd5b090 100644 --- a/mapie/__init__.py +++ b/mapie/__init__.py @@ -4,6 +4,7 @@ regression, utils, risk_control, + risk_control_draft, calibration, subsample, ) @@ -13,6 +14,7 @@ "regression", "classification", "risk_control", + "risk_control_draft", "calibration", "metrics", "utils", diff --git a/mapie/control_risk/ltt.py b/mapie/control_risk/ltt.py index 9b7d7e124..e19d3b849 100644 --- a/mapie/control_risk/ltt.py +++ b/mapie/control_risk/ltt.py @@ -9,11 +9,12 @@ def ltt_procedure( - r_hat: NDArray, - alpha_np: NDArray, + r_hat: NDArray[np.float32], + alpha_np: NDArray[np.float32], delta: Optional[float], - n_obs: int -) -> Tuple[List[List[Any]], NDArray]: + n_obs: int, + binary: bool = False, # TODO: maybe should pass p_values fonction instead +) -> Tuple[List[List[Any]], NDArray[np.float32]]: """ Apply the Learn-Then-Test procedure for risk control. Note that we will do a multiple test for ``r_hat`` that are @@ -63,13 +64,14 @@ def ltt_procedure( "Invalid delta: delta cannot be None while" + " controlling precision with LTT. " ) - p_values = compute_hoeffdding_bentkus_p_value(r_hat, n_obs, alpha_np) + p_values = compute_hoeffdding_bentkus_p_value(r_hat, n_obs, alpha_np, binary) N = len(p_values) valid_index = [] for i in range(len(alpha_np)): l_index = np.where(p_values[:, i] <= delta/N)[0].tolist() valid_index.append(l_index) - return valid_index, p_values + return valid_index, p_values # TODO : p_values is not used, we could remove it + # Or return corrected p_values def find_lambda_control_star( diff --git a/mapie/control_risk/p_values.py b/mapie/control_risk/p_values.py index 2305c126f..d1a420a4c 100644 --- a/mapie/control_risk/p_values.py +++ b/mapie/control_risk/p_values.py @@ -8,10 +8,11 @@ def compute_hoeffdding_bentkus_p_value( - r_hat: NDArray, + r_hat: NDArray[np.float32], n_obs: int, - alpha: Union[float, NDArray] -) -> NDArray: + alpha: Union[float, NDArray[np.float32]], + binary: bool = False, +) -> NDArray[np.float32]: """ The method computes the p_values according to the Hoeffding_Bentkus inequality for each @@ -63,7 +64,7 @@ def compute_hoeffdding_bentkus_p_value( ) hoeffding_p_value = np.exp( -n_obs * _h1( - np.where( + np.where( # TODO : shouldn't we use np.minimum ? r_hat_repeat > alpha_repeat, alpha_repeat, r_hat_repeat @@ -71,10 +72,11 @@ def compute_hoeffdding_bentkus_p_value( alpha_repeat ) ) - bentkus_p_value = np.e * binom.cdf( + factor = 1 if binary else np.e + bentkus_p_value = factor * binom.cdf( np.ceil(n_obs * r_hat_repeat), n_obs, alpha_repeat ) - hb_p_value = np.where( + hb_p_value = np.where( # TODO : shouldn't we use np.minimum ? bentkus_p_value > hoeffding_p_value, hoeffding_p_value, bentkus_p_value @@ -83,9 +85,8 @@ def compute_hoeffdding_bentkus_p_value( def _h1( - r_hats: NDArray, - alphas: NDArray -) -> NDArray: + r_hats: NDArray[np.float32], alphas: NDArray[np.float32] +) -> NDArray[np.float32]: """ This function allow us to compute the tighter version of hoeffding inequality. @@ -114,6 +115,11 @@ def _h1( ------- NDArray of shape a(n_lambdas, n_alpha). """ - elt1 = r_hats * np.log(r_hats/alphas) - elt2 = (1-r_hats) * np.log((1-r_hats)/(1-alphas)) + elt1 = np.zeros_like(r_hats, dtype=float) + + # Compute only where r_hats != 0 to avoid log(0) + # TODO: check Angelopoulos implementation + mask = r_hats != 0 + elt1[mask] = r_hats[mask] * np.log(r_hats[mask] / alphas[mask]) + elt2 = (1 - r_hats) * np.log((1 - r_hats) / (1 - alphas)) return elt1 + elt2 diff --git a/mapie/risk_control_draft.py b/mapie/risk_control_draft.py new file mode 100644 index 000000000..a4f1f9485 --- /dev/null +++ b/mapie/risk_control_draft.py @@ -0,0 +1,168 @@ +from typing import Any, Optional, Union + +import numpy as np +from numpy._typing import ArrayLike, NDArray +from sklearn.utils import check_random_state + +from mapie.control_risk.ltt import ltt_procedure +from mapie.utils import _check_n_jobs, _check_verbose + +# General TODOs: +# TODO: maybe use type float instead of float32? +# TODO : in calibration and prediction, +# use _transform_pred_proba or a function adapted to binary +# to get the probabilities depending on the classifier + + +class BinaryClassificationController: # pragma: no cover + # TODO : test that this is working with a sklearn pipeline + # TODO : test that this is working with a pandas dataframes + """ + Controller for the calibration of our binary classifier. + + Parameters + ---------- + fitted_binary_classifier: Any + Any object that provides a `predict_proba` method. + + metric: str + The performance metric we want to control (ex: "precision") + + target_level: float + The target performance level we want to achieve (ex: 0.8) + + confidence_level: float + The maximum acceptable probability of the precision falling below the + target precision level (ex: 0.8) + + Attributes + ---------- + precision_per_threshold: NDArray + Precision of the binary classifier on the calibration set for each + threshold from self._thresholds. + + valid_threshold: NDArray + Thresholds that meet the target precision with the desired confidence. + + best_threshold: float + Valid threshold that maximizes the recall, i.e. the smallest valid + threshold. + """ + + def __init__( + self, + fitted_binary_classifier: Any, + metric: str, + target_level: float, + confidence_level: float = 0.9, + n_jobs: Optional[int] = None, + random_state: Optional[Union[int, np.random.RandomState]] = None, + verbose: int = 0 + ): + _check_n_jobs(n_jobs) + _check_verbose(verbose) + check_random_state(random_state) + + self._classifier = fitted_binary_classifier + self._alpha = 1 - target_level + self._delta = 1 - confidence_level + self._n_jobs = n_jobs # TODO : use this in the class or delete + self._random_state = random_state # TODO : use this in the class or delete + self._verbose = verbose # TODO : use this in the class or delete + + self._thresholds: NDArray[np.float32] = np.arange(0, 1, 0.01) + # TODO: add a _is_calibrated attribute to check at prediction time + + self.valid_thresholds: Optional[NDArray[np.float32]] = None + self.best_threshold: Optional[float] = None + + def calibrate(self, X_calibrate: ArrayLike, y_calibrate: ArrayLike) -> None: + """ + Find the threshold that statistically guarantees the desired precision + level while maximizing the recall. + + Parameters + ---------- + X_calibrate: ArrayLike + Features of the calibration set. + + y_calibrate: ArrayLike + True labels of the calibration set. + + Raises + ------ + ValueError + If no thresholds that meet the target precision with the desired + confidence level are found. + """ + # TODO: Make sure this works with sklearn train_test_split/Series + y_calibrate_ = np.asarray(y_calibrate) + + predictions_proba = self._classifier.predict_proba(X_calibrate)[:, 1] + + risk_per_threshold = 1 - self._compute_precision( + predictions_proba, y_calibrate_ + ) + + valid_thresholds_index, _ = ltt_procedure( + risk_per_threshold, + np.array([self._alpha]), + self._delta, + len(y_calibrate_), + True, + ) + self.valid_thresholds = self._thresholds[valid_thresholds_index[0]] + if len(self.valid_thresholds) == 0: + # TODO: just warn, and raise error at prediction if no valid thresholds + raise ValueError("No valid thresholds found") + + # Minimum in case of precision control only + self.best_threshold = min(self.valid_thresholds) + + def predict(self, X_test: ArrayLike) -> NDArray: + """ + Predict binary labels on the test set, using the best threshold found + during calibration. + + Parameters + ---------- + X_test: ArrayLike + Features of the test set. + + Returns + ------- + ArrayLike + Predicted labels (0 or 1) for each sample in the test set. + """ + predictions_proba = self._classifier.predict_proba(X_test)[:, 1] + return (predictions_proba >= self.best_threshold).astype(int) + + def _compute_precision( # TODO: use sklearn or MAPIE ? + self, predictions_proba: NDArray[np.float32], y_cal: NDArray[np.float32] + ) -> NDArray[np.float32]: + """ + Compute the precision for each threshold. + """ + predictions_per_threshold = ( + predictions_proba[:, np.newaxis] >= self._thresholds + ).astype(int) + + true_positives = np.sum( + (predictions_per_threshold == 1) & (y_cal[:, np.newaxis] == 1), + axis=0, + ) + false_positives = np.sum( + (predictions_per_threshold == 1) & (y_cal[:, np.newaxis] == 0), + axis=0, + ) + + positive_predictions = true_positives + false_positives + + # Avoid division by zero + precision_per_threshold = np.ones_like(self._thresholds, dtype=float) + nonzero_mask = positive_predictions > 0 + precision_per_threshold[nonzero_mask] = ( + true_positives[nonzero_mask] / positive_predictions[nonzero_mask] + ) + + return precision_per_threshold From 3f99b9c7e954ba6ea2685438bdc99b5fa185fca5 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Fri, 25 Jul 2025 14:46:46 +0200 Subject: [PATCH 02/18] ENH: theoretical tests notebook --- ...risk_control_theoretical_tests_proto.ipynb | 247 +++++++++++++++++ ...control_theoretical_tests_target_api.ipynb | 255 ++++++++++++++++++ mapie/risk_control_draft.py | 42 ++- 3 files changed, 539 insertions(+), 5 deletions(-) create mode 100644 mapie/control_risk/risk_control_theoretical_tests_proto.ipynb create mode 100644 mapie/control_risk/risk_control_theoretical_tests_target_api.ipynb diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb new file mode 100644 index 000000000..5baf86003 --- /dev/null +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -0,0 +1,247 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2ae91ff6-9706-41f1-bfdb-c39f5f2bfb9d", + "metadata": { + "id": "2ae91ff6-9706-41f1-bfdb-c39f5f2bfb9d" + }, + "source": [ + "# Binary classification risk control - Theoretical tests prototype" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "1c564c4f-1e63-4c2f-bdd5-d84029c1473a", + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "f1c2e64a", + "metadata": { + "id": "f1c2e64a" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import itertools\n", + "\n", + "from mapie.risk_control_draft import BinaryClassificationController" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "1fef2dc6-b5b1-43bc-ad05-e5e1fe7844bd", + "metadata": { + "id": "1fef2dc6-b5b1-43bc-ad05-e5e1fe7844bd" + }, + "outputs": [], + "source": [ + "class RandomClassifier:\n", + " def __init__(self, seed=42, threshold=0.5):\n", + " self.random_state = np.random.RandomState(seed)\n", + " self.threshold = threshold\n", + "\n", + " def predict_proba(self, X):\n", + " probs = np.round(self.random_state.rand(len(X)), 2)\n", + " return np.vstack([1 - probs, probs]).T\n", + "\n", + " def predict(self, X):\n", + " probs = self.predict_proba(X)[:, 1]\n", + " return (probs >= self.threshold).astype(int)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", + "metadata": { + "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" + }, + "outputs": [], + "source": [ + "N = 100 # size of the calibration set\n", + "p = 0.7 # proportion of positives in the calibration set\n", + "metric = \"recall\"\n", + "target_level = 0.6\n", + "predict_params = np.linspace(0, 0.99, 100)\n", + "confidence_level = 0.6\n", + "\n", + "n_repeats = 100" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "b415f516-7782-4d76-b304-3d630af365fc", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "b415f516-7782-4d76-b304-3d630af365fc", + "outputId": "3ff7579e-f564-49fb-9e95-ef78fa4239d0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "N = 100\n", + "Metric = recall\n", + "Target level = 0.6\n", + "Predict params = [0. 0.01 0.02 0.03 0.04 0.05 0.06 0.07 0.08 0.09 0.1 0.11 0.12 0.13\n", + " 0.14 0.15 0.16 0.17 0.18 0.19 0.2 0.21 0.22 0.23 0.24 0.25 0.26 0.27\n", + " 0.28 0.29 0.3 0.31 0.32 0.33 0.34 0.35 0.36 0.37 0.38 0.39 0.4 0.41\n", + " 0.42 0.43 0.44 0.45 0.46 0.47 0.48 0.49 0.5 0.51 0.52 0.53 0.54 0.55\n", + " 0.56 0.57 0.58 0.59 0.6 0.61 0.62 0.63 0.64 0.65 0.66 0.67 0.68 0.69\n", + " 0.7 0.71 0.72 0.73 0.74 0.75 0.76 0.77 0.78 0.79 0.8 0.81 0.82 0.83\n", + " 0.84 0.85 0.86 0.87 0.88 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97\n", + " 0.98 0.99]\n", + "Confidence Level = 0.6\n" + ] + } + ], + "source": [ + "print(f\"N = {N}\")\n", + "print(f\"Metric = {metric}\")\n", + "print(f\"Target level = {target_level}\")\n", + "print(f\"Predict params = {predict_params}\")\n", + "print(f\"Confidence Level = {confidence_level}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "649f5ef0-3c5e-410a-949e-e7aa3142d5fc", + "metadata": { + "id": "649f5ef0-3c5e-410a-949e-e7aa3142d5fc" + }, + "outputs": [], + "source": [ + "X_calibrate = list(range(1, N+1))\n", + "y_calibrate = [1] * int(p*N) + [0] * (N - int(p*N))\n", + "np.random.seed(42)\n", + "np.random.shuffle(y_calibrate)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "03383363-b86d-4593-adf4-80215b6f1dcf", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 376 + }, + "id": "03383363-b86d-4593-adf4-80215b6f1dcf", + "outputId": "b15146cf-518e-4a93-8128-6c1865a08b01" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Risk controlled! Proportion of actual valid parameters: 1.0\n" + ] + } + ], + "source": [ + "clf = RandomClassifier(threshold=0.8)\n", + "\n", + "if metric == \"precision\":\n", + " theoretical_value = p\n", + "elif metric == \"recall\":\n", + " theoretical_value = 1 - clf.threshold\n", + "\n", + "all_valid_parameters = []\n", + "\n", + "for _ in range(n_repeats):\n", + "\n", + " controller = BinaryClassificationController(\n", + " fitted_binary_classifier=clf,\n", + " metric=\"precision\",\n", + " target_level=target_level,\n", + " confidence_level=confidence_level,\n", + " )\n", + " controller.calibrate(X_calibrate, y_calibrate)\n", + "\n", + " valid_parameters = controller.valid_thresholds\n", + " all_valid_parameters.append(valid_parameters)\n", + " \n", + "all_valid_parameters = np.concatenate([x for x in all_valid_parameters if x.size > 0]) if any(x.size > 0 for x in all_valid_parameters) else np.array([])\n", + "\n", + "if metric == \"precision\":\n", + " nb_actual_valid = sum(1 for x in all_valid_parameters if p >= theoretical_value)\n", + "elif metric == \"recall\":\n", + " nb_actual_valid = sum(1 for x in all_valid_parameters if x <= (1 - theoretical_value))\n", + "\n", + "if nb_actual_valid/len(all_valid_parameters) >= confidence_level:\n", + " print(f\"Risk controlled! Proportion of actual valid parameters: {nb_actual_valid/len(all_valid_parameters)}\")\n", + "else:\n", + " print(f\"Risk not controlled. Proportion of actual valid parameters: {nb_actual_valid/len(all_valid_parameters)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "42f85519-17ae-46c7-bfdc-e26c1d87017a", + "metadata": { + "id": "42f85519-17ae-46c7-bfdc-e26c1d87017a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "All valid thresholds = [0. 0.01 0.02 ... 0.25 0.26 0.27]\n", + "Theoretical value = 0.19999999999999996\n" + ] + } + ], + "source": [ + "print(f\"All valid thresholds = {all_valid_parameters}\")\n", + "print(f\"Theoretical value = {theoretical_value}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24afeb59-e8a8-48a5-b555-c797fda6bac5", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mapie/control_risk/risk_control_theoretical_tests_target_api.ipynb b/mapie/control_risk/risk_control_theoretical_tests_target_api.ipynb new file mode 100644 index 000000000..89836e3cb --- /dev/null +++ b/mapie/control_risk/risk_control_theoretical_tests_target_api.ipynb @@ -0,0 +1,255 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2ae91ff6-9706-41f1-bfdb-c39f5f2bfb9d", + "metadata": {}, + "source": [ + "# Binary classification risk control - Theoretical tests - Target API" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "f1c2e64a", + "metadata": {}, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'mapie'", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mModuleNotFoundError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[41]\u001b[39m\u001b[32m, line 4\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnumpy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnp\u001b[39;00m\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mitertools\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mmapie\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mbinary_risk_control_target_api\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m BinaryClassificationRisk, BinaryClassificationController\n", + "\u001b[31mModuleNotFoundError\u001b[39m: No module named 'mapie'" + ] + } + ], + "source": [ + "import numpy as np\n", + "import itertools\n", + "\n", + "from mapie.binary_risk_control_target_api import BinaryClassificationRisk, BinaryClassificationController" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "1fef2dc6-b5b1-43bc-ad05-e5e1fe7844bd", + "metadata": {}, + "outputs": [], + "source": [ + "class RandomClassifier:\n", + " def __init__(self, seed=42, threshold=0.5):\n", + " self.random_state = np.random.RandomState(seed)\n", + " self.threshold = threshold\n", + "\n", + " def predict_proba(self, X):\n", + " probs = np.round(self.random_state.rand(len(X)), 2)\n", + " return np.vstack([1 - probs, probs]).T\n", + "\n", + " def predict(self, X):\n", + " probs = self.predict_proba(X)[:, 1]\n", + " return (probs >= self.threshold).astype(int)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "837bbfd2-0e30-4b27-99e6-71a6ee5c21d7", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'BinaryClassificationRisk' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[43]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m precision = \u001b[43mBinaryClassificationRisk\u001b[49m(\n\u001b[32m 2\u001b[39m occurrence=\u001b[38;5;28;01mlambda\u001b[39;00m y_true, y_pred: \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01mif\u001b[39;00m y_pred == \u001b[32m0\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mint\u001b[39m(y_pred == y_true),\n\u001b[32m 3\u001b[39m higher_is_better=\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[32m 4\u001b[39m binary=\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[32m 5\u001b[39m )\n\u001b[32m 7\u001b[39m false_discovery_rate = precision.transform_to_opposite()\n\u001b[32m 9\u001b[39m recall = BinaryClassificationRisk(\n\u001b[32m 10\u001b[39m occurrence=\u001b[38;5;28;01mlambda\u001b[39;00m y_true, y_pred: \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01mif\u001b[39;00m y_true == \u001b[32m0\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mint\u001b[39m(y_pred == y_true),\n\u001b[32m 11\u001b[39m higher_is_better=\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[32m 12\u001b[39m binary=\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[32m 13\u001b[39m )\n", + "\u001b[31mNameError\u001b[39m: name 'BinaryClassificationRisk' is not defined" + ] + } + ], + "source": [ + "precision = BinaryClassificationRisk(\n", + " occurrence=lambda y_true, y_pred: None if y_pred == 0 else int(y_pred == y_true),\n", + " higher_is_better=True,\n", + " binary=True,\n", + ")\n", + "\n", + "false_discovery_rate = precision.transform_to_opposite()\n", + "\n", + "recall = BinaryClassificationRisk(\n", + " occurrence=lambda y_true, y_pred: None if y_true == 0 else int(y_pred == y_true),\n", + " higher_is_better=True,\n", + " binary=True,\n", + ")\n", + "\n", + "false_negative_rate = recall.transform_to_opposite()\n", + "\n", + "accuracy = BinaryClassificationRisk(\n", + " occurrence=lambda y_true, y_pred: int(y_pred == y_true),\n", + " higher_is_better=True,\n", + " binary=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", + "metadata": {}, + "outputs": [], + "source": [ + "N_values = [1, 100] # size of the calibration set\n", + "p = 0.5 # proportion of positives in the calibration set\n", + "metrics = ['recall', 'precision']\n", + "target_levels = [0.2, 0.8]\n", + "predict_params_sets = [np.linspace(0, 0.99, 100), [0.5]]\n", + "confidence_levels = [0.1, 0.9]\n", + "\n", + "n_repeats = 100" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "b415f516-7782-4d76-b304-3d630af365fc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Combination 1:\n", + "N = 1\n", + "Metric = recall\n", + "Target level = 0.2\n", + "Predict params = [0. 0.01 0.02 0.03 0.04 0.05 0.06 0.07 0.08 0.09 0.1 0.11 0.12 0.13\n", + " 0.14 0.15 0.16 0.17 0.18 0.19 0.2 0.21 0.22 0.23 0.24 0.25 0.26 0.27\n", + " 0.28 0.29 0.3 0.31 0.32 0.33 0.34 0.35 0.36 0.37 0.38 0.39 0.4 0.41\n", + " 0.42 0.43 0.44 0.45 0.46 0.47 0.48 0.49 0.5 0.51 0.52 0.53 0.54 0.55\n", + " 0.56 0.57 0.58 0.59 0.6 0.61 0.62 0.63 0.64 0.65 0.66 0.67 0.68 0.69\n", + " 0.7 0.71 0.72 0.73 0.74 0.75 0.76 0.77 0.78 0.79 0.8 0.81 0.82 0.83\n", + " 0.84 0.85 0.86 0.87 0.88 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97\n", + " 0.98 0.99]\n", + "Confidence Level = 0.1\n" + ] + } + ], + "source": [ + "combinations = list(itertools.product(N_values, metrics, target_levels, predict_params_sets, confidence_levels))\n", + "\n", + "# for i, combination in enumerate(combinations[0], 1):\n", + "i, combination = 1, combinations[0]\n", + "\n", + "N, metric, target_level, predict_params, confidence_level = combination\n", + "print(f\"Combination {i}:\")\n", + "print(f\"N = {N}\")\n", + "print(f\"Metric = {metric}\")\n", + "print(f\"Target level = {target_level}\")\n", + "print(f\"Predict params = {predict_params}\")\n", + "print(f\"Confidence Level = {confidence_level}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "649f5ef0-3c5e-410a-949e-e7aa3142d5fc", + "metadata": {}, + "outputs": [], + "source": [ + "X_calibrate = list(range(1, N+1))\n", + "y_calibrate = [1] * int(p*N) + [0] * (N - int(p*N))\n", + "np.random.seed(42)\n", + "np.random.shuffle(y_calibrate)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "03383363-b86d-4593-adf4-80215b6f1dcf", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'recall' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[47]\u001b[39m\u001b[32m, line 7\u001b[39m\n\u001b[32m 5\u001b[39m theoretical_value = p\n\u001b[32m 6\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m metric == \u001b[33m'\u001b[39m\u001b[33mrecall\u001b[39m\u001b[33m'\u001b[39m:\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m risk = \u001b[43mrecall\u001b[49m\n\u001b[32m 8\u001b[39m theoretical_value = \u001b[32m1\u001b[39m - clf.threshold\n\u001b[32m 10\u001b[39m all_valid_parameters = []\n", + "\u001b[31mNameError\u001b[39m: name 'recall' is not defined" + ] + } + ], + "source": [ + "clf = RandomClassifier()\n", + "\n", + "if metric == 'precision':\n", + " risk = precision\n", + " theoretical_value = p\n", + "elif metric == 'recall':\n", + " risk = recall\n", + " theoretical_value = 1 - clf.threshold\n", + "\n", + "all_valid_parameters = []\n", + "\n", + "for _ in range(n_repeats):\n", + " \n", + " controller = BinaryClassificationController(\n", + " predict_function=clf.predict_proba,\n", + " risk=risk,\n", + " target_level=target_level,\n", + " confidence_level=confidence_level,\n", + " best_predict_param_choice=\"auto\",\n", + " )\n", + " controller.calibrate(X_calibrate, y_calibrate)\n", + " \n", + " valid_parameters = controller.valid_thresholds\n", + " all_valid_parameters.append(valid_parameters)\n", + "\n", + "if metric == 'precision':\n", + " nb_actual_valid = sum(1 for x in all_valid_parameters if p >= theoretical_value)\n", + "elif metric == 'recall':\n", + " nb_actual_valid = sum(1 for x in all_valid_parameters if x <= (1 - theoretical_value))\n", + "\n", + "if nb_actual_valid/len(all_valid_parameters) >= confidence_level:\n", + " print(\"Risk controlled\")\n", + "else:\n", + " print(\"Risk not controlled\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42f85519-17ae-46c7-bfdc-e26c1d87017a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.17" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mapie/risk_control_draft.py b/mapie/risk_control_draft.py index a4f1f9485..e8ee74224 100644 --- a/mapie/risk_control_draft.py +++ b/mapie/risk_control_draft.py @@ -1,3 +1,4 @@ +import warnings from typing import Any, Optional, Union import numpy as np @@ -100,7 +101,7 @@ def calibrate(self, X_calibrate: ArrayLike, y_calibrate: ArrayLike) -> None: predictions_proba = self._classifier.predict_proba(X_calibrate)[:, 1] - risk_per_threshold = 1 - self._compute_precision( + risk_per_threshold = 1 - self._compute_recall( predictions_proba, y_calibrate_ ) @@ -113,11 +114,12 @@ def calibrate(self, X_calibrate: ArrayLike, y_calibrate: ArrayLike) -> None: ) self.valid_thresholds = self._thresholds[valid_thresholds_index[0]] if len(self.valid_thresholds) == 0: - # TODO: just warn, and raise error at prediction if no valid thresholds - raise ValueError("No valid thresholds found") + warnings.warn("No valid thresholds found", UserWarning) + + else: + # Minimum in case of precision control only + self.best_threshold = min(self.valid_thresholds) - # Minimum in case of precision control only - self.best_threshold = min(self.valid_thresholds) def predict(self, X_test: ArrayLike) -> NDArray: """ @@ -166,3 +168,33 @@ def _compute_precision( # TODO: use sklearn or MAPIE ? ) return precision_per_threshold + + def _compute_recall( + self, predictions_proba: NDArray[np.float32], y_cal: NDArray[np.float32] + ) -> NDArray[np.float32]: + """ + Compute the recall for each threshold. + """ + predictions_per_threshold = ( + predictions_proba[:, np.newaxis] >= self._thresholds + ).astype(int) + + true_positives = np.sum( + (predictions_per_threshold == 1) & (y_cal[:, np.newaxis] == 1), + axis=0, + ) + false_negatives = np.sum( + (predictions_per_threshold == 0) & (y_cal[:, np.newaxis] == 1), + axis=0, + ) + + actual_positives = true_positives + false_negatives + + # Avoid division by zero + recall_per_threshold = np.ones_like(self._thresholds, dtype=float) + nonzero_mask = actual_positives > 0 + recall_per_threshold[nonzero_mask] = ( + true_positives[nonzero_mask] / actual_positives[nonzero_mask] + ) + + return recall_per_threshold From ee6983d1cb763ce38cdc05582d4e4886ad830fb6 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Fri, 25 Jul 2025 15:35:01 +0200 Subject: [PATCH 03/18] ENH: theoretical tests notebook --- ...risk_control_theoretical_tests_proto.ipynb | 208 +++++++++++++++++- mapie/risk_control_draft.py | 2 +- 2 files changed, 202 insertions(+), 8 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 5baf86003..6c9928f66 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 1, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 2, "id": "1fef2dc6-b5b1-43bc-ad05-e5e1fe7844bd", "metadata": { "id": "1fef2dc6-b5b1-43bc-ad05-e5e1fe7844bd" @@ -61,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 3, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -80,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 4, "id": "b415f516-7782-4d76-b304-3d630af365fc", "metadata": { "colab": { @@ -119,7 +119,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 5, "id": "649f5ef0-3c5e-410a-949e-e7aa3142d5fc", "metadata": { "id": "649f5ef0-3c5e-410a-949e-e7aa3142d5fc" @@ -134,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 7, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -149,6 +149,200 @@ "name": "stdout", "output_type": "stream", "text": [ + "[array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", + " 0.33]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", + " 0.33, 0.34]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", + " 0.33, 0.34]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", + " 0.33]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", + " 0.33, 0.34, 0.35, 0.36]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", + " 0.33, 0.34]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", + " 0.33, 0.34, 0.35, 0.36, 0.37, 0.38]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", + " 0.33]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", + " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", + " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27])]\n", "Risk controlled! Proportion of actual valid parameters: 1.0\n" ] } @@ -175,7 +369,7 @@ "\n", " valid_parameters = controller.valid_thresholds\n", " all_valid_parameters.append(valid_parameters)\n", - " \n", + "print(all_valid_parameters)\n", "all_valid_parameters = np.concatenate([x for x in all_valid_parameters if x.size > 0]) if any(x.size > 0 for x in all_valid_parameters) else np.array([])\n", "\n", "if metric == \"precision\":\n", diff --git a/mapie/risk_control_draft.py b/mapie/risk_control_draft.py index e8ee74224..3c96e0e79 100644 --- a/mapie/risk_control_draft.py +++ b/mapie/risk_control_draft.py @@ -101,7 +101,7 @@ def calibrate(self, X_calibrate: ArrayLike, y_calibrate: ArrayLike) -> None: predictions_proba = self._classifier.predict_proba(X_calibrate)[:, 1] - risk_per_threshold = 1 - self._compute_recall( + risk_per_threshold = 1 - self._compute_precision( predictions_proba, y_calibrate_ ) From b291ac21b14a20497ed7f9adb756805f7bdecb57 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Fri, 25 Jul 2025 16:08:51 +0200 Subject: [PATCH 04/18] ENH: theoretical tests notebook --- ...risk_control_theoretical_tests_proto.ipynb | 242 +++--------------- mapie/risk_control_draft.py | 1 - 2 files changed, 33 insertions(+), 210 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 6c9928f66..1867c7193 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "1c564c4f-1e63-4c2f-bdd5-d84029c1473a", "metadata": {}, "outputs": [], @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "1fef2dc6-b5b1-43bc-ad05-e5e1fe7844bd", "metadata": { "id": "1fef2dc6-b5b1-43bc-ad05-e5e1fe7844bd" @@ -61,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 11, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -69,8 +69,8 @@ "outputs": [], "source": [ "N = 100 # size of the calibration set\n", - "p = 0.7 # proportion of positives in the calibration set\n", - "metric = \"recall\"\n", + "p = 0.5 # proportion of positives in the calibration set\n", + "metric = \"precision\"\n", "target_level = 0.6\n", "predict_params = np.linspace(0, 0.99, 100)\n", "confidence_level = 0.6\n", @@ -80,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 12, "id": "b415f516-7782-4d76-b304-3d630af365fc", "metadata": { "colab": { @@ -95,7 +95,7 @@ "output_type": "stream", "text": [ "N = 100\n", - "Metric = recall\n", + "Metric = precision\n", "Target level = 0.6\n", "Predict params = [0. 0.01 0.02 0.03 0.04 0.05 0.06 0.07 0.08 0.09 0.1 0.11 0.12 0.13\n", " 0.14 0.15 0.16 0.17 0.18 0.19 0.2 0.21 0.22 0.23 0.24 0.25 0.26 0.27\n", @@ -105,7 +105,7 @@ " 0.7 0.71 0.72 0.73 0.74 0.75 0.76 0.77 0.78 0.79 0.8 0.81 0.82 0.83\n", " 0.84 0.85 0.86 0.87 0.88 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97\n", " 0.98 0.99]\n", - "Confidence Level = 0.6\n" + "Confidence level = 0.6\n" ] } ], @@ -114,12 +114,12 @@ "print(f\"Metric = {metric}\")\n", "print(f\"Target level = {target_level}\")\n", "print(f\"Predict params = {predict_params}\")\n", - "print(f\"Confidence Level = {confidence_level}\")" + "print(f\"Confidence level = {confidence_level}\")" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 13, "id": "649f5ef0-3c5e-410a-949e-e7aa3142d5fc", "metadata": { "id": "649f5ef0-3c5e-410a-949e-e7aa3142d5fc" @@ -134,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 14, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -149,200 +149,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "[array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", - " 0.33]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", - " 0.33, 0.34]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", - " 0.33, 0.34]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", - " 0.33]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", - " 0.33, 0.34, 0.35, 0.36]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", - " 0.33, 0.34]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", - " 0.33, 0.34, 0.35, 0.36, 0.37, 0.38]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 ]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n", - " 0.33]), array([0. , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n", - " 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n", - " 0.22, 0.23, 0.24, 0.25, 0.26, 0.27])]\n", + "[array([], dtype=float64), array([0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.89, 0.9 , 0.91, 0.92, 0.93]), array([0.95, 0.96]), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([0.98, 0.99]), array([0.97, 0.98, 0.99]), array([0.99]), array([0.99]), array([0.98, 0.99]), array([0.87, 0.88, 0.89, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([0.98]), array([], dtype=float64), array([], dtype=float64), array([0.99]), array([0.99]), array([], dtype=float64), array([0.82, 0.83, 0.84, 0.85, 0.86, 0.87, 0.88, 0.89, 0.9 , 0.92, 0.93]), array([], dtype=float64), array([], dtype=float64), array([0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([0.99]), array([0.99]), array([0.95, 0.96, 0.97, 0.98, 0.99]), array([0.97, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([0.97, 0.98, 0.99]), array([0.83, 0.84, 0.85]), array([0.94, 0.95, 0.96, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([0.95, 0.96]), array([0.89, 0.9 , 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.99]), array([0.97, 0.98, 0.99]), array([], dtype=float64), array([0.99]), array([], dtype=float64), array([0.97, 0.98, 0.99]), array([0.99]), array([0.98, 0.99]), array([0.99]), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([0.99]), array([], dtype=float64), array([0.92, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.99]), array([0.99]), array([], dtype=float64), array([0.95, 0.96, 0.97]), array([0.98, 0.99]), array([], dtype=float64), array([0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([0.97, 0.98, 0.99]), array([0.97, 0.98, 0.99]), array([0.99]), array([], dtype=float64), array([0.89, 0.9 , 0.91, 0.92, 0.93, 0.99]), array([0.77, 0.78, 0.79, 0.82, 0.83, 0.84, 0.85, 0.86, 0.89, 0.9 , 0.91,\n", + " 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([0.89, 0.9 , 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([0.99]), array([0.99]), array([], dtype=float64), array([0.99]), array([0.97, 0.98, 0.99]), array([], dtype=float64), array([0.97, 0.98, 0.99]), array([], dtype=float64), array([0.98, 0.99]), array([], dtype=float64), array([0.99]), array([0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([0.99]), array([0.83, 0.84, 0.85, 0.86, 0.87, 0.88, 0.89, 0.9 , 0.91, 0.92, 0.93,\n", + " 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.99]), array([], dtype=float64), array([0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64)]\n", "Risk controlled! Proportion of actual valid parameters: 1.0\n" ] } @@ -385,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 15, "id": "42f85519-17ae-46c7-bfdc-e26c1d87017a", "metadata": { "id": "42f85519-17ae-46c7-bfdc-e26c1d87017a" @@ -395,8 +204,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "All valid thresholds = [0. 0.01 0.02 ... 0.25 0.26 0.27]\n", - "Theoretical value = 0.19999999999999996\n" + "All valid thresholds = [0.94 0.95 0.96 0.97 0.98 0.99 0.89 0.9 0.91 0.92 0.93 0.95 0.96 0.98\n", + " 0.99 0.97 0.98 0.99 0.99 0.99 0.98 0.99 0.87 0.88 0.89 0.93 0.94 0.95\n", + " 0.96 0.97 0.98 0.99 0.98 0.99 0.99 0.82 0.83 0.84 0.85 0.86 0.87 0.88\n", + " 0.89 0.9 0.92 0.93 0.98 0.99 0.99 0.99 0.95 0.96 0.97 0.98 0.99 0.97\n", + " 0.98 0.99 0.97 0.98 0.99 0.83 0.84 0.85 0.94 0.95 0.96 0.98 0.99 0.95\n", + " 0.96 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99 0.99 0.97\n", + " 0.98 0.99 0.99 0.97 0.98 0.99 0.99 0.98 0.99 0.99 0.99 0.92 0.95 0.96\n", + " 0.97 0.98 0.99 0.99 0.99 0.95 0.96 0.97 0.98 0.99 0.94 0.95 0.96 0.97\n", + " 0.98 0.99 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99 0.97 0.98 0.99 0.97\n", + " 0.98 0.99 0.99 0.89 0.9 0.91 0.92 0.93 0.99 0.77 0.78 0.79 0.82 0.83\n", + " 0.84 0.85 0.86 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99\n", + " 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99 0.99 0.99 0.99\n", + " 0.97 0.98 0.99 0.97 0.98 0.99 0.98 0.99 0.99 0.95 0.96 0.97 0.98 0.99\n", + " 0.95 0.96 0.97 0.98 0.99 0.99 0.83 0.84 0.85 0.86 0.87 0.88 0.89 0.9\n", + " 0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99 0.99 0.94 0.95 0.96 0.97\n", + " 0.98 0.99]\n", + "Theoretical value = 0.5\n" ] } ], diff --git a/mapie/risk_control_draft.py b/mapie/risk_control_draft.py index 3c96e0e79..1c3b7d7b8 100644 --- a/mapie/risk_control_draft.py +++ b/mapie/risk_control_draft.py @@ -120,7 +120,6 @@ def calibrate(self, X_calibrate: ArrayLike, y_calibrate: ArrayLike) -> None: # Minimum in case of precision control only self.best_threshold = min(self.valid_thresholds) - def predict(self, X_test: ArrayLike) -> NDArray: """ Predict binary labels on the test set, using the best threshold found From d94ea805dee6e148dd1df27bedf634ed08e3d597 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Mon, 28 Jul 2025 09:25:35 +0200 Subject: [PATCH 05/18] ENH: theoretical tests notebook --- ...risk_control_theoretical_tests_proto.ipynb | 164 +++++------------- 1 file changed, 46 insertions(+), 118 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 1867c7193..947710a0c 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 47, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -32,26 +32,31 @@ "source": [ "import numpy as np\n", "import itertools\n", + "from matplotlib import pyplot as plt\n", + "from collections import Counter\n", "\n", "from mapie.risk_control_draft import BinaryClassificationController" ] }, { "cell_type": "code", - "execution_count": 3, - "id": "1fef2dc6-b5b1-43bc-ad05-e5e1fe7844bd", - "metadata": { - "id": "1fef2dc6-b5b1-43bc-ad05-e5e1fe7844bd" - }, + "execution_count": 48, + "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", + "metadata": {}, "outputs": [], "source": [ "class RandomClassifier:\n", " def __init__(self, seed=42, threshold=0.5):\n", - " self.random_state = np.random.RandomState(seed)\n", + " self.seed = seed\n", " self.threshold = threshold\n", "\n", + " def _get_prob(self, x):\n", + " local_seed = hash((x, self.seed)) % (2**32)\n", + " rng = np.random.RandomState(local_seed)\n", + " return np.round(rng.rand(), 2)\n", + "\n", " def predict_proba(self, X):\n", - " probs = np.round(self.random_state.rand(len(X)), 2)\n", + " probs = np.array([self._get_prob(x) for x in X])\n", " return np.vstack([1 - probs, probs]).T\n", "\n", " def predict(self, X):\n", @@ -61,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 123, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -71,70 +76,16 @@ "N = 100 # size of the calibration set\n", "p = 0.5 # proportion of positives in the calibration set\n", "metric = \"precision\"\n", - "target_level = 0.6\n", + "target_level = 0.8\n", "predict_params = np.linspace(0, 0.99, 100)\n", - "confidence_level = 0.6\n", + "confidence_level = 0.7\n", "\n", "n_repeats = 100" ] }, { "cell_type": "code", - "execution_count": 12, - "id": "b415f516-7782-4d76-b304-3d630af365fc", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "b415f516-7782-4d76-b304-3d630af365fc", - "outputId": "3ff7579e-f564-49fb-9e95-ef78fa4239d0" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "N = 100\n", - "Metric = precision\n", - "Target level = 0.6\n", - "Predict params = [0. 0.01 0.02 0.03 0.04 0.05 0.06 0.07 0.08 0.09 0.1 0.11 0.12 0.13\n", - " 0.14 0.15 0.16 0.17 0.18 0.19 0.2 0.21 0.22 0.23 0.24 0.25 0.26 0.27\n", - " 0.28 0.29 0.3 0.31 0.32 0.33 0.34 0.35 0.36 0.37 0.38 0.39 0.4 0.41\n", - " 0.42 0.43 0.44 0.45 0.46 0.47 0.48 0.49 0.5 0.51 0.52 0.53 0.54 0.55\n", - " 0.56 0.57 0.58 0.59 0.6 0.61 0.62 0.63 0.64 0.65 0.66 0.67 0.68 0.69\n", - " 0.7 0.71 0.72 0.73 0.74 0.75 0.76 0.77 0.78 0.79 0.8 0.81 0.82 0.83\n", - " 0.84 0.85 0.86 0.87 0.88 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97\n", - " 0.98 0.99]\n", - "Confidence level = 0.6\n" - ] - } - ], - "source": [ - "print(f\"N = {N}\")\n", - "print(f\"Metric = {metric}\")\n", - "print(f\"Target level = {target_level}\")\n", - "print(f\"Predict params = {predict_params}\")\n", - "print(f\"Confidence level = {confidence_level}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "649f5ef0-3c5e-410a-949e-e7aa3142d5fc", - "metadata": { - "id": "649f5ef0-3c5e-410a-949e-e7aa3142d5fc" - }, - "outputs": [], - "source": [ - "X_calibrate = list(range(1, N+1))\n", - "y_calibrate = [1] * int(p*N) + [0] * (N - int(p*N))\n", - "np.random.seed(42)\n", - "np.random.shuffle(y_calibrate)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, + "execution_count": 124, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -149,15 +100,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "[array([], dtype=float64), array([0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.89, 0.9 , 0.91, 0.92, 0.93]), array([0.95, 0.96]), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([0.98, 0.99]), array([0.97, 0.98, 0.99]), array([0.99]), array([0.99]), array([0.98, 0.99]), array([0.87, 0.88, 0.89, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([0.98]), array([], dtype=float64), array([], dtype=float64), array([0.99]), array([0.99]), array([], dtype=float64), array([0.82, 0.83, 0.84, 0.85, 0.86, 0.87, 0.88, 0.89, 0.9 , 0.92, 0.93]), array([], dtype=float64), array([], dtype=float64), array([0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([0.99]), array([0.99]), array([0.95, 0.96, 0.97, 0.98, 0.99]), array([0.97, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([0.97, 0.98, 0.99]), array([0.83, 0.84, 0.85]), array([0.94, 0.95, 0.96, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([0.95, 0.96]), array([0.89, 0.9 , 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.99]), array([0.97, 0.98, 0.99]), array([], dtype=float64), array([0.99]), array([], dtype=float64), array([0.97, 0.98, 0.99]), array([0.99]), array([0.98, 0.99]), array([0.99]), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([], dtype=float64), array([0.99]), array([], dtype=float64), array([0.92, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.99]), array([0.99]), array([], dtype=float64), array([0.95, 0.96, 0.97]), array([0.98, 0.99]), array([], dtype=float64), array([0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([0.97, 0.98, 0.99]), array([0.97, 0.98, 0.99]), array([0.99]), array([], dtype=float64), array([0.89, 0.9 , 0.91, 0.92, 0.93, 0.99]), array([0.77, 0.78, 0.79, 0.82, 0.83, 0.84, 0.85, 0.86, 0.89, 0.9 , 0.91,\n", - " 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([0.89, 0.9 , 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([], dtype=float64), array([0.99]), array([0.99]), array([], dtype=float64), array([0.99]), array([0.97, 0.98, 0.99]), array([], dtype=float64), array([0.97, 0.98, 0.99]), array([], dtype=float64), array([0.98, 0.99]), array([], dtype=float64), array([0.99]), array([0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64), array([0.99]), array([0.83, 0.84, 0.85, 0.86, 0.87, 0.88, 0.89, 0.9 , 0.91, 0.92, 0.93,\n", - " 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([0.99]), array([], dtype=float64), array([0.94, 0.95, 0.96, 0.97, 0.98, 0.99]), array([], dtype=float64)]\n", - "Risk controlled! Proportion of actual valid parameters: 1.0\n" + "Number of valid thresholds according to LTT across all iterations: 105\n", + "Number of actual valid thresholds across all iterations: 0\n" ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAHHCAYAAACle7JuAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAN0lJREFUeJzt3QeUU2X6x/Fn6L33XldAioKKFEUpi4oIgqsgSxPBRZD2pysgoIKNJs1CURZFkaLCiosIiBTpfUFEEFZh6F0GhPs/z7snOUkmw2RCQvJmvp9zIs7Nzc1735T7y1vujXMcxxEAAAALpYl0AQAAAIJFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAZAi+/btk7/+9a+SM2dOiYuLk4ULF0o0eeCBB6Ry5cqRLgYiqFSpUtKhQ4dIFwO3CEEGYbNr1y75+9//LkWLFpWMGTNKkSJFpE2bNmY57NW+fXvZsWOHvPrqqzJr1iy56667Il0khNDkyZNl5syZkS4GELB0ga8KBG7+/PnSunVryZMnj3Tq1ElKly4tBw8elGnTpsnnn38uc+bMkccffzzSxUQK/fHHH7J27Vp58cUXpXv37pEuDsIUZPLly0eLBqxBkEHI7d+/X9q2bStlypSR77//XvLnz+++r2fPnnLfffeZ+7dv327WiSaXLl2SLFmyJFr+559/yvXr1yVDhgySmh0/ftz8mytXrkgXxWoXL16UrFmzSmrB5wfhRNcSQu7NN980geC9997zCjFKf+m9++675ov8jTfe8Lrvt99+M6032gWlXVHaitO1a1e5cuWKe50zZ85I7969TR+4rlOsWDFp166dnDhxwtyvTeI6bkNbfzytWLHCLNd/fcdSbNq0Se6//34TYAYPHmweq+u+9dZbMm7cOClbtqx5rt27d5vH7dmzR5544gnT2pQpUybTtfLll196PZ+rHKtXr5Y+ffqYetADl7ZCucKAp6+//lrq1asn2bNnlxw5csjdd98tH3/8sdc6P/74ozz00ENmbIqWVdfX7Xs6f/689OrVy10/BQoUkEaNGsnmzZuTfd22bNkiDz/8sHn+bNmySYMGDWTdunXu+19++WUpWbKk+f9+/fqZ/dPnuZGEhAQZNmyYlCtXzpSnePHi0r9/f7Pc04wZM6R+/fqmvLpepUqVZMqUKX63GUhdKX29HnzwQVNX2r3p+35Liu6XtjbNnj1bbrvtNvMa16hRw4RyT7/++qs8//zzZp3MmTNL3rx55W9/+1ui957rvbBy5Uqzvu6jvm+D2cYPP/wgPXr0MO8nDZPPPfec+Xzo50I/B7lz5zY3rWPHcby2oUFC38+333672aeCBQuax58+fdq9jr6e2vWrZdXn05t+Tlz0efT9pa+jvk76ur7++utm2y7JfX586WdQXydfuk193fSz5qLbrF27tqknrS99XbSFNzn63tUy+Urq+0LfY/qDSz+z+j5r0qQJXeJRjBYZhNxXX31lvhD1i8AfDQ16/+LFi93Lfv/9d7nnnnvMF2WXLl2kQoUKJtjol5SGIv0ld+HCBbPN//znP/LMM89I9erVTYDREPHf//7XhKSUOnnypDl4t2rVyozn0S93z4Pr5cuXTXn0i1iDi36Z1alTx3zBDhw40HzRffbZZ9K8eXOZN29eou6yF154wRxY9GCuX5b6xa4HyU8//dTry1T3Rw8wgwYNMgcoDRVLliyRp59+2qzz3XffmXLqF7duK02aNO6D/6pVq0zdqX/84x+mzvQ5NAzo/unBT+tM6yspul9atxoM9CCYPn16Ezj1IKYHtZo1a0qLFi1M2TRIarfhI488YgJPUvRA9Nhjj5nn1zqsWLGiGVszduxY+emnn7wGCWto0f3X9dOlS2feQ3qA121069YtRXWl9OCsoU/L/OSTT5o6GTBggFSpUsXUY3J0n/U10tCgr712t+j21q9f7x5IvGHDBlmzZo1572gw0ddX90PrTA/avi17uj8aQIYOHWqCfDDb0PdToUKFZPjw4SZk6o8FrQPdRokSJeS1116Tf/3rX+bHhJZTw42Lhhatv44dO5r9OnDggEycONHUnwZifc31/anPoa+rdh8q12dCP4caIPVzqdvS59Pn1dfhyJEj5rGe/H1+/HnqqadM0Dh69KjZNxd93+j3gtaNy/jx4817RMfaaYDTLmoNfosWLTJhIxR03JeOA2vcuLEJabrf+prUrVvX1FVy4R0R4AAhdObMGf0Z6DRr1uyG6z322GNmvXPnzpm/27Vr56RJk8bZsGFDonWvX79u/h06dKh5zPz585NcZ8aMGWadAwcOeN2/fPlys1z/dalXr55ZNnXqVK919bG6PEeOHM6xY8e87mvQoIFTpUoV5/Lly17PXbt2bad8+fLuZa5yNGzY0F021bt3bydt2rSmnlz1lT17dqdmzZrOH3/84Xef9F/dduPGjb22denSJad06dJOo0aN3Mty5szpdOvWzUmp5s2bOxkyZHD279/vXvb777+bst1///2J6ubNN99MdpuzZs0yr+mqVau8lmt96zZWr17ttS++dH/LlCnj/juQuvJ8XT/66CP3soSEBKdQoUJOy5Ytky23PlZvGzdudC/79ddfnUyZMjmPP/74Dcu8du3aRM/tei/UrVvX+fPPP73WT+k2fN8DtWrVcuLi4px//OMf7mX6HMWKFTP14KKvgT5+9uzZXs+1ZMmSRMtvv/12r8e6jBw50smaNavz008/eS0fOHCgeU8fOnQo2c+PP3v37jXrv/POO17Ln3/+eSdbtmxedeRbX1euXHEqV67s1K9f32t5yZIlnfbt27v/HjZsmHkOX77fF+fPn3dy5crldO7c2Wu9o0ePms+W73JEB7qWEFLataG0OfZGXPefO3fO/OrWX+dNmzb1OwPG1SSsLR7VqlXzO0jYX7NxIPSXov5C9adly5ZeXWOnTp0yLSP6C1/3U1uD9KatHvrrTacl669VT/pr1LNs2upx7do106Wgli5daralrTva3O9vn7Zu3Wq2rS0O+lyu59Vf9dr9o10erqZ9/XWuXVD6SzZQWp5///vfplXJc8xS4cKFzXPqL2N9nVJq7ty5phVGW9dcZdabtiKp5cuXu9fVbgKXs2fPmvX01/8vv/xi/g60rly0RUFb2Fy0RU9brXR7gahVq5Zp/XLR1odmzZrJN998Y+rLt8xXr141r412tehr4K8rr3PnzpI2bVqvZSndhna9eu6rtpRp9tLlLvoc+jny3Fd9LbRLUrsZPV8L3UetK8/XIim6DX3/aguj5zYaNmxo6sS3683385OUv/zlL3LHHXd4tVLq9rQVTb8TPOvI8/+11U3fG1qmQLpOA6HvMW0V1hZHz33UOtW6DqSecOvRtYSQcgUUV6AJJPDomBE9UCZ37g8dRKxfjqGkXURJDUDUMTqefv75Z3PQGDJkiLn5c+zYMbNNzwOgJz0IKNe4BN0ndaN91xCjtLk7KfqFrtvWcSC6no5h0IOUdv9o98KNBlVr/WvzuY7T8KVBREPS4cOHTXdOSmi5tUsrqYOZ1pWLdm1ol5nOiNKy+O6bHoQDqSsX7abxDTdaPzrAPBDly5f3e8DVsml9aReIzuAaNWqU6ULRAOs5JsUVvm70flIp3Ybv+0nrRenr7bvcc+yLvha6PR2fk9xrkRTdhtZfIK9nUvubFO1e0vFpWgf6+dGxbLo9Xe5Ju5BeeeUVE+49x1kF+0Mmqc+aK2z70q5XRB+CDEJKv0D1l3xyBwy9X7+w9ItBv8xDJakvNNevaF+ev/CSu8/V6tG3b1/TAuOP/pr25PsL3MV3IOaNuJ5Xxz3oL1d/XGNVtLVIf6EuWLDAtLLoY7SfX6fDBzI2JJS03DomZcyYMX7vdx18NaBoy5K23Oi6ulzDpY710PE0ngNJAxWKek+OjiXRAKKDX7UFx3WCQB3T4a/M/t5rKd1GUvvlb7nnvuq2NMToAGZ/Amk50W1oi46OofJHg16gny1fGlh0rI22+mhd6LgzrQsdl+SiY8F0fIyOsdMxS/o9o+N6tP78DfYO5nvBVec6TsZzvI6Ljt9C9OFVQcg9+uij8v7775suCR0g50u/kHRQow4YdH2JaqDZuXPnDbersx+SW8fV4qHNw55cXTk3w9WqoV+e2pweCrpPSvfLNwT5rqN1FMjz6he8DizVm/6q1UG+evK6pIKM1r8OKt27d2+i+3SGlg4s9v3FHwgt97Zt20xIudEvZh3Yq7+uddC2Z4uDbzN+IHUVKq5f5p50gLLWk+ugr10f2vr19ttvu9fRwa2+770bCcU2AqF19+2335qB6skFjKReK92GDrgP1Xvft/VGu/60e0kHqmvw1q5O7fp10a5l7VLU7j3P5RpkkuP5veB56gDf7wXXe0xDXzj2E+HBGBmEnE7N1S9LDSra5+9Jx5nozBo9IOh6Sg+U+qWlB7SNGzcm+ctSu5X0wKitDUmt4/oi8uyv119dOrvjZumXm84m0dk8OkvDl79p1cnRU/1r95p2L+gBzN8+aReR7pdOPdUDSVLPq/vp2x2hZdbp7L7TnX1/zWs5vvjiC69pqPHx8eaXrobRYJrUtXVIuwo01PrSVjjXzB1Xa4Jvt4rvASqQugoV7eLyHHehXWtaP1oGV3n1X9/nfeedd5Js/fMnFNsI9LXQbY4cOdLvOV48g5POxPMXpHQbWi8aJHzp+rqdm6GtMjoTa/r06WZcim+3ktaVhizPutH3ayCXyPD3vaDvvw8//NBrPW1p1fe6zv7SMUuh+Iwj/GiRQcjp+AL9gtApktq14HtmX/2S+uSTT9xfLkq/OLQrRAd4uqbqaljQpmZt2dFfURp89BesTrfUKbh6gNdgpL/kp06dagYC6ziOe++91zRT63065VOnaN7sl6zLpEmTzIFd90sHb2orjR7w9Qtep4Br0EoJ/dLU7pNnn33WnA9FB9fqr0fdjo7H0HrUoPfBBx+YFhXdPx2crN1yGhK01UK3oSFQxx3p2BA974bWhXY36a9wneLr+YvfHx13oAMddd+0JUeb0DWwaQAK9PwrvvSkh9pFoMFVy6mtAXoQ0lYeXa4HRB2UquFAu5J0YKeGXw1rGn40hHkGxkDqKlR0HI4e1DynXyud9uzZ8qhdENoFolPd9T2g9a3nOAlUKLYRCP1cad1qCNTxJVrn2rKoLU/6GdNpza7ztejnSqcb63tCW770ddAxI/r508+allnP+qvraRjQKfX6udTPdzCnQPAMStptqzf93Pq2iOj0au161O4mfe21tVE/j1rG5LqydX+1tU+/i3Q/NBRpYNLWtUOHDnm9x3Tf9b2rLZnaxedaR08Xoe9hnbKOKBPpaVOIXdu3b3dat27tFC5c2EmfPr2Z/qp/79ixw+/6OsVVp2Hnz5/fyZgxo5l6q1OJdeqsy8mTJ53u3bs7RYsWNdOFdZqpTrM8ceKEex2dQqzTnnUbBQsWdAYPHuwsXbrU7/RrnWrqK7kpxrp9Lafuj+6XluXRRx91Pv/880TTOn2nk/ubBq6+/PJLM4U7c+bMZtrqPffc43zyySde62zZssVp0aKFkzdvXrNvOsX0ySefdJYtW2bu13rq16+fU61aNTNNWafK6v9PnjzZCcTmzZvN9F6d8polSxbnwQcfdNasWZOiuvGl02Nff/11U89a5ty5czs1atRwhg8f7pw9e9Zr/6tWrWqmOJcqVco8Zvr06X6n0idXV0m9rvo+0TpLjj6nvu/++c9/mmnvWu4777wz0Wt2+vRpp2PHjk6+fPlMnWnd7dmzJ9HU36TeC6HYhmta8fHjxxPtq77+vt577z1T/1p3+h7RUwn079/fTLX3nGrcpEkTc79u23Mqtk5PHjRokFOuXDnz+dNy62vx1ltvmdc6mPeIpzp16pjHPvvss37vnzZtmvs1qVChgqkXf1OrfetPbdq0yUzd13KXKFHCGTNmzA1P16CvhU651vdk2bJlnQ4dOnhNyUf0iNP/RDpMAUC00O4LPQkfv7wBOzBGBgAAWIsgAwAArEWQAQAA1mLWEgB4YNggYBdaZAAAgLUIMgAAwFoR7Vp6+eWXvU4wpfTCdXrCLKVn7/y///s/c0IzPTGXnqBKT0xVsGDBgJ9Dr52hVwLWM4KG6sJiAAAg/N28eqJPPTu5nhg0asfI6JlK9UyW/i7K1bt3b3M2Rdcl6PUaHC1atDBXyg2UhphgrhMDAAAiTy8Romctj9ogo8HF31VG9Vorejp7vdaL65Lqeu0VPXW9Xo9DT0MfCG2JcVUEl2AHAMAO586dMw0RruN41AYZvdaHNhvpVU31MvZ6LRC9JsamTZvMRbs8r7dRoUIFc59ejySpIKNdUJ4XyNNmKaUhhiADAIBdkhsWEtHBvjVr1pSZM2fKkiVLzIW6Dhw4IPfdd58JH0ePHjUXkvO85LrS8TF6X1I0CGk3lOtGtxIAALEroi0yejVfl6pVq5pgU7JkSXNl3MyZMwe1Tb3qcZ8+fRI1TQEAgNgTVdOvtfXlL3/5i/z8889m3MyVK1fkzJkzXuvEx8f7HVPjkjFjRnc3Et1JAADEtqgKMhcuXJD9+/dL4cKFpUaNGpI+fXpZtmyZ+/69e/fKoUOHzFgaAACAiHYt9e3bV5o2bWq6k3Sa9LBhwyRt2rTSunVrM76lU6dOppsoT548pmXlhRdeMCEm0BlLAAAgtkU0yPz3v/81oeXkyZOSP39+qVu3rplarf+vxo4da06C07JlS68T4gEAAKg4J8avkKaDfbV1R89Lw3gZAABi6/gdVWNkAAAAUoIgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwVkTP7AsAAIJXauDiSBdBDo5uEtHnp0UGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1oibIjB49WuLi4qRXr17uZZcvX5Zu3bpJ3rx5JVu2bNKyZUuJj4+PaDkBAED0iIogs2HDBnn33XelatWqXst79+4tX331lcydO1dWrlwpv//+u7Ro0SJi5QQAANEl4kHmwoUL0qZNG3n//fcld+7c7uVnz56VadOmyZgxY6R+/fpSo0YNmTFjhqxZs0bWrVsX0TIDAIDoEPEgo11HTZo0kYYNG3ot37Rpk1y9etVreYUKFaREiRKydu3aJLeXkJAg586d87oBAIDYlC6STz5nzhzZvHmz6VrydfToUcmQIYPkypXLa3nBggXNfUkZNWqUDB8+PCzlBQAA0SViLTKHDx+Wnj17yuzZsyVTpkwh2+6gQYNMt5Trps8DAABiU8SCjHYdHTt2TKpXry7p0qUzNx3QO2HCBPP/2vJy5coVOXPmjNfjdNZSoUKFktxuxowZJUeOHF43AAAQmyLWtdSgQQPZsWOH17KOHTuacTADBgyQ4sWLS/r06WXZsmVm2rXau3evHDp0SGrVqhWhUgMAgGgSsSCTPXt2qVy5steyrFmzmnPGuJZ36tRJ+vTpI3ny5DEtKy+88IIJMffee2+ESg0AAKJJRAf7Jmfs2LGSJk0a0yKjs5EaN24skydPjnSxAABAlIhzHMeRGKbTr3PmzGkG/jJeBgAQS0oNXBzpIsjB0U0ievyO+HlkAAAAgkWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAACB1BZnNmzfLjh073H9/8cUX0rx5cxk8eLBcuXIllOUDAAAIbZB57rnn5KeffjL//8svv0irVq0kS5YsMnfuXOnfv3/A25kyZYpUrVpVcuTIYW61atWSr7/+2n3/5cuXpVu3bpI3b17Jli2btGzZUuLj44MpMgAAiEFBBRkNMXfccYf5fw0v999/v3z88ccyc+ZMmTdvXsDbKVasmIwePVo2bdokGzdulPr160uzZs1k165d5v7evXvLV199ZZ5j5cqV8vvvv0uLFi2CKTIAAIhB6YJ5kOM4cv36dfP/3377rTz66KPm/4sXLy4nTpwIeDtNmzb1+vvVV181rTTr1q0zIWfatGkmIGnAUTNmzJCKFSua+++9995gig4AAFJ7i8xdd90lr7zyisyaNcu0lDRp0sQsP3DggBQsWDCogly7dk3mzJkjFy9eNF1M2kpz9epVadiwoXudChUqSIkSJWTt2rVBPQcAAIgtQbXIjBs3Ttq0aSMLFy6UF198UcqVK2eWf/7551K7du0UbUsHDWtw0fEwOg5mwYIFUqlSJdm6datkyJBBcuXK5bW+BqWjR48mub2EhARzczl37lyK9w8AAMRwkNEBup6zllzefPNNSZs2bYq2ddttt5nQcvbsWROE2rdvb1p5gjVq1CgZPnx40I8HAACp4DwyZ86ckQ8++EAGDRokp06dMst2794tx44dS9F2tNVFW3Rq1KhhQki1atVk/PjxUqhQITOVW5/Hk85a0vuSouXRUOS6HT58OMg9BAAAMdkis337dmnQoIHp9jl48KB07txZ8uTJI/Pnz5dDhw7JRx99FHSBdBCxdg1psEmfPr0sW7bMTLtWe/fuNdvXrqikZMyY0dwAAEDsCyrI9OnTRzp27ChvvPGGZM+e3b38kUcekaeffjrg7WjrycMPP2wG8J4/f97MUFqxYoV88803kjNnTunUqZN5Lg1Jep6ZF154wYQYZiwBAICgg8yGDRvk3XffTbS8aNGiNxyI60u7odq1aydHjhwxwUXH3miIadSokbl/7NixkiZNGtMio600jRs3lsmTJ/PKAQCA4IOMdt34mw2kJ8rLnz9/wNvR88TcSKZMmWTSpEnmBgAAEJLBvo899piMGDHCnOdFxcXFmbErAwYMcI9nAQAAiMog8/bbb8uFCxekQIEC8scff0i9evXMzCMdL6Nn5wUAAIjariUdz7J06VJZvXq1bNu2zYSa6tWre52FFwAAICqDjEudOnXMDQAAwJqupR49esiECRMSLZ84caL06tUrFOUCAAAIT5CZN2+e35YYvc6SXmYAAAAgaoPMyZMnzTgZX3rSuhMnToSiXAAAAOEJMjpDacmSJYmWf/3111KmTJlgNgkAAHDrLlHQvXt3OX78uNSvX98s02si6bTscePGBbNJAACAWxNknnnmGXPJAD1nzMiRI82yUqVKyZQpU8wlBwAAAKJ6+nXXrl3NTVtlMmfOLNmyZQttyQAAAMJ5HhmVkmsrAQAARHywb3x8vLRt21aKFCki6dKlk7Rp03rdAAAAorZFpkOHDuYikUOGDJHChQubi0YCAABYEWR++OEHWbVqldxxxx2hLxEAAEA4u5aKFy8ujuME81AAAIDIBhk9V8zAgQPl4MGDoS8RAABAOLuWnnrqKbl06ZKULVtWsmTJIunTp/e6/9SpU8FsFgAAIPxBhrP3AgAAa4NM+/btQ18SAACAWzFGRu3fv19eeuklad26tRw7dsx90chdu3YFu0kAAIDwB5mVK1dKlSpV5Mcff5T58+fLhQsXzPJt27bJsGHDgtkkAADArQkyOmPplVdekaVLl0qGDBncy/VK2OvWrQtmkwAAALcmyOzYsUMef/zxRMsLFCggJ06cCGaTAAAAtybI5MqVS44cOZJo+ZYtW6Ro0aLBbBIAAODWBJlWrVrJgAED5OjRo+Y6S9evX5fVq1dL3759pV27dsFsEgAA4NYEmddee00qVKhgLlWgA30rVaok999/v9SuXdvMZAIAAIjK88joNZa0JWbChAkydOhQM15Gw8ydd94p5cuXD08pAQAAQhVkypUrZ84Xo8FFW2UAAACs6FpKkyaNCTAnT54MT4kAAADCOUZm9OjR0q9fP9m5c2cwDwcAAIjctZZ0ZpJe/bpatWrmhHiZM2f2up+rXwMAgFuBq18DAIDUE2SuXr1qrrU0ZMgQKV26dHhKBQAAEI4xMunTp5d58+al9GEAAADRMdi3efPmsnDhwtCXBgAAINxjZHT69YgRI8xlCWrUqCFZs2b1ur9Hjx7BbBYAACD8QWbatGnmwpGbNm0yN0967SWCDAAAiNogc+DAgdCXBAAA4FaMkQEAALC2ReaZZ5654f3Tp08PtjwAAADhDTKnT59OdG4ZvVzBmTNnpH79+sFsEgAA4NYEmQULFiRadv36denatauULVs2mE0CAABEboyMXhW7T58+Mnbs2FBtEgAA4NYN9t2/f7/8+eefodwkAABAaLuWtOXFk+M4cuTIEVm8eLG0b98+mE0CAADcmiCzZcuWRN1K+fPnl7fffjvZGU0AAAARDTLLly8PWQEAAABu6RgZPbPvvn37Ei3XZQcPHgy6MAAAAGEPMh06dJA1a9YkWv7jjz+a+wAAAKI2yOgYmTp16iRafu+998rWrVtDUS4AAIDwBBm9wvX58+cTLT979qxcu3YtmE0CAADcmiBz//33y6hRo7xCi/6/Lqtbt24wmwQAALg1s5Zef/11E2Zuu+02ue+++8yyVatWyblz5+S7774LZpMAAAC3pkWmUqVKsn37dnnyySfl2LFjppupXbt2smfPHqlcuXIwmwQAALg1LTKqSJEi8tprrwX7cAAAgMi0yMyYMUPmzp2baLku+/DDD2++VAAAAOEKMjqoN1++fImWFyhQgFYaAAAQ3UHm0KFDUrp06UTLS5Ysae4DAACI2iCjLS862NfXtm3bJG/evKEoFwAAQHiCTOvWraVHjx7m4pF6/hi96bTrnj17SqtWrYLZJAAAwK2ZtTRy5EhzccgGDRpIunT/24SGmfbt2zNGBgAARHeQyZAhg3z66afSt29fE2gyZ84sVapUMWNkAAAAorZr6cyZM9KtWzcza0kvEqldSXrF6zfffNPcl9LZT3fffbdkz57djLtp3ry57N2712udy5cvm+fTsTfZsmWTli1bSnx8fEqLDQAAUnuLzKlTp6RWrVry22+/SZs2baRixYpm+e7du2XmzJmybNkyWbNmjeTOnTug7a1cudKEFA0zf/75pwwePFj++te/mu1lzZrVrNO7d29ZvHixOUdNzpw5pXv37tKiRQtZvXp1MPsLAABiSJzjOE6gK/fq1cuElW+//VYKFizodd/Ro0dNCNFxM2PHjg2qMMePHzctMxpw9FpOejXt/Pnzy8cffyxPPPGEWUcvg6ABau3ataZFKDl6/ScNQLqtHDlyBFUuAACiUamBiyNdBDk4uklYthvo8TtFXUsLFy6Ut956K1GIUYUKFZI33nhDFixYEFyJRUxhVZ48ecy/mzZtkqtXr0rDhg3d61SoUEFKlChhgow/CQkJZuc9bwAAIDalKMgcOXJEbr/99iTv1wtGastMMK5fv25afOrUqeO+8KRuSwcW58qVy2tdDVJJPY+Ou9EE57oVL148qPIAAIAYCzI6wFdnKSXlwIED7taUlNKxMjt37pQ5c+bIzRg0aJBp2XHdDh8+fFPbAwAAMRJkGjduLC+++KJcuXLFb5fOkCFD5KGHHkpxIXQA76JFi8wJ9ooVK+bVXaXP5TsbSmct6X3+ZMyY0fSled4AAEBsStGspREjRshdd90l5cuXNy0oOl5Fxwr/5z//kcmTJ5swM2vWrIC3p4994YUXzLiaFStWJLp+U40aNSR9+vRmgLFOu1Y6PVuv56SzpwAAQOqWoiCjrSU6yPb55583XTiuCU9xcXHSqFEjmThxYorGpGgY0hlJX3zxhTmXjGvci45t0ZPs6b+dOnWSPn36mC4rbV3R4KMhJpAZSwAAILal+My+2mry9ddfy+nTp2Xfvn1mWbly5YIaGzNlyhTz7wMPPOC1fMaMGeYke0qncqdJk8a0yGiLj3ZvaesPAABAis4jYyPOIwMAiFWlOI9McFe/BgAAiAYEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWCtdpAsAAEg9Sg1cLNHg4OgmMVFO0CIDAAAsRpABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKwV0SDz/fffS9OmTaVIkSISFxcnCxcu9LrfcRwZOnSoFC5cWDJnziwNGzaUffv2Ray8AAAgukQ0yFy8eFGqVasmkyZN8nv/G2+8IRMmTJCpU6fKjz/+KFmzZpXGjRvL5cuXb3lZAQBA9InotZYefvhhc/NHW2PGjRsnL730kjRr1sws++ijj6RgwYKm5aZVq1a3uLQAACDaRO0YmQMHDsjRo0dNd5JLzpw5pWbNmrJ27dokH5eQkCDnzp3zugEAgNgUtUFGQ4zSFhhP+rfrPn9GjRplAo/rVrx48bCXFQAAREbUBplgDRo0SM6ePeu+HT58ONJFAgAAqS3IFCpUyPwbHx/vtVz/dt3nT8aMGSVHjhxeNwAAEJuiNsiULl3aBJZly5a5l+l4F529VKtWrYiWDQAARIeIzlq6cOGC/Pzzz14DfLdu3Sp58uSREiVKSK9eveSVV16R8uXLm2AzZMgQc86Z5s2bR7LYAAAgSkQ0yGzcuFEefPBB9999+vQx/7Zv315mzpwp/fv3N+ea6dKli5w5c0bq1q0rS5YskUyZMkWw1AAAIFpENMg88MAD5nwxSdGz/Y4YMcLcAAAAfEXtGBkAAIDkEGQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANaK6AnxAMAGpQYujnQR5ODoJjFTTiCUaJEBAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsFa6SBcAQOiVGrhYosHB0U1iopwAohctMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWCtdpAsA2KbUwMWRLoIcHN0k0kUAgKhAiwwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLWYfm35NNxApuJSzsAxrRkA7EKLDAAAsBZBBgAAWMuKIDNp0iQpVaqUZMqUSWrWrCnr16+PdJEAAEAUiPog8+mnn0qfPn1k2LBhsnnzZqlWrZo0btxYjh07FumiAQCACIv6IDNmzBjp3LmzdOzYUSpVqiRTp06VLFmyyPTp0yNdNAAAEGFRHWSuXLkimzZtkoYNG7qXpUmTxvy9du3aiJYNAABEXlRPvz5x4oRcu3ZNChYs6LVc/96zZ4/fxyQkJJiby9mzZ82/586dC3n5ridckmiQ3L5RzsAF8j6xoZzRUEZFOUOH92ZoUc7QCcfx1XO7juPceEUniv32229aemfNmjVey/v16+fcc889fh8zbNgw8xhu3Lhx48aNm1h/O3z48A2zQlS3yOTLl0/Spk0r8fHxXsv170KFCvl9zKBBg8zgYJfr16/LqVOnJG/evBIXFxf2MsciTcXFixeXw4cPS44cOSJdHKtRl6FFfYYOdRla1OfN05aY8+fPS5EiRW64XlQHmQwZMkiNGjVk2bJl0rx5c3cw0b+7d+/u9zEZM2Y0N0+5cuW6JeWNdfph5AMZGtRlaFGfoUNdhhb1eXNy5syZ7DpRHWSUtq60b99e7rrrLrnnnntk3LhxcvHiRTOLCQAApG5RH2SeeuopOX78uAwdOlSOHj0qd9xxhyxZsiTRAGAAAJD6RH2QUdqNlFRXEsJPu+r0hIS+XXZIOeoytKjP0KEuQ4v6vHXidMRvpAsBAAAQcyfEAwAAuBGCDAAAsBZBBgAAWIsgAwAArEWQSYUmTZokpUqVkkyZMknNmjVl/fr1Sa579epVGTFihJQtW9asX61aNTP9/Wa2GWtCXZ+jRo2Su+++W7Jnzy4FChQwJ4Pcu3evpAbheG+6jB492pzdu1evXpJahKM+f/vtN/n73/9uzpaeOXNmqVKlimzcuFFiXajrUq8jOGTIECldurSpR1135MiRyV9XCImF8tpIiH5z5sxxMmTI4EyfPt3ZtWuX07lzZydXrlxOfHy83/X79+/vFClSxFm8eLGzf/9+Z/LkyU6mTJmczZs3B73NWBKO+mzcuLEzY8YMZ+fOnc7WrVudRx55xClRooRz4cIFJ5aFoy5d1q9f75QqVcqpWrWq07NnTyc1CEd9njp1yilZsqTToUMH58cff3R++eUX55tvvnF+/vlnJ5aFoy5fffVVJ2/evM6iRYucAwcOOHPnznWyZcvmjB8//hbuWWwgyKQyerHNbt26uf++du2a+cCNGjXK7/qFCxd2Jk6c6LWsRYsWTps2bYLeZiwJR336OnbsmLlw2sqVK51YFq66PH/+vFO+fHln6dKlTr169VJNkAlHfQ4YMMCpW7euk9qEoy6bNGniPPPMMzdcB4GhaykVuXLlimzatEkaNmzoXpYmTRrz99q1a/0+JiEhwTSNetJm0B9++CHobcaKcNSnP2fPnjX/5smTR2JVOOuyW7du0qRJE69tx7pw1eeXX35pLhfzt7/9zXR73nnnnfL+++9LLAtXXdauXdtcN/Cnn34yf2/bts3c//DDD4dtX2IVQSYVOXHihOmX9b28g/6tl3/wp3HjxjJmzBjZt2+fuWDn0qVLZf78+XLkyJGgtxkrwlGfvnQdHdNRp04dqVy5ssSqcNXlnDlzZPPmzWbcUWoSrvr85ZdfZMqUKVK+fHn55ptvpGvXrtKjRw/58MMPJVaFqy4HDhworVq1kgoVKkj69OlNKNTPeps2bcK+T7GGIIMbGj9+vPnS0g+bXo1cLxWhF+zUXyQIf31qa8LOnTvNARkpq8vDhw9Lz549Zfbs2Yl+HSO496YelKtXry6vvfaaOfB26dJFOnfuLFOnTo1o2W2sy88++8y8Nz/++GMTtjUMvvXWWzEdCsOFo1Eqki9fPkmbNq3Ex8d7Lde/CxUq5Pcx+fPnl4ULF5orjv/666+yZ88eyZYtm5QpUybobcaKcNSnJ/3yW7RokSxfvlyKFSsmsSwcdandAceOHTMH3nTp0pnbypUrZcKECeb/9Vd2rArXe7Nw4cJSqVIlr8dVrFhRDh06JLEqXHXZr18/d6uMzvxq27at9O7dO9W1HoYCQSYV0V8GNWrUMP2ynr+w9O9atWrd8LH6i7Zo0aLy559/yrx586RZs2Y3vU3bhaM+lQ7C1xCzYMEC+e6778z0zFgXjrps0KCB7NixQ7Zu3eq+6fgObbrX/9eDU6wK13tTuzh9TwWgYzxKliwpsSpcdXnp0qVELbH6ntRtI4UCHBSMGJpGmDFjRmfmzJnO7t27nS5duphphEePHjX3t23b1hk4cKB7/XXr1jnz5s0zUwi///57p379+k7p0qWd06dPB7zNWBaO+uzatauTM2dOZ8WKFc6RI0fct0uXLjmxLBx16Ss1zVoKR33qNPZ06dKZqcP79u1zZs+e7WTJksX55z//6cSycNRl+/btnaJFi7qnX8+fP9/Jly+fmbqNlCHIpELvvPOOOS+JnhdBpxXqh87zi14/YC56MK1YsaL5EOs5D/QD+9tvv6Vom7Eu1PWpvy/83fTcMrEuHO/N1BpkwlWfX331lVO5cmWzXoUKFZz33nvPSQ1CXZfnzp0z70Xdpp5jpkyZMs6LL77oJCQk3NL9igVx+p+UtuIAAABEA8bIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABgBCLi4sz19oBEH4EGSCGdejQwRxU9abXjClXrpyMGDHCXPvFVoQEAJ7Sef0FIOY89NBDMmPGDElISJB//etf0q1bN0mfPr0MGjQoxdvSK0ZrkPC92J2Nrl69auoBgN3s/zYCcEMZM2aUQoUKmSsUd+3aVRo2bChffvmluW/MmDFSpUoVyZo1qxQvXlyef/55uXDhgvuxM2fOlFy5cpn1K1WqZLZ16NAh2bBhgzRq1Ejy5csnOXPmlHr16snmzZu9nlcDz7vvviuPPvqoZMmSRSpWrChr166Vn3/+WR544AHznLVr15b9+/d7Pe6LL76Q6tWrmysHlylTRoYPH+5uQSpVqpT59/HHHzfbd/2d3ONc5ZkyZYo89thj5rlfffXVRHU1ePBgqVmzZqLl1apVMy1ZKpB997RixQrz3GfOnHEv06tv67KDBw+6l/3www9y3333SebMmc1r0aNHD7l48WKS2wXwPwQZIJXRA+WVK1fM/2vLyoQJE2TXrl3y4YcfynfffSf9+/f3Wv/SpUvy+uuvywcffGDWK1CggJw/f17at29vDr7r1q2T8uXLyyOPPGKWexo5cqS0a9fOHLgrVKggTz/9tDz33HOmNWjjxo160Vrp3r27e/1Vq1aZ9Xv27Cm7d+82QUjDlCt0aIhQ2sJ05MgR99/JPc7l5ZdfNiFox44d8swzzySqmzZt2sj69eu9wpXu8/bt203ZVaD7nhL6fNpy1rJlS/Ncn376qdm+Z90ASEKkr1oJIHz0irzNmjUz/3/9+nVn6dKl5oq8ffv29bv+3LlzzdV6XfSK2/o1sXXr1hs+z7Vr15zs2bObKyO76ONeeukl999r1641y6ZNm+Ze9sknn5gr/7o0aNDAee2117y2PWvWLKdw4cJe212wYIHXOoE+rlevXk5yqlWr5owYMcL996BBg5yaNWumeN9dZVy+fLn5+/Tp0+77t2zZYpYdOHDA/N2pUyenS5cuXttdtWqVkyZNGuePP/5ItsxAasYYGSDGLVq0SLJly2bGhFy/ft20LGjLhPr2229l1KhRsmfPHjl37pzpirl8+bJphdHuIKWDhKtWreq1zfj4eHnppZdMt8mxY8fM2Bl9jHY7efJ8XMGCBc2/2pXluUyfT587R44csm3bNlm9erVXS4pu27dMvgJ93F133ZVsfWmrzPTp02XIkCGmxeiTTz6RPn36pHjfU0LLry0xs2fPdi/T59bX68CBA6ZbDoB/BBkgxj344INmbIgGkiJFiki6dP/72Ov4DB2/ouNmNADkyZPHdGd06tTJdD25Dv7aFaXjOTxp18rJkydl/PjxZuyNjp2pVauWu8vKxXMwrWsb/pbpAVvp+Bwd29KiRYtE+6FjX5IS6ON0bExyWrduLQMGDDDjXv744w85fPiwPPXUUynedxfXwOj/NdT8j4ZK3/Jrl5uOi/FVokSJZMsMpGYEGSDG6cFbp1372rRpkwkQb7/9tvtg+9lnnwW0TW39mDx5shkbovRgf+LEiZsuqw7W3bt3r9/yumgQ0laQlD4uUMWKFTMDeLV1RIOMDuzVcUHB7nv+/PnNvzqmJ3fu3Ob/dcyQb/l1bE8oyg+kNgQZIJXSg6a2DLzzzjvStGlTc4CeOnVqQI/VAa6zZs0yXTXaLdSvXz/TcnOzhg4dalqJtBXiiSeeMAFLu1127twpr7zyillHZyotW7ZM6tSpY1pDNBwE8riU0O6lYcOGmVaWsWPH3tS+az3rLCTtztOWr59++smER0/aAnTvvfeawb3PPvusCZ8abJYuXSoTJ05McfmB1IRZS0AqpVOKdfq1zkiqXLmyaYHQ8TKBmDZtmpw+fdq0JLRt29Z0iXi2WgSrcePGZkzPv//9b7n77rvNwV2DhHbhuGgI0AO8hoM777wz4MelhIYh7T7SsS/Nmze/qX3XFiQdZ6PjkHTMkNa3b7jS5StXrjQhR6dg635pONOuQAA3FqcjfpNZBwAAICrRIgMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACA2Or/ATVtNGs3JxI6AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "clf = RandomClassifier(threshold=0.8)\n", + "clf = RandomClassifier()\n", "\n", "if metric == \"precision\":\n", " theoretical_value = p\n", @@ -168,6 +127,10 @@ "\n", "for _ in range(n_repeats):\n", "\n", + " X_calibrate = list(range(1, N+1))\n", + " y_calibrate = [1] * int(p*N) + [0] * (N - int(p*N))\n", + " np.random.shuffle(y_calibrate)\n", + "\n", " controller = BinaryClassificationController(\n", " fitted_binary_classifier=clf,\n", " metric=\"precision\",\n", @@ -178,55 +141,20 @@ "\n", " valid_parameters = controller.valid_thresholds\n", " all_valid_parameters.append(valid_parameters)\n", - "print(all_valid_parameters)\n", + "\n", "all_valid_parameters = np.concatenate([x for x in all_valid_parameters if x.size > 0]) if any(x.size > 0 for x in all_valid_parameters) else np.array([])\n", "\n", - "if metric == \"precision\":\n", - " nb_actual_valid = sum(1 for x in all_valid_parameters if p >= theoretical_value)\n", - "elif metric == \"recall\":\n", - " nb_actual_valid = sum(1 for x in all_valid_parameters if x <= (1 - theoretical_value))\n", + "nb_actual_valid = sum(1 for x in all_valid_parameters if theoretical_value >= target_level)\n", "\n", - "if nb_actual_valid/len(all_valid_parameters) >= confidence_level:\n", - " print(f\"Risk controlled! Proportion of actual valid parameters: {nb_actual_valid/len(all_valid_parameters)}\")\n", - "else:\n", - " print(f\"Risk not controlled. Proportion of actual valid parameters: {nb_actual_valid/len(all_valid_parameters)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "42f85519-17ae-46c7-bfdc-e26c1d87017a", - "metadata": { - "id": "42f85519-17ae-46c7-bfdc-e26c1d87017a" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "All valid thresholds = [0.94 0.95 0.96 0.97 0.98 0.99 0.89 0.9 0.91 0.92 0.93 0.95 0.96 0.98\n", - " 0.99 0.97 0.98 0.99 0.99 0.99 0.98 0.99 0.87 0.88 0.89 0.93 0.94 0.95\n", - " 0.96 0.97 0.98 0.99 0.98 0.99 0.99 0.82 0.83 0.84 0.85 0.86 0.87 0.88\n", - " 0.89 0.9 0.92 0.93 0.98 0.99 0.99 0.99 0.95 0.96 0.97 0.98 0.99 0.97\n", - " 0.98 0.99 0.97 0.98 0.99 0.83 0.84 0.85 0.94 0.95 0.96 0.98 0.99 0.95\n", - " 0.96 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99 0.99 0.97\n", - " 0.98 0.99 0.99 0.97 0.98 0.99 0.99 0.98 0.99 0.99 0.99 0.92 0.95 0.96\n", - " 0.97 0.98 0.99 0.99 0.99 0.95 0.96 0.97 0.98 0.99 0.94 0.95 0.96 0.97\n", - " 0.98 0.99 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99 0.97 0.98 0.99 0.97\n", - " 0.98 0.99 0.99 0.89 0.9 0.91 0.92 0.93 0.99 0.77 0.78 0.79 0.82 0.83\n", - " 0.84 0.85 0.86 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99\n", - " 0.89 0.9 0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99 0.99 0.99 0.99\n", - " 0.97 0.98 0.99 0.97 0.98 0.99 0.98 0.99 0.99 0.95 0.96 0.97 0.98 0.99\n", - " 0.95 0.96 0.97 0.98 0.99 0.99 0.83 0.84 0.85 0.86 0.87 0.88 0.89 0.9\n", - " 0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99 0.99 0.94 0.95 0.96 0.97\n", - " 0.98 0.99]\n", - "Theoretical value = 0.5\n" - ] - } - ], - "source": [ - "print(f\"All valid thresholds = {all_valid_parameters}\")\n", - "print(f\"Theoretical value = {theoretical_value}\")" + "print(f\"Number of valid thresholds according to LTT across all iterations: {len(all_valid_parameters)}\")\n", + "print(f\"Number of actual valid thresholds across all iterations: {nb_actual_valid}\")\n", + "\n", + "counter = Counter(all_valid_parameters)\n", + "plt.bar(counter.keys(), counter.values(), width=0.008)\n", + "plt.xlabel('Parameter value')\n", + "plt.ylabel('Occurrences')\n", + "plt.title('Occurrences of each parameter value')\n", + "plt.show()" ] }, { @@ -257,7 +185,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.3" + "version": "3.10.17" } }, "nbformat": 4, From bec9a380d62f0aaa0e892c3e6e186930ebdb295c Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Mon, 28 Jul 2025 11:35:26 +0200 Subject: [PATCH 06/18] ENH: theoretical tests notebook --- .../risk_control_theoretical_tests_proto.ipynb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 947710a0c..4b59faa2a 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 1, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 2, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 19, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -85,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 20, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -100,13 +100,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Number of valid thresholds according to LTT across all iterations: 105\n", + "Number of valid thresholds according to LTT across all iterations: 79\n", "Number of actual valid thresholds across all iterations: 0\n" ] }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAHHCAYAAACle7JuAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAN0lJREFUeJzt3QeUU2X6x/Fn6L33XldAioKKFEUpi4oIgqsgSxPBRZD2pysgoIKNJs1CURZFkaLCiosIiBTpfUFEEFZh6F0GhPs/z7snOUkmw2RCQvJmvp9zIs7Nzc1735T7y1vujXMcxxEAAAALpYl0AQAAAIJFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAZAi+/btk7/+9a+SM2dOiYuLk4ULF0o0eeCBB6Ry5cqRLgYiqFSpUtKhQ4dIFwO3CEEGYbNr1y75+9//LkWLFpWMGTNKkSJFpE2bNmY57NW+fXvZsWOHvPrqqzJr1iy56667Il0khNDkyZNl5syZkS4GELB0ga8KBG7+/PnSunVryZMnj3Tq1ElKly4tBw8elGnTpsnnn38uc+bMkccffzzSxUQK/fHHH7J27Vp58cUXpXv37pEuDsIUZPLly0eLBqxBkEHI7d+/X9q2bStlypSR77//XvLnz+++r2fPnnLfffeZ+7dv327WiSaXLl2SLFmyJFr+559/yvXr1yVDhgySmh0/ftz8mytXrkgXxWoXL16UrFmzSmrB5wfhRNcSQu7NN980geC9997zCjFKf+m9++675ov8jTfe8Lrvt99+M6032gWlXVHaitO1a1e5cuWKe50zZ85I7969TR+4rlOsWDFp166dnDhxwtyvTeI6bkNbfzytWLHCLNd/fcdSbNq0Se6//34TYAYPHmweq+u+9dZbMm7cOClbtqx5rt27d5vH7dmzR5544gnT2pQpUybTtfLll196PZ+rHKtXr5Y+ffqYetADl7ZCucKAp6+//lrq1asn2bNnlxw5csjdd98tH3/8sdc6P/74ozz00ENmbIqWVdfX7Xs6f/689OrVy10/BQoUkEaNGsnmzZuTfd22bNkiDz/8sHn+bNmySYMGDWTdunXu+19++WUpWbKk+f9+/fqZ/dPnuZGEhAQZNmyYlCtXzpSnePHi0r9/f7Pc04wZM6R+/fqmvLpepUqVZMqUKX63GUhdKX29HnzwQVNX2r3p+35Liu6XtjbNnj1bbrvtNvMa16hRw4RyT7/++qs8//zzZp3MmTNL3rx55W9/+1ui957rvbBy5Uqzvu6jvm+D2cYPP/wgPXr0MO8nDZPPPfec+Xzo50I/B7lz5zY3rWPHcby2oUFC38+333672aeCBQuax58+fdq9jr6e2vWrZdXn05t+Tlz0efT9pa+jvk76ur7++utm2y7JfX586WdQXydfuk193fSz5qLbrF27tqknrS99XbSFNzn63tUy+Urq+0LfY/qDSz+z+j5r0qQJXeJRjBYZhNxXX31lvhD1i8AfDQ16/+LFi93Lfv/9d7nnnnvMF2WXLl2kQoUKJtjol5SGIv0ld+HCBbPN//znP/LMM89I9erVTYDREPHf//7XhKSUOnnypDl4t2rVyozn0S93z4Pr5cuXTXn0i1iDi36Z1alTx3zBDhw40HzRffbZZ9K8eXOZN29eou6yF154wRxY9GCuX5b6xa4HyU8//dTry1T3Rw8wgwYNMgcoDRVLliyRp59+2qzz3XffmXLqF7duK02aNO6D/6pVq0zdqX/84x+mzvQ5NAzo/unBT+tM6yspul9atxoM9CCYPn16Ezj1IKYHtZo1a0qLFi1M2TRIarfhI488YgJPUvRA9Nhjj5nn1zqsWLGiGVszduxY+emnn7wGCWto0f3X9dOlS2feQ3qA121069YtRXWl9OCsoU/L/OSTT5o6GTBggFSpUsXUY3J0n/U10tCgr712t+j21q9f7x5IvGHDBlmzZo1572gw0ddX90PrTA/avi17uj8aQIYOHWqCfDDb0PdToUKFZPjw4SZk6o8FrQPdRokSJeS1116Tf/3rX+bHhJZTw42Lhhatv44dO5r9OnDggEycONHUnwZifc31/anPoa+rdh8q12dCP4caIPVzqdvS59Pn1dfhyJEj5rGe/H1+/HnqqadM0Dh69KjZNxd93+j3gtaNy/jx4817RMfaaYDTLmoNfosWLTJhIxR03JeOA2vcuLEJabrf+prUrVvX1FVy4R0R4AAhdObMGf0Z6DRr1uyG6z322GNmvXPnzpm/27Vr56RJk8bZsGFDonWvX79u/h06dKh5zPz585NcZ8aMGWadAwcOeN2/fPlys1z/dalXr55ZNnXqVK919bG6PEeOHM6xY8e87mvQoIFTpUoV5/Lly17PXbt2bad8+fLuZa5yNGzY0F021bt3bydt2rSmnlz1lT17dqdmzZrOH3/84Xef9F/dduPGjb22denSJad06dJOo0aN3Mty5szpdOvWzUmp5s2bOxkyZHD279/vXvb777+bst1///2J6ubNN99MdpuzZs0yr+mqVau8lmt96zZWr17ttS++dH/LlCnj/juQuvJ8XT/66CP3soSEBKdQoUJOy5Ytky23PlZvGzdudC/79ddfnUyZMjmPP/74Dcu8du3aRM/tei/UrVvX+fPPP73WT+k2fN8DtWrVcuLi4px//OMf7mX6HMWKFTP14KKvgT5+9uzZXs+1ZMmSRMtvv/12r8e6jBw50smaNavz008/eS0fOHCgeU8fOnQo2c+PP3v37jXrv/POO17Ln3/+eSdbtmxedeRbX1euXHEqV67s1K9f32t5yZIlnfbt27v/HjZsmHkOX77fF+fPn3dy5crldO7c2Wu9o0ePms+W73JEB7qWEFLataG0OfZGXPefO3fO/OrWX+dNmzb1OwPG1SSsLR7VqlXzO0jYX7NxIPSXov5C9adly5ZeXWOnTp0yLSP6C1/3U1uD9KatHvrrTacl669VT/pr1LNs2upx7do106Wgli5daralrTva3O9vn7Zu3Wq2rS0O+lyu59Vf9dr9o10erqZ9/XWuXVD6SzZQWp5///vfplXJc8xS4cKFzXPqL2N9nVJq7ty5phVGW9dcZdabtiKp5cuXu9fVbgKXs2fPmvX01/8vv/xi/g60rly0RUFb2Fy0RU9brXR7gahVq5Zp/XLR1odmzZrJN998Y+rLt8xXr141r412tehr4K8rr3PnzpI2bVqvZSndhna9eu6rtpRp9tLlLvoc+jny3Fd9LbRLUrsZPV8L3UetK8/XIim6DX3/aguj5zYaNmxo6sS3683385OUv/zlL3LHHXd4tVLq9rQVTb8TPOvI8/+11U3fG1qmQLpOA6HvMW0V1hZHz33UOtW6DqSecOvRtYSQcgUUV6AJJPDomBE9UCZ37g8dRKxfjqGkXURJDUDUMTqefv75Z3PQGDJkiLn5c+zYMbNNzwOgJz0IKNe4BN0ndaN91xCjtLk7KfqFrtvWcSC6no5h0IOUdv9o98KNBlVr/WvzuY7T8KVBREPS4cOHTXdOSmi5tUsrqYOZ1pWLdm1ol5nOiNKy+O6bHoQDqSsX7abxDTdaPzrAPBDly5f3e8DVsml9aReIzuAaNWqU6ULRAOs5JsUVvm70flIp3Ybv+0nrRenr7bvcc+yLvha6PR2fk9xrkRTdhtZfIK9nUvubFO1e0vFpWgf6+dGxbLo9Xe5Ju5BeeeUVE+49x1kF+0Mmqc+aK2z70q5XRB+CDEJKv0D1l3xyBwy9X7+w9ItBv8xDJakvNNevaF+ev/CSu8/V6tG3b1/TAuOP/pr25PsL3MV3IOaNuJ5Xxz3oL1d/XGNVtLVIf6EuWLDAtLLoY7SfX6fDBzI2JJS03DomZcyYMX7vdx18NaBoy5K23Oi6ulzDpY710PE0ngNJAxWKek+OjiXRAKKDX7UFx3WCQB3T4a/M/t5rKd1GUvvlb7nnvuq2NMToAGZ/Amk50W1oi46OofJHg16gny1fGlh0rI22+mhd6LgzrQsdl+SiY8F0fIyOsdMxS/o9o+N6tP78DfYO5nvBVec6TsZzvI6Ljt9C9OFVQcg9+uij8v7775suCR0g50u/kHRQow4YdH2JaqDZuXPnDbersx+SW8fV4qHNw55cXTk3w9WqoV+e2pweCrpPSvfLNwT5rqN1FMjz6he8DizVm/6q1UG+evK6pIKM1r8OKt27d2+i+3SGlg4s9v3FHwgt97Zt20xIudEvZh3Yq7+uddC2Z4uDbzN+IHUVKq5f5p50gLLWk+ugr10f2vr19ttvu9fRwa2+770bCcU2AqF19+2335qB6skFjKReK92GDrgP1Xvft/VGu/60e0kHqmvw1q5O7fp10a5l7VLU7j3P5RpkkuP5veB56gDf7wXXe0xDXzj2E+HBGBmEnE7N1S9LDSra5+9Jx5nozBo9IOh6Sg+U+qWlB7SNGzcm+ctSu5X0wKitDUmt4/oi8uyv119dOrvjZumXm84m0dk8OkvDl79p1cnRU/1r95p2L+gBzN8+aReR7pdOPdUDSVLPq/vp2x2hZdbp7L7TnX1/zWs5vvjiC69pqPHx8eaXrobRYJrUtXVIuwo01PrSVjjXzB1Xa4Jvt4rvASqQugoV7eLyHHehXWtaP1oGV3n1X9/nfeedd5Js/fMnFNsI9LXQbY4cOdLvOV48g5POxPMXpHQbWi8aJHzp+rqdm6GtMjoTa/r06WZcim+3ktaVhizPutH3ayCXyPD3vaDvvw8//NBrPW1p1fe6zv7SMUuh+Iwj/GiRQcjp+AL9gtApktq14HtmX/2S+uSTT9xfLkq/OLQrRAd4uqbqaljQpmZt2dFfURp89BesTrfUKbh6gNdgpL/kp06dagYC6ziOe++91zRT63065VOnaN7sl6zLpEmTzIFd90sHb2orjR7w9Qtep4Br0EoJ/dLU7pNnn33WnA9FB9fqr0fdjo7H0HrUoPfBBx+YFhXdPx2crN1yGhK01UK3oSFQxx3p2BA974bWhXY36a9wneLr+YvfHx13oAMddd+0JUeb0DWwaQAK9PwrvvSkh9pFoMFVy6mtAXoQ0lYeXa4HRB2UquFAu5J0YKeGXw1rGn40hHkGxkDqKlR0HI4e1DynXyud9uzZ8qhdENoFolPd9T2g9a3nOAlUKLYRCP1cad1qCNTxJVrn2rKoLU/6GdNpza7ztejnSqcb63tCW770ddAxI/r508+allnP+qvraRjQKfX6udTPdzCnQPAMStptqzf93Pq2iOj0au161O4mfe21tVE/j1rG5LqydX+1tU+/i3Q/NBRpYNLWtUOHDnm9x3Tf9b2rLZnaxedaR08Xoe9hnbKOKBPpaVOIXdu3b3dat27tFC5c2EmfPr2Z/qp/79ixw+/6OsVVp2Hnz5/fyZgxo5l6q1OJdeqsy8mTJ53u3bs7RYsWNdOFdZqpTrM8ceKEex2dQqzTnnUbBQsWdAYPHuwsXbrU7/RrnWrqK7kpxrp9Lafuj+6XluXRRx91Pv/880TTOn2nk/ubBq6+/PJLM4U7c+bMZtrqPffc43zyySde62zZssVp0aKFkzdvXrNvOsX0ySefdJYtW2bu13rq16+fU61aNTNNWafK6v9PnjzZCcTmzZvN9F6d8polSxbnwQcfdNasWZOiuvGl02Nff/11U89a5ty5czs1atRwhg8f7pw9e9Zr/6tWrWqmOJcqVco8Zvr06X6n0idXV0m9rvo+0TpLjj6nvu/++c9/mmnvWu4777wz0Wt2+vRpp2PHjk6+fPlMnWnd7dmzJ9HU36TeC6HYhmta8fHjxxPtq77+vt577z1T/1p3+h7RUwn079/fTLX3nGrcpEkTc79u23Mqtk5PHjRokFOuXDnz+dNy62vx1ltvmdc6mPeIpzp16pjHPvvss37vnzZtmvs1qVChgqkXf1OrfetPbdq0yUzd13KXKFHCGTNmzA1P16CvhU651vdk2bJlnQ4dOnhNyUf0iNP/RDpMAUC00O4LPQkfv7wBOzBGBgAAWIsgAwAArEWQAQAA1mLWEgB4YNggYBdaZAAAgLUIMgAAwFoR7Vp6+eWXvU4wpfTCdXrCLKVn7/y///s/c0IzPTGXnqBKT0xVsGDBgJ9Dr52hVwLWM4KG6sJiAAAg/N28eqJPPTu5nhg0asfI6JlK9UyW/i7K1bt3b3M2Rdcl6PUaHC1atDBXyg2UhphgrhMDAAAiTy8Romctj9ogo8HF31VG9Vorejp7vdaL65Lqeu0VPXW9Xo9DT0MfCG2JcVUEl2AHAMAO586dMw0RruN41AYZvdaHNhvpVU31MvZ6LRC9JsamTZvMRbs8r7dRoUIFc59ejySpIKNdUJ4XyNNmKaUhhiADAIBdkhsWEtHBvjVr1pSZM2fKkiVLzIW6Dhw4IPfdd58JH0ePHjUXkvO85LrS8TF6X1I0CGk3lOtGtxIAALEroi0yejVfl6pVq5pgU7JkSXNl3MyZMwe1Tb3qcZ8+fRI1TQEAgNgTVdOvtfXlL3/5i/z8889m3MyVK1fkzJkzXuvEx8f7HVPjkjFjRnc3Et1JAADEtqgKMhcuXJD9+/dL4cKFpUaNGpI+fXpZtmyZ+/69e/fKoUOHzFgaAACAiHYt9e3bV5o2bWq6k3Sa9LBhwyRt2rTSunVrM76lU6dOppsoT548pmXlhRdeMCEm0BlLAAAgtkU0yPz3v/81oeXkyZOSP39+qVu3rplarf+vxo4da06C07JlS68T4gEAAKg4J8avkKaDfbV1R89Lw3gZAABi6/gdVWNkAAAAUoIgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwVkTP7AsAAIJXauDiSBdBDo5uEtHnp0UGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1oibIjB49WuLi4qRXr17uZZcvX5Zu3bpJ3rx5JVu2bNKyZUuJj4+PaDkBAED0iIogs2HDBnn33XelatWqXst79+4tX331lcydO1dWrlwpv//+u7Ro0SJi5QQAANEl4kHmwoUL0qZNG3n//fcld+7c7uVnz56VadOmyZgxY6R+/fpSo0YNmTFjhqxZs0bWrVsX0TIDAIDoEPEgo11HTZo0kYYNG3ot37Rpk1y9etVreYUKFaREiRKydu3aJLeXkJAg586d87oBAIDYlC6STz5nzhzZvHmz6VrydfToUcmQIYPkypXLa3nBggXNfUkZNWqUDB8+PCzlBQAA0SViLTKHDx+Wnj17yuzZsyVTpkwh2+6gQYNMt5Trps8DAABiU8SCjHYdHTt2TKpXry7p0qUzNx3QO2HCBPP/2vJy5coVOXPmjNfjdNZSoUKFktxuxowZJUeOHF43AAAQmyLWtdSgQQPZsWOH17KOHTuacTADBgyQ4sWLS/r06WXZsmVm2rXau3evHDp0SGrVqhWhUgMAgGgSsSCTPXt2qVy5steyrFmzmnPGuJZ36tRJ+vTpI3ny5DEtKy+88IIJMffee2+ESg0AAKJJRAf7Jmfs2LGSJk0a0yKjs5EaN24skydPjnSxAABAlIhzHMeRGKbTr3PmzGkG/jJeBgAQS0oNXBzpIsjB0U0ievyO+HlkAAAAgkWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAACB1BZnNmzfLjh073H9/8cUX0rx5cxk8eLBcuXIllOUDAAAIbZB57rnn5KeffjL//8svv0irVq0kS5YsMnfuXOnfv3/A25kyZYpUrVpVcuTIYW61atWSr7/+2n3/5cuXpVu3bpI3b17Jli2btGzZUuLj44MpMgAAiEFBBRkNMXfccYf5fw0v999/v3z88ccyc+ZMmTdvXsDbKVasmIwePVo2bdokGzdulPr160uzZs1k165d5v7evXvLV199ZZ5j5cqV8vvvv0uLFi2CKTIAAIhB6YJ5kOM4cv36dfP/3377rTz66KPm/4sXLy4nTpwIeDtNmzb1+vvVV181rTTr1q0zIWfatGkmIGnAUTNmzJCKFSua+++9995gig4AAFJ7i8xdd90lr7zyisyaNcu0lDRp0sQsP3DggBQsWDCogly7dk3mzJkjFy9eNF1M2kpz9epVadiwoXudChUqSIkSJWTt2rVBPQcAAIgtQbXIjBs3Ttq0aSMLFy6UF198UcqVK2eWf/7551K7du0UbUsHDWtw0fEwOg5mwYIFUqlSJdm6datkyJBBcuXK5bW+BqWjR48mub2EhARzczl37lyK9w8AAMRwkNEBup6zllzefPNNSZs2bYq2ddttt5nQcvbsWROE2rdvb1p5gjVq1CgZPnx40I8HAACp4DwyZ86ckQ8++EAGDRokp06dMst2794tx44dS9F2tNVFW3Rq1KhhQki1atVk/PjxUqhQITOVW5/Hk85a0vuSouXRUOS6HT58OMg9BAAAMdkis337dmnQoIHp9jl48KB07txZ8uTJI/Pnz5dDhw7JRx99FHSBdBCxdg1psEmfPr0sW7bMTLtWe/fuNdvXrqikZMyY0dwAAEDsCyrI9OnTRzp27ChvvPGGZM+e3b38kUcekaeffjrg7WjrycMPP2wG8J4/f97MUFqxYoV88803kjNnTunUqZN5Lg1Jep6ZF154wYQYZiwBAICgg8yGDRvk3XffTbS8aNGiNxyI60u7odq1aydHjhwxwUXH3miIadSokbl/7NixkiZNGtMio600jRs3lsmTJ/PKAQCA4IOMdt34mw2kJ8rLnz9/wNvR88TcSKZMmWTSpEnmBgAAEJLBvo899piMGDHCnOdFxcXFmbErAwYMcI9nAQAAiMog8/bbb8uFCxekQIEC8scff0i9evXMzCMdL6Nn5wUAAIjariUdz7J06VJZvXq1bNu2zYSa6tWre52FFwAAICqDjEudOnXMDQAAwJqupR49esiECRMSLZ84caL06tUrFOUCAAAIT5CZN2+e35YYvc6SXmYAAAAgaoPMyZMnzTgZX3rSuhMnToSiXAAAAOEJMjpDacmSJYmWf/3111KmTJlgNgkAAHDrLlHQvXt3OX78uNSvX98s02si6bTscePGBbNJAACAWxNknnnmGXPJAD1nzMiRI82yUqVKyZQpU8wlBwAAAKJ6+nXXrl3NTVtlMmfOLNmyZQttyQAAAMJ5HhmVkmsrAQAARHywb3x8vLRt21aKFCki6dKlk7Rp03rdAAAAorZFpkOHDuYikUOGDJHChQubi0YCAABYEWR++OEHWbVqldxxxx2hLxEAAEA4u5aKFy8ujuME81AAAIDIBhk9V8zAgQPl4MGDoS8RAABAOLuWnnrqKbl06ZKULVtWsmTJIunTp/e6/9SpU8FsFgAAIPxBhrP3AgAAa4NM+/btQ18SAACAWzFGRu3fv19eeuklad26tRw7dsx90chdu3YFu0kAAIDwB5mVK1dKlSpV5Mcff5T58+fLhQsXzPJt27bJsGHDgtkkAADArQkyOmPplVdekaVLl0qGDBncy/VK2OvWrQtmkwAAALcmyOzYsUMef/zxRMsLFCggJ06cCGaTAAAAtybI5MqVS44cOZJo+ZYtW6Ro0aLBbBIAAODWBJlWrVrJgAED5OjRo+Y6S9evX5fVq1dL3759pV27dsFsEgAA4NYEmddee00qVKhgLlWgA30rVaok999/v9SuXdvMZAIAAIjK88joNZa0JWbChAkydOhQM15Gw8ydd94p5cuXD08pAQAAQhVkypUrZ84Xo8FFW2UAAACs6FpKkyaNCTAnT54MT4kAAADCOUZm9OjR0q9fP9m5c2cwDwcAAIjctZZ0ZpJe/bpatWrmhHiZM2f2up+rXwMAgFuBq18DAIDUE2SuXr1qrrU0ZMgQKV26dHhKBQAAEI4xMunTp5d58+al9GEAAADRMdi3efPmsnDhwtCXBgAAINxjZHT69YgRI8xlCWrUqCFZs2b1ur9Hjx7BbBYAACD8QWbatGnmwpGbNm0yN0967SWCDAAAiNogc+DAgdCXBAAA4FaMkQEAALC2ReaZZ5654f3Tp08PtjwAAADhDTKnT59OdG4ZvVzBmTNnpH79+sFsEgAA4NYEmQULFiRadv36denatauULVs2mE0CAABEboyMXhW7T58+Mnbs2FBtEgAA4NYN9t2/f7/8+eefodwkAABAaLuWtOXFk+M4cuTIEVm8eLG0b98+mE0CAADcmiCzZcuWRN1K+fPnl7fffjvZGU0AAAARDTLLly8PWQEAAABu6RgZPbPvvn37Ei3XZQcPHgy6MAAAAGEPMh06dJA1a9YkWv7jjz+a+wAAAKI2yOgYmTp16iRafu+998rWrVtDUS4AAIDwBBm9wvX58+cTLT979qxcu3YtmE0CAADcmiBz//33y6hRo7xCi/6/Lqtbt24wmwQAALg1s5Zef/11E2Zuu+02ue+++8yyVatWyblz5+S7774LZpMAAAC3pkWmUqVKsn37dnnyySfl2LFjppupXbt2smfPHqlcuXIwmwQAALg1LTKqSJEi8tprrwX7cAAAgMi0yMyYMUPmzp2baLku+/DDD2++VAAAAOEKMjqoN1++fImWFyhQgFYaAAAQ3UHm0KFDUrp06UTLS5Ysae4DAACI2iCjLS862NfXtm3bJG/evKEoFwAAQHiCTOvWraVHjx7m4pF6/hi96bTrnj17SqtWrYLZJAAAwK2ZtTRy5EhzccgGDRpIunT/24SGmfbt2zNGBgAARHeQyZAhg3z66afSt29fE2gyZ84sVapUMWNkAAAAorZr6cyZM9KtWzcza0kvEqldSXrF6zfffNPcl9LZT3fffbdkz57djLtp3ry57N2712udy5cvm+fTsTfZsmWTli1bSnx8fEqLDQAAUnuLzKlTp6RWrVry22+/SZs2baRixYpm+e7du2XmzJmybNkyWbNmjeTOnTug7a1cudKEFA0zf/75pwwePFj++te/mu1lzZrVrNO7d29ZvHixOUdNzpw5pXv37tKiRQtZvXp1MPsLAABiSJzjOE6gK/fq1cuElW+//VYKFizodd/Ro0dNCNFxM2PHjg2qMMePHzctMxpw9FpOejXt/Pnzy8cffyxPPPGEWUcvg6ABau3ataZFKDl6/ScNQLqtHDlyBFUuAACiUamBiyNdBDk4uklYthvo8TtFXUsLFy6Ut956K1GIUYUKFZI33nhDFixYEFyJRUxhVZ48ecy/mzZtkqtXr0rDhg3d61SoUEFKlChhgow/CQkJZuc9bwAAIDalKMgcOXJEbr/99iTv1wtGastMMK5fv25afOrUqeO+8KRuSwcW58qVy2tdDVJJPY+Ou9EE57oVL148qPIAAIAYCzI6wFdnKSXlwIED7taUlNKxMjt37pQ5c+bIzRg0aJBp2XHdDh8+fFPbAwAAMRJkGjduLC+++KJcuXLFb5fOkCFD5KGHHkpxIXQA76JFi8wJ9ooVK+bVXaXP5TsbSmct6X3+ZMyY0fSled4AAEBsStGspREjRshdd90l5cuXNy0oOl5Fxwr/5z//kcmTJ5swM2vWrIC3p4994YUXzLiaFStWJLp+U40aNSR9+vRmgLFOu1Y6PVuv56SzpwAAQOqWoiCjrSU6yPb55583XTiuCU9xcXHSqFEjmThxYorGpGgY0hlJX3zxhTmXjGvci45t0ZPs6b+dOnWSPn36mC4rbV3R4KMhJpAZSwAAILal+My+2mry9ddfy+nTp2Xfvn1mWbly5YIaGzNlyhTz7wMPPOC1fMaMGeYke0qncqdJk8a0yGiLj3ZvaesPAABAis4jYyPOIwMAiFWlOI9McFe/BgAAiAYEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWCtdpAsAAEg9Sg1cLNHg4OgmMVFO0CIDAAAsRpABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKwV0SDz/fffS9OmTaVIkSISFxcnCxcu9LrfcRwZOnSoFC5cWDJnziwNGzaUffv2Ray8AAAgukQ0yFy8eFGqVasmkyZN8nv/G2+8IRMmTJCpU6fKjz/+KFmzZpXGjRvL5cuXb3lZAQBA9InotZYefvhhc/NHW2PGjRsnL730kjRr1sws++ijj6RgwYKm5aZVq1a3uLQAACDaRO0YmQMHDsjRo0dNd5JLzpw5pWbNmrJ27dokH5eQkCDnzp3zugEAgNgUtUFGQ4zSFhhP+rfrPn9GjRplAo/rVrx48bCXFQAAREbUBplgDRo0SM6ePeu+HT58ONJFAgAAqS3IFCpUyPwbHx/vtVz/dt3nT8aMGSVHjhxeNwAAEJuiNsiULl3aBJZly5a5l+l4F529VKtWrYiWDQAARIeIzlq6cOGC/Pzzz14DfLdu3Sp58uSREiVKSK9eveSVV16R8uXLm2AzZMgQc86Z5s2bR7LYAAAgSkQ0yGzcuFEefPBB9999+vQx/7Zv315mzpwp/fv3N+ea6dKli5w5c0bq1q0rS5YskUyZMkWw1AAAIFpENMg88MAD5nwxSdGz/Y4YMcLcAAAAfEXtGBkAAIDkEGQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANaK6AnxAMAGpQYujnQR5ODoJjFTTiCUaJEBAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsFa6SBcAQOiVGrhYosHB0U1iopwAohctMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWCtdpAsA2KbUwMWRLoIcHN0k0kUAgKhAiwwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLWYfm35NNxApuJSzsAxrRkA7EKLDAAAsBZBBgAAWMuKIDNp0iQpVaqUZMqUSWrWrCnr16+PdJEAAEAUiPog8+mnn0qfPn1k2LBhsnnzZqlWrZo0btxYjh07FumiAQCACIv6IDNmzBjp3LmzdOzYUSpVqiRTp06VLFmyyPTp0yNdNAAAEGFRHWSuXLkimzZtkoYNG7qXpUmTxvy9du3aiJYNAABEXlRPvz5x4oRcu3ZNChYs6LVc/96zZ4/fxyQkJJiby9mzZ82/586dC3n5ridckmiQ3L5RzsAF8j6xoZzRUEZFOUOH92ZoUc7QCcfx1XO7juPceEUniv32229aemfNmjVey/v16+fcc889fh8zbNgw8xhu3Lhx48aNm1h/O3z48A2zQlS3yOTLl0/Spk0r8fHxXsv170KFCvl9zKBBg8zgYJfr16/LqVOnJG/evBIXFxf2MsciTcXFixeXw4cPS44cOSJdHKtRl6FFfYYOdRla1OfN05aY8+fPS5EiRW64XlQHmQwZMkiNGjVk2bJl0rx5c3cw0b+7d+/u9zEZM2Y0N0+5cuW6JeWNdfph5AMZGtRlaFGfoUNdhhb1eXNy5syZ7DpRHWSUtq60b99e7rrrLrnnnntk3LhxcvHiRTOLCQAApG5RH2SeeuopOX78uAwdOlSOHj0qd9xxhyxZsiTRAGAAAJD6RH2QUdqNlFRXEsJPu+r0hIS+XXZIOeoytKjP0KEuQ4v6vHXidMRvpAsBAAAQcyfEAwAAuBGCDAAAsBZBBgAAWIsgAwAArEWQSYUmTZokpUqVkkyZMknNmjVl/fr1Sa579epVGTFihJQtW9asX61aNTP9/Wa2GWtCXZ+jRo2Su+++W7Jnzy4FChQwJ4Pcu3evpAbheG+6jB492pzdu1evXpJahKM+f/vtN/n73/9uzpaeOXNmqVKlimzcuFFiXajrUq8jOGTIECldurSpR1135MiRyV9XCImF8tpIiH5z5sxxMmTI4EyfPt3ZtWuX07lzZydXrlxOfHy83/X79+/vFClSxFm8eLGzf/9+Z/LkyU6mTJmczZs3B73NWBKO+mzcuLEzY8YMZ+fOnc7WrVudRx55xClRooRz4cIFJ5aFoy5d1q9f75QqVcqpWrWq07NnTyc1CEd9njp1yilZsqTToUMH58cff3R++eUX55tvvnF+/vlnJ5aFoy5fffVVJ2/evM6iRYucAwcOOHPnznWyZcvmjB8//hbuWWwgyKQyerHNbt26uf++du2a+cCNGjXK7/qFCxd2Jk6c6LWsRYsWTps2bYLeZiwJR336OnbsmLlw2sqVK51YFq66PH/+vFO+fHln6dKlTr169VJNkAlHfQ4YMMCpW7euk9qEoy6bNGniPPPMMzdcB4GhaykVuXLlimzatEkaNmzoXpYmTRrz99q1a/0+JiEhwTSNetJm0B9++CHobcaKcNSnP2fPnjX/5smTR2JVOOuyW7du0qRJE69tx7pw1eeXX35pLhfzt7/9zXR73nnnnfL+++9LLAtXXdauXdtcN/Cnn34yf2/bts3c//DDD4dtX2IVQSYVOXHihOmX9b28g/6tl3/wp3HjxjJmzBjZt2+fuWDn0qVLZf78+XLkyJGgtxkrwlGfvnQdHdNRp04dqVy5ssSqcNXlnDlzZPPmzWbcUWoSrvr85ZdfZMqUKVK+fHn55ptvpGvXrtKjRw/58MMPJVaFqy4HDhworVq1kgoVKkj69OlNKNTPeps2bcK+T7GGIIMbGj9+vPnS0g+bXo1cLxWhF+zUXyQIf31qa8LOnTvNARkpq8vDhw9Lz549Zfbs2Yl+HSO496YelKtXry6vvfaaOfB26dJFOnfuLFOnTo1o2W2sy88++8y8Nz/++GMTtjUMvvXWWzEdCsOFo1Eqki9fPkmbNq3Ex8d7Lde/CxUq5Pcx+fPnl4ULF5orjv/666+yZ88eyZYtm5QpUybobcaKcNSnJ/3yW7RokSxfvlyKFSsmsSwcdandAceOHTMH3nTp0pnbypUrZcKECeb/9Vd2rArXe7Nw4cJSqVIlr8dVrFhRDh06JLEqXHXZr18/d6uMzvxq27at9O7dO9W1HoYCQSYV0V8GNWrUMP2ynr+w9O9atWrd8LH6i7Zo0aLy559/yrx586RZs2Y3vU3bhaM+lQ7C1xCzYMEC+e6778z0zFgXjrps0KCB7NixQ7Zu3eq+6fgObbrX/9eDU6wK13tTuzh9TwWgYzxKliwpsSpcdXnp0qVELbH6ntRtI4UCHBSMGJpGmDFjRmfmzJnO7t27nS5duphphEePHjX3t23b1hk4cKB7/XXr1jnz5s0zUwi///57p379+k7p0qWd06dPB7zNWBaO+uzatauTM2dOZ8WKFc6RI0fct0uXLjmxLBx16Ss1zVoKR33qNPZ06dKZqcP79u1zZs+e7WTJksX55z//6cSycNRl+/btnaJFi7qnX8+fP9/Jly+fmbqNlCHIpELvvPOOOS+JnhdBpxXqh87zi14/YC56MK1YsaL5EOs5D/QD+9tvv6Vom7Eu1PWpvy/83fTcMrEuHO/N1BpkwlWfX331lVO5cmWzXoUKFZz33nvPSQ1CXZfnzp0z70Xdpp5jpkyZMs6LL77oJCQk3NL9igVx+p+UtuIAAABEA8bIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABgBCLi4sz19oBEH4EGSCGdejQwRxU9abXjClXrpyMGDHCXPvFVoQEAJ7Sef0FIOY89NBDMmPGDElISJB//etf0q1bN0mfPr0MGjQoxdvSK0ZrkPC92J2Nrl69auoBgN3s/zYCcEMZM2aUQoUKmSsUd+3aVRo2bChffvmluW/MmDFSpUoVyZo1qxQvXlyef/55uXDhgvuxM2fOlFy5cpn1K1WqZLZ16NAh2bBhgzRq1Ejy5csnOXPmlHr16snmzZu9nlcDz7vvviuPPvqoZMmSRSpWrChr166Vn3/+WR544AHznLVr15b9+/d7Pe6LL76Q6tWrmysHlylTRoYPH+5uQSpVqpT59/HHHzfbd/2d3ONc5ZkyZYo89thj5rlfffXVRHU1ePBgqVmzZqLl1apVMy1ZKpB997RixQrz3GfOnHEv06tv67KDBw+6l/3www9y3333SebMmc1r0aNHD7l48WKS2wXwPwQZIJXRA+WVK1fM/2vLyoQJE2TXrl3y4YcfynfffSf9+/f3Wv/SpUvy+uuvywcffGDWK1CggJw/f17at29vDr7r1q2T8uXLyyOPPGKWexo5cqS0a9fOHLgrVKggTz/9tDz33HOmNWjjxo160Vrp3r27e/1Vq1aZ9Xv27Cm7d+82QUjDlCt0aIhQ2sJ05MgR99/JPc7l5ZdfNiFox44d8swzzySqmzZt2sj69eu9wpXu8/bt203ZVaD7nhL6fNpy1rJlS/Ncn376qdm+Z90ASEKkr1oJIHz0irzNmjUz/3/9+nVn6dKl5oq8ffv29bv+3LlzzdV6XfSK2/o1sXXr1hs+z7Vr15zs2bObKyO76ONeeukl999r1641y6ZNm+Ze9sknn5gr/7o0aNDAee2117y2PWvWLKdw4cJe212wYIHXOoE+rlevXk5yqlWr5owYMcL996BBg5yaNWumeN9dZVy+fLn5+/Tp0+77t2zZYpYdOHDA/N2pUyenS5cuXttdtWqVkyZNGuePP/5ItsxAasYYGSDGLVq0SLJly2bGhFy/ft20LGjLhPr2229l1KhRsmfPHjl37pzpirl8+bJphdHuIKWDhKtWreq1zfj4eHnppZdMt8mxY8fM2Bl9jHY7efJ8XMGCBc2/2pXluUyfT587R44csm3bNlm9erVXS4pu27dMvgJ93F133ZVsfWmrzPTp02XIkCGmxeiTTz6RPn36pHjfU0LLry0xs2fPdi/T59bX68CBA6ZbDoB/BBkgxj344INmbIgGkiJFiki6dP/72Ov4DB2/ouNmNADkyZPHdGd06tTJdD25Dv7aFaXjOTxp18rJkydl/PjxZuyNjp2pVauWu8vKxXMwrWsb/pbpAVvp+Bwd29KiRYtE+6FjX5IS6ON0bExyWrduLQMGDDDjXv744w85fPiwPPXUUynedxfXwOj/NdT8j4ZK3/Jrl5uOi/FVokSJZMsMpGYEGSDG6cFbp1372rRpkwkQb7/9tvtg+9lnnwW0TW39mDx5shkbovRgf+LEiZsuqw7W3bt3r9/yumgQ0laQlD4uUMWKFTMDeLV1RIOMDuzVcUHB7nv+/PnNvzqmJ3fu3Ob/dcyQb/l1bE8oyg+kNgQZIJXSg6a2DLzzzjvStGlTc4CeOnVqQI/VAa6zZs0yXTXaLdSvXz/TcnOzhg4dalqJtBXiiSeeMAFLu1127twpr7zyillHZyotW7ZM6tSpY1pDNBwE8riU0O6lYcOGmVaWsWPH3tS+az3rLCTtztOWr59++smER0/aAnTvvfeawb3PPvusCZ8abJYuXSoTJ05McfmB1IRZS0AqpVOKdfq1zkiqXLmyaYHQ8TKBmDZtmpw+fdq0JLRt29Z0iXi2WgSrcePGZkzPv//9b7n77rvNwV2DhHbhuGgI0AO8hoM777wz4MelhIYh7T7SsS/Nmze/qX3XFiQdZ6PjkHTMkNa3b7jS5StXrjQhR6dg635pONOuQAA3FqcjfpNZBwAAICrRIgMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACA2Or/ATVtNGs3JxI6AAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAPkBJREFUeJzt3Qd8U+X+x/FfWWWXTdlb2aioTBEZVkBkeQXkL6CIgqCMK6MqIkPBBS5A75WhXARFljhQNghFmYIiCAjCVYYgpcyCcP6v33NfiUna0rSmJM3zeb9eoeQkOXny5OTkm2ecE+E4jiMAAAAWyRLsAgAAAFxrBCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIABpsmfPHrnzzjslKipKIiIiZOHChRJKmjZtKjVr1gx2MRBE5cuXl549ewa7GAhxBCBkmB9++EH+7//+T0qVKiWRkZFSsmRJ6datm1mOzKtHjx6yY8cOef7552XmzJly8803B7tICKDJkyfLjBkzgl0MIMNly/ingI3mz58vXbt2lUKFCkmvXr2kQoUKcuDAAZk6dap8/PHHMmfOHOnQoUOwi4k0On/+vMTFxcnTTz8t/fv3D3ZxkEEBqEiRIrSgIOwRgBBw+/btkwceeEAqVqwoa9askaJFi7pvGzBggNx2223m9u3bt5v7hJJz585J7ty5kyz/888/5cqVK5IjRw6x2e+//27+FihQINhFydTOnj0refLkEVvw+UEoogsMAffyyy+bIPGvf/3LK/wo/WX5zjvvmC+Al156yeu2X3/91bQWaVeZdplpq1Hfvn3l4sWL7vvEx8fLoEGDTB+/3qd06dLSvXt3OX78uLldm+51XIq2NnlatWqVWa5/fceKbN68WZo0aWKCz1NPPWUeq/d95ZVX5LXXXpNKlSqZ59q5c6d53K5du+Tee+81rVs5c+Y0XUCffPKJ1/O5yrFu3ToZPHiwqQf9wtNWL1eI8PTFF1/I7bffLvny5ZP8+fPLLbfcIh988IHXfb755hu56667zNgbLaveX9fv6fTp0zJw4EB3/RQrVkxatmwpW7ZsSfV927p1q7Rq1co8f968eaV58+ayYcMG9+3PPfeclCtXzvx/yJAh5vXp81xNYmKijBw5UipXrmzKU6ZMGRk6dKhZ7mn69OnSrFkzU169X/Xq1WXKlCnJrtOfulL6ft1xxx2mrrQb1nd7S4m+Lm3dmjVrllx//fXmPa5bt64J855++eUXeeyxx8x9cuXKJYULF5Z//OMfSbY917awevVqc399jbrdpmcdX3/9tTzxxBNme9IQ+uijj5rPh34u9HNQsGBBc9E6dhzHax0aQHR7rlGjhnlNxYsXN48/efKk+z76fmoXtZZVn08v+jlx0efR7UvfR32f9H198cUXzbpdUvv8+NLPoL5PvnSd+r7pZ81F19mwYUNTT1pf+r5oi3JqdNvVMvlKaX+h25j+UNPPrG5nbdq0oes+DNEChIBbvHix2ZHqDiQ5Gjb09s8++8y97LfffpNbb73V7GAfeeQRqVq1qglEunPTMKW/HM+cOWPW+eOPP8pDDz0kN910kwk+Gj7++9//mnCVVidOnDBf+l26dDHjlfRLwfNL+cKFC6Y8ugPXwKM7wUaNGpkd8/Dhw80O8qOPPpL27dvLvHnzknTrPf744+YLSUOA7mT1C0G/XD/88EOvnbC+Hv1iio2NNV9sGkaWLFki999/v7nPihUrTDl1h6/rypIlizs0rF271tSd6tOnj6kzfQ4NEfr69EtT60zrKyX6urRuNVDol2f27NlNUNUvP/0yrFevnnTs2NGUTQOodm+2bt3aBKWU6BfYPffcY55f67BatWpm7NDEiRPlp59+8ho8rWFHX7/eP1u2bGYb0mCg6+jXr1+a6krpl7qGRS3zfffdZ+pk2LBhUqtWLVOPqdHXrO+Rhg1977VbSNf37bffugdYb9y4UdavX2+2HQ00+v7q69A60y9735ZEfT0aXJ599lnzAyA969DtKTo6WkaNGmXCqf7I0DrQdZQtW1ZeeOEF+fzzz82PEC2nhiIXDTtafw8++KB5Xfv375e33nrL1J8GaX3PdfvU59D3Vbs5leszoZ9DDZ76udR16fPp8+r7cPjwYfNYT8l9fpLTuXNnE1COHDliXpuLbje6X9C6cXn99dfNNqJjCTX4aVe6BsZPP/3UhJRA0HFtOs4tJibGhDt93fqeNG7c2NRVaqEfmYgDBFB8fLz+7HTatWt31fvdc8895n4JCQnmevfu3Z0sWbI4GzduTHLfK1eumL/PPvusecz8+fNTvM/06dPNffbv3+91+8qVK81y/ety++23m2Vvv/221331sbo8f/78zrFjx7xua968uVOrVi3nwoULXs/dsGFDp0qVKu5lrnK0aNHCXTY1aNAgJ2vWrKaeXPWVL18+p169es758+eTfU36V9cdExPjta5z5845FSpUcFq2bOleFhUV5fTr189Jq/bt2zs5cuRw9u3b517222+/mbI1adIkSd28/PLLqa5z5syZ5j1du3at13Ktb13HunXrvF6LL329FStWdF/3p64839f333/fvSwxMdGJjo52OnXqlGq59bF62bRpk3vZL7/84uTMmdPp0KHDVcscFxeX5Lld20Ljxo2dP//80+v+aV2H7zbQoEEDJyIiwunTp497mT5H6dKlTT246Hugj581a5bXcy1ZsiTJ8ho1ang91mXMmDFOnjx5nJ9++slr+fDhw802ffDgwVQ/P8nZvXu3uf+bb77ptfyxxx5z8ubN61VHvvV18eJFp2bNmk6zZs28lpcrV87p0aOH+/rIkSPNc/jy3V+cPn3aKVCggNO7d2+v+x05csR8tnyXI3OjCwwBpV0wSpuNr8Z1e0JCgvmVr60Bbdu2TXZGkavpWltY6tSpk+zg6eSat/2hv0z1F3FyOnXq5NWF98cff5iWGG1R0NeprU960VYW/bWo08P117En/fXrWTZtZbl8+bLp+lBLly4169LWJO2WSO41bdu2zaxbWzj0uVzPq60I2k2lXTOuLghtDdCuMv3l7C8tz1dffWVasTzHZJUoUcI8p/4S1/cprebOnWtafbQ1z1VmvWirlVq5cqX7vtqd4XLq1ClzP21t+Pnnn811f+vKRVswtEXPRVsQtZVM1+ePBg0amNY2F23taNeunXz55ZemvnzLfOnSJfPeaJeQvgfJdTn27t1bsmbN6rUsrevQLmLP16otc5rZdLmLPod+jjxfq74X2nWq3aGe74W+Rq0rz/ciJboO3X61RdNzHS1atDB14ttF6Pv5Scl1110nN9xwg1erqK5PW+10n+BZR57/11Y+3Ta0TP508fpDtzFthdYWTs/XqHWqde1PPSHzoAsMAeUKNq4g5E9Q0jEx+gWb2rFbdHC17lQDSbuyUhqYqWOQPO3du9d82YwYMcJcknPs2DGzTs8vTk/65aFc4y70NamrvXYNP0qb5VOiXwS6bh3novfTMRr65abdVNoNcrXB5lr/2syv41B8aYDRcHXo0CHT7ZQWWm7tekvpS1DrykW7YLRrT2eYaVl8X5t+eftTVy7aneQbirR+dOC9P6pUqZLsF7WWTetLu2p0Rty4ceNMV48GX88xN67QdrXtSaV1Hb7bk9aL0vfbd7nn2B59L3R9Ov4otfciJboOrT9/3s+UXm9KtBtMx99pHejnR8fq6fp0uSft6ho7dqz5UeA5jiy9P4BS+qy5Qrov7SJG+CAAIaB0x6stB6l90ejtuqPTHYp+CQRKSjtC1692X56/KFO7zdXK8uSTT5oWn+Tor3dPvr/4XXwHqF6N63l1XIf+Uk6OayyOtk7pL+IFCxaYVh19jI5j0MMS+DP2JZC03DrmZsKECcne7vrS1mCjLVnaUqT31eUaSnUsi44X8hxg669A1HtqdKyMBhcdFKwtRq4DQ+qYleTKnNy2ltZ1pPS6klvu+Vp1XRp+dGB3cvxpqdF1aAuSjhFLjgZEfz9bvjTo6FgibWXSutBxdVoXOu7KRce66fgfHUOoY7J0P6PjlrT+khsEn579gqvOdRyQ53gkFx2fhvDBu4mAu/vuu+Xf//636TrRgYO+dEemgz11IKVr56tB6Pvvv7/qenU2SWr3cbWwaDO2J1eX09/hakXRna42+weCvialr8s3PPneR+vIn+fVLwYdcKsX/RWtg5/1oIUpBSCtfx1su3v37iS36Yw3HXDt28LgDy33d999Z8LN1X6h64Bn/TWvg9k9Wzh8uxv8qatAcbUEeNKB21pPrrCgXTTa2vbqq6+676ODfn23vasJxDr8oXW3bNkyM4A/tWCS0nul69CJCIHa9n1bi7SLUrvBdAC/BnbtktUuahftAteuT+2G9FyuASg1nvsFz0M4+O4XXNuYhsWMeJ0ILYwBQsDpFGndyWrA0TENnnQcjc5U0i8SvZ/SL1jd2ekX4aZNm1L8JavdX/qFqq0bKd3HtQPzHI+gv/J0tszfpTtFnZ2js6N01ouv5Ka3p0ZPKaHdgNoNol98yb0m7crS16VTgPULKKXn1dfp222iZdbDCvhOO/dtPdByLFq0yGs68NGjR80vaw2x6Wn619Yo7dLQMOxLW/1cM6FcrRe+3T++X2z+1FWgaFec57gS7QLU+tEyuMqrf32f980330yxtTE5gViHv++FrnPMmDHJHqPHM3DpzMbkApiuQ+tFA4gvvb+u5+/QViCd2TZt2jQz7sa3+0vrSsOZZ93o9urPqViS2y/o9vfee+953U9bdnVb19l0OiYrEJ9xhC5agBBwOn5Cdyw6VVW7QHyPBK07t9mzZ7t3Skp3ONplowNfXVOmNWRok7i2JOmvNg1M+otZp73qVGgNBhqotOXg7bffNgOkdZxK/fr1TXO63qZTb3Wq7N/dObtMmjTJBAJ9XTqoVVuFNCjoF4NOxdeAlha6s9Vunocfftgcz0YHHeuvVV2PjjfRetSA+O6775oWHH19Omhbuw81XGgria5Dw6OOq9KxL3rcFK0L7RbTX/061dqzhSE5Oq5CB4Dqa9OWI23q16Cnwcnf4+f40oNdaleGBl4tp7Y+6JeXtirpcv0i1cG6Giq0y0sHvGpo1pCnoUnDm2fQ9KeuAkXHGemXoec0eKXTzz1bOrWrRLtq9JADug1ofesxavwViHX4Qz9XWrcaHnX8jNa5tmRqS5d+xnR6uet4O/q50mnfuk1oS5u+DzomRj9/+lnTMutRovV+GiL00Ab6udTPd3oOReEZsLR7WS/6ufVtgdFp7tpFqt1i+t5r66Z+HrWMqXW56+vV1kXdF+nr0DClQUtb8w4ePOi1jelr121XW061K9J1Hz1sh27DeugAhIlgT0ND+Nq+fbvTtWtXp0SJEk727NnNNGS9vmPHjmTvr1ONdTp80aJFncjISDMFWqd06xRmlxMnTjj9+/d3SpUqZaZt63Rfne56/Phx9310KrdOP9d1FC9e3HnqqaecpUuXJjsNXqf8+kptqreuX8upr0dfl5bl7rvvdj7++OMk02t9p/UnNx1fffLJJ2Yqfa5cucz04VtvvdWZPXu21322bt3qdOzY0SlcuLB5bTrV97777nOWL19ubtd6GjJkiFOnTh0zXVynLOv/J0+e7Phjy5YtZpq1Tj3OnTu3c8cddzjr169PU9340mnKL774oqlnLXPBggWdunXrOqNGjXJOnTrl9fpr165tppqXL1/ePGbatGnJHtIgtbpK6X3V7UTrLDX6nLrd/ec//zGHH9By33jjjUnes5MnTzoPPvigU6RIEVNnWne7du1KMgU7pW0hEOtwTe/+/fffk7xWff99/etf/zL1r3Wn24ge0mHo0KHmkAeeU77btGljbtd1e06J12nisbGxTuXKlc3nT8ut78Urr7xi3uv0bCOeGjVqZB778MMPJ3v71KlT3e9J1apVTb0kN8Xdt/7U5s2bzSEUtNxly5Z1JkyYcNXDZuh7oVPfdZusVKmS07NnT69DIyDzi9B/gh3CACBUaDeLHnyRX/pAeGMMEAAAsA4BCAAAWIcABAAArMMsMADwwLBIwA60AAEAAOsQgAAAgHXCvgtMz+2iZ8bWI8gG6oR5AAAg47uj9QCvejR7PSBsoIV9ANLwk57zGAEAgODTU9HoUe4DLewDkLb8uCowPeczAgAA115CQoJpwHB9jwda2AcgV7eXhh8CEAAAmUtGDV9hEDQAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOtmCXQAAAHBtlB/+WcDXeWB8G8mMaAECAADWIQABAADrEIAAAIB1QiYAjR8/XiIiImTgwIHuZRcuXJB+/fpJ4cKFJW/evNKpUyc5evRoUMsJAAAyv5AIQBs3bpR33nlHateu7bV80KBBsnjxYpk7d66sXr1afvvtN+nYsWPQygkAAMJD0APQmTNnpFu3bvLvf/9bChYs6F5+6tQpmTp1qkyYMEGaNWsmdevWlenTp8v69etlw4YNQS0zAADI3IIegLSLq02bNtKiRQuv5Zs3b5ZLly55La9ataqULVtW4uLiUlxfYmKiJCQkeF0AAABC5jhAc+bMkS1btpguMF9HjhyRHDlySIECBbyWFy9e3NyWknHjxsmoUaMypLwAACA8BK0F6NChQzJgwACZNWuW5MyZM2DrjY2NNd1nros+DwAAQEgEIO3iOnbsmNx0002SLVs2c9GBzm+88Yb5v7b0XLx4UeLj470ep7PAoqOjU1xvZGSk5M+f3+sCAAAQEl1gzZs3lx07dngte/DBB804n2HDhkmZMmUke/bssnz5cjP9Xe3evVsOHjwoDRo0CFKpAQBAOAhaAMqXL5/UrFnTa1mePHnMMX9cy3v16iWDBw+WQoUKmZacxx9/3ISf+vXrB6nUAAAgHIT0yVAnTpwoWbJkMS1AOrsrJiZGJk+eHOxiAQCATC7CcRxHwphOg4+KijIDohkPBACwWWY6G3xCBn9/B/04QAAAANcaAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYJ2gBqApU6ZI7dq1JX/+/ObSoEED+eKLL9y3N23aVCIiIrwuffr0CWaRAQBAGMgWzCcvXbq0jB8/XqpUqSKO48h7770n7dq1k61bt0qNGjXMfXr37i2jR492PyZ37txBLDEAAAgHQQ1Abdu29br+/PPPm1ahDRs2uAOQBp7o6OgglRAAAISjkBkDdPnyZZkzZ46cPXvWdIW5zJo1S4oUKSI1a9aU2NhYOXfu3FXXk5iYKAkJCV4XAACAkGkBUjt27DCB58KFC5I3b15ZsGCBVK9e3dx2//33S7ly5aRkyZKyfft2GTZsmOzevVvmz5+f4vrGjRsno0aNuoavAAAAZDYRjg6+CaKLFy/KwYMH5dSpU/Lxxx/Lu+++K6tXr3aHIE8rVqyQ5s2by969e6VSpUoptgDpxUVbgMqUKWPWrwOtAQCwVfnhnwV8nQfGt5GMoN/fUVFRGfb9HfQWoBw5ckjlypXN/+vWrSsbN26U119/Xd55550k961Xr575e7UAFBkZaS4AAAAhPwbI5cqVK14tOJ62bdtm/pYoUeIalwoAAISToLYA6aDmVq1aSdmyZeX06dPywQcfyKpVq+TLL7+Uffv2meutW7eWwoULmzFAgwYNkiZNmphjBwEAAGTKAHTs2DHp3r27HD582PTzabDR8NOyZUs5dOiQLFu2TF577TUzM0zH8XTq1EmeeeaZYBYZAACEgaAGoKlTp6Z4mwYeHQwNAAAQ9mOAAAAAMhoBCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgnaAGoClTpkjt2rUlf/785tKgQQP54osv3LdfuHBB+vXrJ4ULF5a8efNKp06d5OjRo8EsMgAACANBDUClS5eW8ePHy+bNm2XTpk3SrFkzadeunfzwww/m9kGDBsnixYtl7ty5snr1avntt9+kY8eOwSwyAAAIAxGO4zjBLoSnQoUKycsvvyz33nuvFC1aVD744APzf7Vr1y6pVq2axMXFSf369f1aX0JCgkRFRcmpU6dMKxMAALYqP/yzgK/zwPg2khEy+vs7ZMYAXb58WebMmSNnz541XWHaKnTp0iVp0aKF+z5Vq1aVsmXLmgCUksTERFNpnhcAAICQCkA7duww43siIyOlT58+smDBAqlevbocOXJEcuTIIQUKFPC6f/Hixc1tKRk3bpxJjK5LmTJlrsGrAAAAmUnQA9D1118v27Ztk2+++Ub69u0rPXr0kJ07d6Z7fbGxsaa5zHU5dOhQQMsLAAAyv2zBLoC28lSuXNn8v27durJx40Z5/fXXpXPnznLx4kWJj4/3agXSWWDR0dEprk9bkvQCAAAQsi1Avq5cuWLG8WgYyp49uyxfvtx92+7du+XgwYNmjBAAAECmbAHS7qpWrVqZgc2nT582M75WrVolX375pRm/06tXLxk8eLCZGaYjwB9//HETfvydAQYAABByAejYsWPSvXt3OXz4sAk8elBEDT8tW7Y0t0+cOFGyZMliDoCorUIxMTEyefLkYBYZAACEgZA7DlCgcRwgAAD+h+MAhfAYIAAAgIxGAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKyTrgC0ZcsW2bFjh/v6okWLpH379vLUU0/JxYsXA1k+AACA0AhAjz76qPz000/m/z///LN06dJFcufOLXPnzpWhQ4cGuowAAADBD0Aafm644Qbzfw09TZo0kQ8++EBmzJgh8+bNC2wJAQAAQiEAOY4jV65cMf9ftmyZtG7d2vy/TJkycvz48cCWEAAAIBQC0M033yxjx46VmTNnyurVq6VNmzZm+f79+6V48eKBLiMAAEDwA9Brr71mBkL3799fnn76aalcubJZ/vHHH0vDhg0DW0IAAIAAy5aeB9WuXdtrFpjLyy+/LFmzZg1EuQAAAELvOEDx8fHy7rvvSmxsrPzxxx9m2c6dO+XYsWN+r2PcuHFyyy23SL58+aRYsWJmKv3u3bu97tO0aVOJiIjwuvTp0ye9xQYAAEhfC9D27dulefPmUqBAATlw4ID07t1bChUqJPPnz5eDBw/K+++/79d6dPxQv379TAj6888/zXGE7rzzThOk8uTJ476frn/06NHu6zrlHgAA4JoGoMGDB8uDDz4oL730kmm9cdHZYPfff7/f61myZInXdZ1Gry1BmzdvNlPrPQNPdHR0eooKAAAQmC6wjRs3moMh+ipVqpQcOXJE0uvUqVPmr7YmeZo1a5YUKVJEatasabrczp07l+I6EhMTJSEhwesCAADwt1uAIiMjkw0WeoDEokWLpmeV5rhCAwcOlEaNGpmg46ItSuXKlZOSJUuarrdhw4aZcULa3ZbSuKJRo0alqwwAAMAOEY4e1TCNHn74YTlx4oR89NFHprVGg4nO/tJBzNp1pdPk06pv377yxRdfyNdffy2lS5dO8X4rVqww44/27t0rlSpVSrYFSC8uGtT0AI3aupQ/f/40lwsAgHBRfvhnAV/ngfH/OxZgoOn3d1RUVIZ9f6erC+zVV1+VM2fOmPE658+fl9tvv90cC0jHAz3//PNpXp8eT+jTTz+VlStXXjX8qHr16pm/GoBSap3SivK8AAAA/O0uME1kS5culXXr1sl3331nwtBNN90kLVq0SNN6tPHp8ccflwULFsiqVaukQoUKqT5m27Zt5m+JEiXSU3QAAID0BSAXHa+jl/TSKfB6EtVFixaZ1iPXAGoNWLly5ZJ9+/aZ23V2WeHChU1X26BBg0w3mx6MEQAA4Jp1gT3xxBPyxhtvJFn+1ltvmYHM/poyZYrp29ODHWqLjuvy4Ycfmttz5MhhTraqxwaqWrWq/POf/5ROnTrJ4sWL01NsAACA9LcAzZs3Tz755JMky/U8YOPHj/d7EHRq46918LIeLBEAACDoLUA6A0y7qXzpgOPjx48HolwAAAChFYB0xpfvUZyVTmOvWLFiIMoFAAAQeqfC0Knrv//+uzRr1swsW758uZken55jAAEAAIR8AHrooYfMwQb1mD9jxowxy8qXL28GNXfv3j3QZQQAAAiNafB65Ga9aCuQTlnPmzdvYEsGAAAQiscBUuk99xcAAECmGgR99OhReeCBB8wJSrNly2bOA+Z5AQAACLsWoJ49e8rBgwdlxIgR5sCFERERgS8ZAABAKAUgPWP72rVr5YYbbgh8iQAAAEKxC0yP0JzaUZwBAADCKgDpsX6GDx8uBw4cCHyJAAAAQrELrHPnznLu3DmpVKmS5M6dW7Jnz+51+x9//BGo8gEAAIRGAOJozwAAwLoA1KNHj8CXBAAAIJTHAKl9+/bJM888I127dpVjx465T4b6ww8/BLJ8AAAAoRGAVq9eLbVq1ZJvvvlG5s+fL2fOnDHLv/vuOxk5cmSgywgAABD8AKQzwMaOHStLly6VHDlyuJfrmeE3bNgQyPIBAACERgDasWOHdOjQIcnyYsWKyfHjxwNRLgAAgNAKQAUKFJDDhw8nWb5161YpVapUIMoFAAAQWgGoS5cuMmzYMDly5Ig5D9iVK1dk3bp18uSTT0r37t0DX0oAAIBgB6AXXnhBqlatak6JoQOgq1evLk2aNJGGDRuamWEAAABhdRwgPQeYtvy88cYb8uyzz5rxQBqCbrzxRqlSpUrGlBIAACDYAahy5crmeD8aeLQVCAAAIKy7wLJkyWKCz4kTJzKmRAAAAKE4Bmj8+PEyZMgQ+f777wNfIgAAgFA8F5jO9NKzwdepU8ccCDFXrlxet3M2eAAAEMo4GzwAALBOmgPQpUuXzLnARowYIRUqVMiYUgEAAITSGKDs2bPLvHnzMqY0AAAAoToIun379rJw4cLAlwYAACBUxwDpNPjRo0eb01/UrVtX8uTJ43X7E088EajyAQAAhEYAmjp1qjkh6ubNm83Fk54bjAAEAADCrgts//79KV5+/vlnv9czbtw4ueWWWyRfvnxSrFgx07W2e/dur/tcuHBB+vXrJ4ULF5a8efNKp06d5OjRo+kpNgAAQPoDUKDobDINNxs2bJClS5eaGWZ33nmnnD171n2fQYMGyeLFi2Xu3Lnm/r/99pt07NgxmMUGAAA2doE99NBDV7192rRpfq1nyZIlXtdnzJhhWoK0W03PLn/q1CnT3fbBBx9Is2bNzH2mT58u1apVM6Gpfv366Sk+AACwXLoC0MmTJ72ua8uNnhYjPj7eHVTSQwOPKlSokPmrQUjX3aJFC/d9qlatKmXLlpW4uLhkA1BiYqK5uCQkJKS7PAAAIDylKwAtWLAgybIrV65I3759pVKlSukqiD5+4MCB0qhRI6lZs6ZZduTIEXOqDR1w7al48eLmtpTGFY0aNSpdZQAAAHYI2BggPUv84MGDZeLEiel6vI4F0lakOXPm/K1yxMbGmpYk1+XQoUN/a30AACD8pKsFKCX79u2TP//8M82P69+/v3z66aeyZs0aKV26tHt5dHS0XLx40XStebYC6SwwvS05kZGR5gIAABDQAKQtPZ4cx5HDhw/LZ599Jj169PB7Pfq4xx9/3HSprVq1Ksm5xfQgi3rqjeXLl5vp70qnyR88eFAaNGiQnqIDAACkLwBt3bo1SfdX0aJF5dVXX011hphvt5fO8Fq0aJE5FpBrXE9UVJTkypXL/O3Vq5cJXDowOn/+/CYwafhhBhgAALimAWjlypUSCFOmTDF/mzZt6rVcp7r37NnT/F/HFGnA0hYgnd0VExMjkydPDsjzAwAAO6UrAOkRn3Wsj54TzNOePXtMl1X58uX97gJLTc6cOWXSpEnmAgAAELRZYNo6s379+iTLv/nmG3fLDQAAQFgFIB0DpMfr8aXjcrZt2xaIcgEAAIRWANIzvp8+fTrJcj3uzuXLlwNRLgAAgNAKQHqeLj3ismfY0f/rssaNGweyfAAAAKExCPrFF180Iej666+X2267zSxbu3atOe/WihUrAl1GAACA4LcAVa9eXbZv3y733XefHDt2zHSHde/eXXbt2uU+jxcAAEDYnQqjZMmS8sILLwS2NAAAAKHaAqQHKpw7d26S5brsvffeC0S5AAAAQisA6WDnIkWKJFlerFgxWoUAAEB4BiA9GanviUtVuXLlzG0AAABhF4C0pUcHQfv67rvvpHDhwoEoFwAAQGgFoK5du8oTTzxhToqqx//Ri05/HzBggHTp0iXwpQQAAAj2LLAxY8bIgQMHpHnz5pIt2/9WoSGoR48ejAECAADhGYBy5MghH374oTz55JMmCOXKlUtq1aplxgABAACEXQCKj4+Xp59+2gSgkydPmmUFCxY0XV9jx46VAgUKZEQ5AQAAghOA/vjjD2nQoIH8+uuv0q1bN6lWrZpZvnPnTpkxY4YsX75c1q9fbwIRAABAWASg0aNHm+6vffv2SfHixZPcduedd5q/EydODHQ5AQAAgjMLbOHChfLKK68kCT8qOjpaXnrpJVmwYEHgSgcAABDsAHT48GGpUaNGirfriVCPHDkSiHIBAACERgDS01/orK+U7N+/XwoVKhSIcgEAAIRGAIqJiTEzwC5evJjktsTERBkxYoTcddddgSwfAABA8AdB33zzzVKlShXp16+fVK1aVRzHkR9//FEmT55sQtDMmTMDX0oAAIBgBaDSpUtLXFycPPbYYxIbG2vCj4qIiJCWLVvKW2+9JWXKlAlk+QAAAIJ/IEQ9C/wXX3xhDoK4Z88es6xy5cqM/QEAAOF9KgylBzu89dZbA1saAACAUD0bPAAAQGZGAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsE5QA9CaNWukbdu2UrJkSXM06YULF3rd3rNnT7Pc88K5xgAAwN8V1AB09uxZqVOnjkyaNCnF+2jgOXz4sPsye/bsa1pGAAAQftJ9JOhAaNWqlblcTWRkpERHR1+zMgEAgPAX8mOAVq1aJcWKFZPrr79e+vbtKydOnLjq/fWM9AkJCV4XAACATBOAtPvr/fffl+XLl8uLL74oq1evNi1Gly9fTvEx48aNk6ioKPeFs9MDAICQ6gJLTZcuXdz/r1WrltSuXVsqVapkWoWaN2+e7GNiY2Nl8ODB7uvaAkQIAgAAmaYFyFfFihWlSJEisnfv3quOGcqfP7/XBQAAINMGoP/+979mDFCJEiWCXRQAAJCJBbUL7MyZM16tOfv375dt27ZJoUKFzGXUqFHSqVMnMwts3759MnToUKlcubLExMQEs9gAACCTC2oA2rRpk9xxxx3u666xOz169JApU6bI9u3b5b333pP4+HhzsMQ777xTxowZY7q5AAAAMmUAatq0qTiOk+LtX3755TUtDwAAsEOmGgMEAAAQCAQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1ghqA1qxZI23btpWSJUtKRESELFy40Ot2x3Hk2WeflRIlSkiuXLmkRYsWsmfPnqCVFwAAhIegBqCzZ89KnTp1ZNKkScne/tJLL8kbb7whb7/9tnzzzTeSJ08eiYmJkQsXLlzzsgIAgPCRLZhP3qpVK3NJjrb+vPbaa/LMM89Iu3btzLL3339fihcvblqKunTpco1LCwAAwkXIjgHav3+/HDlyxHR7uURFRUm9evUkLi4uxcclJiZKQkKC1wUAACBkWoCuRsOP0hYfT3rddVtyxo0bJ6NGjcrw8gEAwlf54Z9lyHoPjG8T1OdCJmgBSq/Y2Fg5deqU+3Lo0KFgFwkAAISYkA1A0dHR5u/Ro0e9lut1123JiYyMlPz583tdAAAAMkUAqlChggk6y5cvdy/T8Tw6G6xBgwZBLRsAAMjcgjoG6MyZM7J3716vgc/btm2TQoUKSdmyZWXgwIEyduxYqVKliglEI0aMMMcMat++fTCLDQAAMrmgBqBNmzbJHXfc4b4+ePBg87dHjx4yY8YMGTp0qDlW0COPPCLx8fHSuHFjWbJkieTMmTOIpQYAAJldUANQ06ZNzfF+UqJHhx49erS5AAAAhP0YIAAAgIxCAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWCekA9Bzzz0nERERXpeqVasGu1gAACCTyyYhrkaNGrJs2TL39WzZQr7IAAAgxIV8mtDAEx0dHexiAACAMBLSXWBqz549UrJkSalYsaJ069ZNDh48eNX7JyYmSkJCgtcFAAAg0wSgevXqyYwZM2TJkiUyZcoU2b9/v9x2221y+vTpFB8zbtw4iYqKcl/KlClzTcsMAABCX0gHoFatWsk//vEPqV27tsTExMjnn38u8fHx8tFHH6X4mNjYWDl16pT7cujQoWtaZgAAEPpCfgyQpwIFCsh1110ne/fuTfE+kZGR5gIAAJApW4B8nTlzRvbt2yclSpQIdlEAAEAmFtIB6Mknn5TVq1fLgQMHZP369dKhQwfJmjWrdO3aNdhFAwAAmVhId4H997//NWHnxIkTUrRoUWncuLFs2LDB/B8AACAsA9CcOXOCXQQAABCGQroLDAAAICMQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdUL6VBgAgNBXfvhnAV/ngfFtrsnzpPRcCH+0AAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWyRbsAmRm5Yd/liHrPTC+Dc8V4OcJ1+fK7O9VuD5XsLeLa/1cQGZECxAAALAOAQgAAFiHAAQAAKyTKQLQpEmTpHz58pIzZ06pV6+efPvtt8EuEgAAyMRCPgB9+OGHMnjwYBk5cqRs2bJF6tSpIzExMXLs2LFgFw0AAGRSIR+AJkyYIL1795YHH3xQqlevLm+//bbkzp1bpk2bFuyiAQCATCqkA9DFixdl8+bN0qJFC/eyLFmymOtxcXFBLRsAAMi8Qvo4QMePH5fLly9L8eLFvZbr9V27diX7mMTERHNxOXXqlPmbkJAQ8PJdSTwnGSG5svJcf+95wvW5Mvt7Fa7PFeztIlyfK7NvF+H6XAkZ8P3quV7HcTJk/brikPXrr7/qq3bWr1/vtXzIkCHOrbfemuxjRo4caR7DhQsXLly4cMn8l0OHDmVIxgjpFqAiRYpI1qxZ5ejRo17L9Xp0dHSyj4mNjTWDpl2uXLkif/zxhxQuXFgiIiICkkjLlCkjhw4dkvz584vNqIu/UBd/oS7+Ql38hbr4C3XhX11oy8/p06elZMmSkhFCOgDlyJFD6tatK8uXL5f27du7A41e79+/f7KPiYyMNBdPBQoUCHjZ9I2yfcN1oS7+Ql38hbr4C3XxF+riL9RF6nURFRUlGSWkA5DS1pwePXrIzTffLLfeequ89tprcvbsWTMrDAAAICwDUOfOneX333+XZ599Vo4cOSI33HCDLFmyJMnAaAAAgLAJQEq7u1Lq8rrWtHtND8ro281mI+riL9TFX6iLv1AXf6Eu/kJdhEZdROhI6Gv+rAAAAEEU0gdCBAAAyAgEIAAAYB0CEAAAsA4BCAAAWMe6ADRp0iQpX7685MyZU+rVqyfffvvtVe+vxx26/vrrJVeuXOZolYMGDZILFy64bx83bpzccsstki9fPilWrJg5YOPu3bu91tG0aVNzFGrPS58+fSTc6uK5555L8jqrVq3qtQ69f79+/cyRufPmzSudOnVKcqTvcKgLXZdvXehFX3s4bReXLl2S0aNHS6VKlcz969SpYw5TkdZ1hsN24U9d2LK/8KcubNlf+FMXmXV/sWbNGmnbtq05UrOWZ+HChak+ZtWqVXLTTTeZWV+VK1eWGTNmBG9/4Vhkzpw5To4cOZxp06Y5P/zwg9O7d2+nQIECztGjR5O9/6xZs5zIyEjzd//+/c6XX37plChRwhk0aJD7PjExMc706dOd77//3tm2bZvTunVrp2zZss6ZM2fc97n99tvNcx0+fNh9OXXqlBNudaHnYatRo4bX6/z999+91tOnTx+nTJkyzvLly51NmzY59evXdxo2bOiEW10cO3bMqx6WLl1qzmmzcuXKsNouhg4d6pQsWdL57LPPnH379jmTJ092cubM6WzZsiVN6wyH7cKfurBlf+FPXdiyv/CnLjLr/uLzzz93nn76aWf+/PmmvAsWLLjq/X/++Wcnd+7czuDBg52dO3c6b775ppM1a1ZnyZIlQdlfWBWA9ASq/fr1c1+/fPmy2TDHjRuX7P31vs2aNfNapm9co0aNUnwO3ZB1Q1i9erXXhjtgwAAn3OtCd2h16tRJ8Tnj4+Od7NmzO3PnznUv+/HHH019xcXFOeG8Xej7X6lSJefKlSthtV1o8Hvrrbe8lnXs2NHp1q2b3+sMl+3Cn7qwZX/hT13Ysr9Iz3aRWfYXnvwJQBoGNfR66ty5s/lhEIz9hTVdYBcvXpTNmzdLixYt3MuyZMlirsfFxSX7mIYNG5rHuJrffv75Z/n888+ldevWKT7PqVOnzN9ChQp5LZ81a5Y5uWvNmjXNCVvPnTsn4VgXe/bsMc2hFStWlG7dusnBgwfdt+njtTnY83m1ybts2bIpPm84bBf6HP/5z3/koYceSnJC3sy+XSQmJppmak/aLfj111/7vc5w2S5Sqwub9hf+1oUN+4u0bheZZX+RHlpHnnWnYmJi3HV3rfcXmeJI0IFw/PhxuXz5cpJTaOj1Xbt2JfuY+++/3zyucePG5qy0f/75p+lzfeqpp5K9v56odeDAgdKoUSOzgXqup1y5cuaDvn37dhk2bJjp958/f76EU11oX6325+rYmMOHD8uoUaPktttuk++//96MedBTmegJbn1PTqvPq7eF63ah/eLx8fHSs2fPJOvJ7NuF7rwmTJggTZo0MWMc9ETFWn5dj7/rDJftIrW6sGl/4U9d2LK/SOt2kVn2F+mh71tydadnhD9//rycPHnymu4vrAlA6aGDtV544QWZPHmy+bDu3btXBgwYIGPGjJERI0Ykub8OytIPr2+yf+SRR9z/r1WrlpQoUUKaN28u+/btMx+IcKmLVq1aue9fu3Ztcz/9wH700UfSq1cvsXW7mDp1qqkb3XGF23bx+uuvS+/evc0vMP21quXWExVPmzZNbJPWugjn/YU/dWHL/iKt20U47y9CjTVdYNpsmDVr1iQjxfV6dHR0so/RL7MHHnhAHn74YbPBdejQwXzx6UwO/fXmSc9V9umnn8rKlSuldOnSVy2LftCVfnGGY124aEK/7rrr3K9T161NnPrrxt/nzex18csvv8iyZcvMfVOTGbeLokWLml+sZ8+eNa9Vf6XprAzt0vB3neGyXaRWFzbtL9JSF+G+v0hLXWSm/UV6aB0lV3f58+c33YLXen9hTQDSJrO6deua5kcX/bLS6w0aNEj2Mdq/qv2PnvTNUa5TqOlf3ZktWLBAVqxYIRUqVEi1LNu2bTN/NcGHU134OnPmjPl14nqd+pzZs2f3el5twtV+/5SeN7PXxfTp08105zZt2oTlduGiYxxKlSplugPnzZsn7dq183ud4bJdpFYXNu0v/KkLW/YXaamLzLS/SA+tI8+6U0uXLnXX3TXfXzgW0el1On15xowZZgreI488YqbXHTlyxNz+wAMPOMOHD/eapZAvXz5n9uzZZvreV199ZUbm33fffe779O3b14mKinJWrVrlNT3x3Llz5va9e/c6o0ePNlP1dMr0okWLnIoVKzpNmjRxwq0u/vnPf5p60Ne5bt06p0WLFk6RIkXMTBfP6Ys67XfFihWmTho0aGAu4VYXrtkL+lqHDRuW5DnDZbvYsGGDM2/ePDO9d82aNWZ2XIUKFZyTJ0/6vc5w2S78qQtb9hf+1IUt+wt/6iKz7i9Onz7tbN261Vw0TkyYMMH8/5dffjG3az1offhOgx8yZIiZuTVp0qRkp8Ffq/2FVQFI6XEHtOL0OAM63U43Ts9phj169HBfv3TpkvPcc8+ZLzc9boMed+Cxxx7z2nD1TU/uosf6UAcPHjQbaaFChcybWrlyZfPmB/v4DRlRFzqdUad86vpKlSplrusH19P58+fN4woWLGg+CB06dDBfAOFWF0qPD6Tbwu7du5M8X7hsF/oFVq1aNfMaChcubHZ2v/76a5rWGS7bhT91Ycv+wp+6sGV/4e9nJDPuL1auXJns9ux6/fpX68P3MTfccIOpOw1xrm0/GPuLCP0nXW1ZAAAAmZQ1Y4AAAABcCEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAFAgOlJL/X8TwBCFwEICGM9e/Y0X8Z60fPsVK5cWUaPHm3OR5RZES4ABEK2gKwFQMi66667zEkWExMT5fPPP5d+/fqZkwnGxsameV2XL182AcT3ZLCZ0aVLl0w9ALBT5t+LAbiqyMhIiY6OlnLlyknfvn2lRYsW8sknn5jbJkyYILVq1ZI8efJImTJl5LHHHjNn5XaZMWOGFChQwNy/evXqZl161uWNGzdKy5YtpUiRIhIVFSW33367bNmyxet5NSi98847cvfdd0vu3LmlWrVqEhcXJ3v37pWmTZua52zYsKE5A7inRYsWyU033WTOnl2xYkUZNWqUu8WqfPny5m+HDh3M+l3XU3ucqzxTpkyRe+65xzz3888/n6SunnrqKalXr16S5XXq1DEtZ8qf1+5p1apV5rnj4+O9zuStyw4cOOBe9vXXX8ttt90muXLlMu/FE088IWfPnk1xvQD+HgIQYBn9gr148aL5v7bkvPHGG/LDDz/Ie++9JytWrJChQ4d63f/cuXPy4osvyrvvvmvuV6xYMTl9+rT06NHDfGlv2LBBqlSpIq1btzbLPY0ZM0a6d+9uvvCrVq0q999/vzz66KOm9WnTpk16Mmbp37+/+/5r16419x8wYIDs3LnTBCgNYa6wouFDaYvW4cOH3ddTe5zLc889Z8LTjh075KGHHkpSN926dZNvv/3WK5Tpa96+fbspu/L3taeFPp+21HXq1Mk814cffmjW71k3AAIszadPBZBp6NmY27VrZ/5/5coVZ+nSpeZs0k8++WSy9587d645Y7WLnqlZdxPbtm276vNcvnzZyZcvn7N48WL3Mn3cM888474eFxdnlk2dOtW9bPbs2U7OnDnd15s3b+688MILXuueOXOmOWu453oXLFjgdR9/Hzdw4EAnNXXq1HFGjx7tvh4bG+vUq1cvza/dVUbXGbNPnjzpvn3r1q1m2f79+831Xr16OY888ojXeteuXetkyZLFnPkaQOAxBggIc59++qnkzZvXjHm5cuWKacnQlhC1bNkyGTdunOzatUsSEhJMl9GFCxdMq492WykdPF27dm2vdR49elSeeeYZ071z7NgxMzZIH6PdY548H1e8eHHzV7vcPJfp8+lz58+fX7777jtZt26dV8uNrtu3TL78fdzNN9+can1pK9C0adNkxIgRpoVq9uzZMnjw4DS/9rTQ8mvLz6xZs9zL9Ln1/dq/f7/pPgQQWAQgIMzdcccdZuyLBpmSJUtKtmz/+9jr+BMdn6PjgjQ4FCpUyHS79OrVy3SRuUKDdpnpeBVP2gV04sQJef31183YIh0b1KBBA3fXmovnIGPXOpJbpl/0Sscf6didjh07JnkdOrYnJf4+Tsf+pKZr164ybNgwM67n/PnzcujQIencuXOaX7uLa8D4/xqG/kfDqG/5tWtQx/34Klu2bKplBpB2BCAgzOmXvk5/97V582YTPF599VX3l/RHH33k1zq1tWXy5Mlm7IvSkHD8+PG/XVYdxLx79+5ky+uiAUpbXdL6OH+VLl3aDGzW1hgNQDrgWcc9pfe1Fy1a1PzVMUsFCxY0/9cxUb7l17FLgSg/AP8QgABL6ZettkS8+eab0rZtW/PF/vbbb/v1WB34O3PmTNOlpN1XQ4YMMS1Ff9ezzz5rWqW01ePee+81wUy7h77//nsZO3asuY/O/Fq+fLk0atTItL5oqPDncWmh3WAjR440rToTJ078W69d61lndWm3o7a0/fTTTyZ0etIWp/r165tBzw8//LAJrRqIli5dKm+99Vaayw8gdcwCAyylU7t1GrzO8KpZs6Zp8dDxQP6YOnWqnDx50rRcPPDAA6brxrOVJL1iYmLMmKWvvvpKbrnlFhMKNIBoV5OLhgcNBhoqbrzxRr8flxYaorSbS8f2tG/f/m+9dm2x0nFEOs5Kx0RpffuGMl2+evVqE450Kry+Lg112mUJIGNE6EjoDFo3AABASKIFCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADr/D8Hz8gRf4RhTgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -160,7 +160,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24afeb59-e8a8-48a5-b555-c797fda6bac5", + "id": "1c1f49d5-234a-4a88-b030-b210bb16af6b", "metadata": {}, "outputs": [], "source": [] From 177efbe79070bde8ad6bc8c3793a149283cbe7b0 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Mon, 28 Jul 2025 18:36:46 +0200 Subject: [PATCH 07/18] ENH: theoretical tests notebook --- ...risk_control_theoretical_tests_proto.ipynb | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 4b59faa2a..3b7210699 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 5, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -80,12 +80,12 @@ "predict_params = np.linspace(0, 0.99, 100)\n", "confidence_level = 0.7\n", "\n", - "n_repeats = 100" + "n_repeats = 10000" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 11, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -97,33 +97,27 @@ }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of valid thresholds according to LTT across all iterations: 79\n", - "Number of actual valid thresholds across all iterations: 0\n" + "ename": "NameError", + "evalue": "name 'all_valid_parameters' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[11], line 26\u001b[0m\n\u001b[1;32m 23\u001b[0m controller\u001b[38;5;241m.\u001b[39mcalibrate(X_calibrate, y_calibrate)\n\u001b[1;32m 25\u001b[0m valid_parameters \u001b[38;5;241m=\u001b[39m controller\u001b[38;5;241m.\u001b[39mvalid_thresholds\n\u001b[0;32m---> 26\u001b[0m \u001b[43mall_valid_parameters\u001b[49m\u001b[38;5;241m.\u001b[39mappend(valid_parameters)\n\u001b[1;32m 28\u001b[0m all_valid_parameters \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate([x \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters \u001b[38;5;28;01mif\u001b[39;00m x\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m]) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28many\u001b[39m(x\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters) \u001b[38;5;28;01melse\u001b[39;00m np\u001b[38;5;241m.\u001b[39marray([])\n\u001b[1;32m 30\u001b[0m nb_actual_valid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msum\u001b[39m(\u001b[38;5;241m1\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters \u001b[38;5;28;01mif\u001b[39;00m theoretical_value \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m target_level)\n", + "\u001b[0;31mNameError\u001b[0m: name 'all_valid_parameters' is not defined" ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAPkBJREFUeJzt3Qd8U+X+x/FfWWWXTdlb2aioTBEZVkBkeQXkL6CIgqCMK6MqIkPBBS5A75WhXARFljhQNghFmYIiCAjCVYYgpcyCcP6v33NfiUna0rSmJM3zeb9eoeQkOXny5OTkm2ecE+E4jiMAAAAWyRLsAgAAAFxrBCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIABpsmfPHrnzzjslKipKIiIiZOHChRJKmjZtKjVr1gx2MRBE5cuXl549ewa7GAhxBCBkmB9++EH+7//+T0qVKiWRkZFSsmRJ6datm1mOzKtHjx6yY8cOef7552XmzJly8803B7tICKDJkyfLjBkzgl0MIMNly/ingI3mz58vXbt2lUKFCkmvXr2kQoUKcuDAAZk6dap8/PHHMmfOHOnQoUOwi4k0On/+vMTFxcnTTz8t/fv3D3ZxkEEBqEiRIrSgIOwRgBBw+/btkwceeEAqVqwoa9askaJFi7pvGzBggNx2223m9u3bt5v7hJJz585J7ty5kyz/888/5cqVK5IjRw6x2e+//27+FihQINhFydTOnj0refLkEVvw+UEoogsMAffyyy+bIPGvf/3LK/wo/WX5zjvvmC+Al156yeu2X3/91bQWaVeZdplpq1Hfvn3l4sWL7vvEx8fLoEGDTB+/3qd06dLSvXt3OX78uLldm+51XIq2NnlatWqVWa5/fceKbN68WZo0aWKCz1NPPWUeq/d95ZVX5LXXXpNKlSqZ59q5c6d53K5du+Tee+81rVs5c+Y0XUCffPKJ1/O5yrFu3ToZPHiwqQf9wtNWL1eI8PTFF1/I7bffLvny5ZP8+fPLLbfcIh988IHXfb755hu56667zNgbLaveX9fv6fTp0zJw4EB3/RQrVkxatmwpW7ZsSfV927p1q7Rq1co8f968eaV58+ayYcMG9+3PPfeclCtXzvx/yJAh5vXp81xNYmKijBw5UipXrmzKU6ZMGRk6dKhZ7mn69OnSrFkzU169X/Xq1WXKlCnJrtOfulL6ft1xxx2mrrQb1nd7S4m+Lm3dmjVrllx//fXmPa5bt64J855++eUXeeyxx8x9cuXKJYULF5Z//OMfSbY917awevVqc399jbrdpmcdX3/9tTzxxBNme9IQ+uijj5rPh34u9HNQsGBBc9E6dhzHax0aQHR7rlGjhnlNxYsXN48/efKk+z76fmoXtZZVn08v+jlx0efR7UvfR32f9H198cUXzbpdUvv8+NLPoL5PvnSd+r7pZ81F19mwYUNTT1pf+r5oi3JqdNvVMvlKaX+h25j+UNPPrG5nbdq0oes+DNEChIBbvHix2ZHqDiQ5Gjb09s8++8y97LfffpNbb73V7GAfeeQRqVq1qglEunPTMKW/HM+cOWPW+eOPP8pDDz0kN910kwk+Gj7++9//mnCVVidOnDBf+l26dDHjlfRLwfNL+cKFC6Y8ugPXwKM7wUaNGpkd8/Dhw80O8qOPPpL27dvLvHnzknTrPf744+YLSUOA7mT1C0G/XD/88EOvnbC+Hv1iio2NNV9sGkaWLFki999/v7nPihUrTDl1h6/rypIlizs0rF271tSd6tOnj6kzfQ4NEfr69EtT60zrKyX6urRuNVDol2f27NlNUNUvP/0yrFevnnTs2NGUTQOodm+2bt3aBKWU6BfYPffcY55f67BatWpm7NDEiRPlp59+8ho8rWFHX7/eP1u2bGYb0mCg6+jXr1+a6krpl7qGRS3zfffdZ+pk2LBhUqtWLVOPqdHXrO+Rhg1977VbSNf37bffugdYb9y4UdavX2+2HQ00+v7q69A60y9735ZEfT0aXJ599lnzAyA969DtKTo6WkaNGmXCqf7I0DrQdZQtW1ZeeOEF+fzzz82PEC2nhiIXDTtafw8++KB5Xfv375e33nrL1J8GaX3PdfvU59D3Vbs5leszoZ9DDZ76udR16fPp8+r7cPjwYfNYT8l9fpLTuXNnE1COHDliXpuLbje6X9C6cXn99dfNNqJjCTX4aVe6BsZPP/3UhJRA0HFtOs4tJibGhDt93fqeNG7c2NRVaqEfmYgDBFB8fLz+7HTatWt31fvdc8895n4JCQnmevfu3Z0sWbI4GzduTHLfK1eumL/PPvusecz8+fNTvM/06dPNffbv3+91+8qVK81y/ety++23m2Vvv/221331sbo8f/78zrFjx7xua968uVOrVi3nwoULXs/dsGFDp0qVKu5lrnK0aNHCXTY1aNAgJ2vWrKaeXPWVL18+p169es758+eTfU36V9cdExPjta5z5845FSpUcFq2bOleFhUV5fTr189Jq/bt2zs5cuRw9u3b517222+/mbI1adIkSd28/PLLqa5z5syZ5j1du3at13Ktb13HunXrvF6LL329FStWdF/3p64839f333/fvSwxMdGJjo52OnXqlGq59bF62bRpk3vZL7/84uTMmdPp0KHDVcscFxeX5Lld20Ljxo2dP//80+v+aV2H7zbQoEEDJyIiwunTp497mT5H6dKlTT246Hugj581a5bXcy1ZsiTJ8ho1ang91mXMmDFOnjx5nJ9++slr+fDhw802ffDgwVQ/P8nZvXu3uf+bb77ptfyxxx5z8ubN61VHvvV18eJFp2bNmk6zZs28lpcrV87p0aOH+/rIkSPNc/jy3V+cPn3aKVCggNO7d2+v+x05csR8tnyXI3OjCwwBpV0wSpuNr8Z1e0JCgvmVr60Bbdu2TXZGkavpWltY6tSpk+zg6eSat/2hv0z1F3FyOnXq5NWF98cff5iWGG1R0NeprU960VYW/bWo08P117En/fXrWTZtZbl8+bLp+lBLly4169LWJO2WSO41bdu2zaxbWzj0uVzPq60I2k2lXTOuLghtDdCuMv3l7C8tz1dffWVasTzHZJUoUcI8p/4S1/cprebOnWtafbQ1z1VmvWirlVq5cqX7vtqd4XLq1ClzP21t+Pnnn811f+vKRVswtEXPRVsQtZVM1+ePBg0amNY2F23taNeunXz55ZemvnzLfOnSJfPeaJeQvgfJdTn27t1bsmbN6rUsrevQLmLP16otc5rZdLmLPod+jjxfq74X2nWq3aGe74W+Rq0rz/ciJboO3X61RdNzHS1atDB14ttF6Pv5Scl1110nN9xwg1erqK5PW+10n+BZR57/11Y+3Ta0TP508fpDtzFthdYWTs/XqHWqde1PPSHzoAsMAeUKNq4g5E9Q0jEx+gWb2rFbdHC17lQDSbuyUhqYqWOQPO3du9d82YwYMcJcknPs2DGzTs8vTk/65aFc4y70NamrvXYNP0qb5VOiXwS6bh3novfTMRr65abdVNoNcrXB5lr/2syv41B8aYDRcHXo0CHT7ZQWWm7tekvpS1DrykW7YLRrT2eYaVl8X5t+eftTVy7aneQbirR+dOC9P6pUqZLsF7WWTetLu2p0Rty4ceNMV48GX88xN67QdrXtSaV1Hb7bk9aL0vfbd7nn2B59L3R9Ov4otfciJboOrT9/3s+UXm9KtBtMx99pHejnR8fq6fp0uSft6ho7dqz5UeA5jiy9P4BS+qy5Qrov7SJG+CAAIaB0x6stB6l90ejtuqPTHYp+CQRKSjtC1692X56/KFO7zdXK8uSTT5oWn+Tor3dPvr/4XXwHqF6N63l1XIf+Uk6OayyOtk7pL+IFCxaYVh19jI5j0MMS+DP2JZC03DrmZsKECcne7vrS1mCjLVnaUqT31eUaSnUsi44X8hxg669A1HtqdKyMBhcdFKwtRq4DQ+qYleTKnNy2ltZ1pPS6klvu+Vp1XRp+dGB3cvxpqdF1aAuSjhFLjgZEfz9bvjTo6FgibWXSutBxdVoXOu7KRce66fgfHUOoY7J0P6PjlrT+khsEn579gqvOdRyQ53gkFx2fhvDBu4mAu/vuu+Xf//636TrRgYO+dEemgz11IKVr56tB6Pvvv7/qenU2SWr3cbWwaDO2J1eX09/hakXRna42+weCvialr8s3PPneR+vIn+fVLwYdcKsX/RWtg5/1oIUpBSCtfx1su3v37iS36Yw3HXDt28LgDy33d999Z8LN1X6h64Bn/TWvg9k9Wzh8uxv8qatAcbUEeNKB21pPrrCgXTTa2vbqq6+676ODfn23vasJxDr8oXW3bNkyM4A/tWCS0nul69CJCIHa9n1bi7SLUrvBdAC/BnbtktUuahftAteuT+2G9FyuASg1nvsFz0M4+O4XXNuYhsWMeJ0ILYwBQsDpFGndyWrA0TENnnQcjc5U0i8SvZ/SL1jd2ekX4aZNm1L8JavdX/qFqq0bKd3HtQPzHI+gv/J0tszfpTtFnZ2js6N01ouv5Ka3p0ZPKaHdgNoNol98yb0m7crS16VTgPULKKXn1dfp222iZdbDCvhOO/dtPdByLFq0yGs68NGjR80vaw2x6Wn619Yo7dLQMOxLW/1cM6FcrRe+3T++X2z+1FWgaFec57gS7QLU+tEyuMqrf32f980330yxtTE5gViHv++FrnPMmDHJHqPHM3DpzMbkApiuQ+tFA4gvvb+u5+/QViCd2TZt2jQz7sa3+0vrSsOZZ93o9urPqViS2y/o9vfee+953U9bdnVb19l0OiYrEJ9xhC5agBBwOn5Cdyw6VVW7QHyPBK07t9mzZ7t3Skp3ONplowNfXVOmNWRok7i2JOmvNg1M+otZp73qVGgNBhqotOXg7bffNgOkdZxK/fr1TXO63qZTb3Wq7N/dObtMmjTJBAJ9XTqoVVuFNCjoF4NOxdeAlha6s9Vunocfftgcz0YHHeuvVV2PjjfRetSA+O6775oWHH19Omhbuw81XGgria5Dw6OOq9KxL3rcFK0L7RbTX/061dqzhSE5Oq5CB4Dqa9OWI23q16Cnwcnf4+f40oNdaleGBl4tp7Y+6JeXtirpcv0i1cG6Giq0y0sHvGpo1pCnoUnDm2fQ9KeuAkXHGemXoec0eKXTzz1bOrWrRLtq9JADug1ofesxavwViHX4Qz9XWrcaHnX8jNa5tmRqS5d+xnR6uet4O/q50mnfuk1oS5u+DzomRj9/+lnTMutRovV+GiL00Ab6udTPd3oOReEZsLR7WS/6ufVtgdFp7tpFqt1i+t5r66Z+HrWMqXW56+vV1kXdF+nr0DClQUtb8w4ePOi1jelr121XW061K9J1Hz1sh27DeugAhIlgT0ND+Nq+fbvTtWtXp0SJEk727NnNNGS9vmPHjmTvr1ONdTp80aJFncjISDMFWqd06xRmlxMnTjj9+/d3SpUqZaZt63Rfne56/Phx9310KrdOP9d1FC9e3HnqqaecpUuXJjsNXqf8+kptqreuX8upr0dfl5bl7rvvdj7++OMk02t9p/UnNx1fffLJJ2Yqfa5cucz04VtvvdWZPXu21322bt3qdOzY0SlcuLB5bTrV97777nOWL19ubtd6GjJkiFOnTh0zXVynLOv/J0+e7Phjy5YtZpq1Tj3OnTu3c8cddzjr169PU9340mnKL774oqlnLXPBggWdunXrOqNGjXJOnTrl9fpr165tppqXL1/ePGbatGnJHtIgtbpK6X3V7UTrLDX6nLrd/ec//zGHH9By33jjjUnes5MnTzoPPvigU6RIEVNnWne7du1KMgU7pW0hEOtwTe/+/fffk7xWff99/etf/zL1r3Wn24ge0mHo0KHmkAeeU77btGljbtd1e06J12nisbGxTuXKlc3nT8ut78Urr7xi3uv0bCOeGjVqZB778MMPJ3v71KlT3e9J1apVTb0kN8Xdt/7U5s2bzSEUtNxly5Z1JkyYcNXDZuh7oVPfdZusVKmS07NnT69DIyDzi9B/gh3CACBUaDeLHnyRX/pAeGMMEAAAsA4BCAAAWIcABAAArMMsMADwwLBIwA60AAEAAOsQgAAAgHXCvgtMz+2iZ8bWI8gG6oR5AAAg47uj9QCvejR7PSBsoIV9ANLwk57zGAEAgODTU9HoUe4DLewDkLb8uCowPeczAgAA115CQoJpwHB9jwda2AcgV7eXhh8CEAAAmUtGDV9hEDQAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOtmCXQAAAHBtlB/+WcDXeWB8G8mMaAECAADWIQABAADrEIAAAIB1QiYAjR8/XiIiImTgwIHuZRcuXJB+/fpJ4cKFJW/evNKpUyc5evRoUMsJAAAyv5AIQBs3bpR33nlHateu7bV80KBBsnjxYpk7d66sXr1afvvtN+nYsWPQygkAAMJD0APQmTNnpFu3bvLvf/9bChYs6F5+6tQpmTp1qkyYMEGaNWsmdevWlenTp8v69etlw4YNQS0zAADI3IIegLSLq02bNtKiRQuv5Zs3b5ZLly55La9ataqULVtW4uLiUlxfYmKiJCQkeF0AAABC5jhAc+bMkS1btpguMF9HjhyRHDlySIECBbyWFy9e3NyWknHjxsmoUaMypLwAACA8BK0F6NChQzJgwACZNWuW5MyZM2DrjY2NNd1nros+DwAAQEgEIO3iOnbsmNx0002SLVs2c9GBzm+88Yb5v7b0XLx4UeLj470ep7PAoqOjU1xvZGSk5M+f3+sCAAAQEl1gzZs3lx07dngte/DBB804n2HDhkmZMmUke/bssnz5cjP9Xe3evVsOHjwoDRo0CFKpAQBAOAhaAMqXL5/UrFnTa1mePHnMMX9cy3v16iWDBw+WQoUKmZacxx9/3ISf+vXrB6nUAAAgHIT0yVAnTpwoWbJkMS1AOrsrJiZGJk+eHOxiAQCATC7CcRxHwphOg4+KijIDohkPBACwWWY6G3xCBn9/B/04QAAAANcaAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYJ2gBqApU6ZI7dq1JX/+/ObSoEED+eKLL9y3N23aVCIiIrwuffr0CWaRAQBAGMgWzCcvXbq0jB8/XqpUqSKO48h7770n7dq1k61bt0qNGjXMfXr37i2jR492PyZ37txBLDEAAAgHQQ1Abdu29br+/PPPm1ahDRs2uAOQBp7o6OgglRAAAISjkBkDdPnyZZkzZ46cPXvWdIW5zJo1S4oUKSI1a9aU2NhYOXfu3FXXk5iYKAkJCV4XAACAkGkBUjt27DCB58KFC5I3b15ZsGCBVK9e3dx2//33S7ly5aRkyZKyfft2GTZsmOzevVvmz5+f4vrGjRsno0aNuoavAAAAZDYRjg6+CaKLFy/KwYMH5dSpU/Lxxx/Lu+++K6tXr3aHIE8rVqyQ5s2by969e6VSpUoptgDpxUVbgMqUKWPWrwOtAQCwVfnhnwV8nQfGt5GMoN/fUVFRGfb9HfQWoBw5ckjlypXN/+vWrSsbN26U119/Xd55550k961Xr575e7UAFBkZaS4AAAAhPwbI5cqVK14tOJ62bdtm/pYoUeIalwoAAISToLYA6aDmVq1aSdmyZeX06dPywQcfyKpVq+TLL7+Uffv2meutW7eWwoULmzFAgwYNkiZNmphjBwEAAGTKAHTs2DHp3r27HD582PTzabDR8NOyZUs5dOiQLFu2TF577TUzM0zH8XTq1EmeeeaZYBYZAACEgaAGoKlTp6Z4mwYeHQwNAAAQ9mOAAAAAMhoBCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgnaAGoClTpkjt2rUlf/785tKgQQP54osv3LdfuHBB+vXrJ4ULF5a8efNKp06d5OjRo8EsMgAACANBDUClS5eW8ePHy+bNm2XTpk3SrFkzadeunfzwww/m9kGDBsnixYtl7ty5snr1avntt9+kY8eOwSwyAAAIAxGO4zjBLoSnQoUKycsvvyz33nuvFC1aVD744APzf7Vr1y6pVq2axMXFSf369f1aX0JCgkRFRcmpU6dMKxMAALYqP/yzgK/zwPg2khEy+vs7ZMYAXb58WebMmSNnz541XWHaKnTp0iVp0aKF+z5Vq1aVsmXLmgCUksTERFNpnhcAAICQCkA7duww43siIyOlT58+smDBAqlevbocOXJEcuTIIQUKFPC6f/Hixc1tKRk3bpxJjK5LmTJlrsGrAAAAmUnQA9D1118v27Ztk2+++Ub69u0rPXr0kJ07d6Z7fbGxsaa5zHU5dOhQQMsLAAAyv2zBLoC28lSuXNn8v27durJx40Z5/fXXpXPnznLx4kWJj4/3agXSWWDR0dEprk9bkvQCAAAQsi1Avq5cuWLG8WgYyp49uyxfvtx92+7du+XgwYNmjBAAAECmbAHS7qpWrVqZgc2nT582M75WrVolX375pRm/06tXLxk8eLCZGaYjwB9//HETfvydAQYAABByAejYsWPSvXt3OXz4sAk8elBEDT8tW7Y0t0+cOFGyZMliDoCorUIxMTEyefLkYBYZAACEgZA7DlCgcRwgAAD+h+MAhfAYIAAAgIxGAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKyTrgC0ZcsW2bFjh/v6okWLpH379vLUU0/JxYsXA1k+AACA0AhAjz76qPz000/m/z///LN06dJFcufOLXPnzpWhQ4cGuowAAADBD0Aafm644Qbzfw09TZo0kQ8++EBmzJgh8+bNC2wJAQAAQiEAOY4jV65cMf9ftmyZtG7d2vy/TJkycvz48cCWEAAAIBQC0M033yxjx46VmTNnyurVq6VNmzZm+f79+6V48eKBLiMAAEDwA9Brr71mBkL3799fnn76aalcubJZ/vHHH0vDhg0DW0IAAIAAy5aeB9WuXdtrFpjLyy+/LFmzZg1EuQAAAELvOEDx8fHy7rvvSmxsrPzxxx9m2c6dO+XYsWN+r2PcuHFyyy23SL58+aRYsWJmKv3u3bu97tO0aVOJiIjwuvTp0ye9xQYAAEhfC9D27dulefPmUqBAATlw4ID07t1bChUqJPPnz5eDBw/K+++/79d6dPxQv379TAj6888/zXGE7rzzThOk8uTJ476frn/06NHu6zrlHgAA4JoGoMGDB8uDDz4oL730kmm9cdHZYPfff7/f61myZInXdZ1Gry1BmzdvNlPrPQNPdHR0eooKAAAQmC6wjRs3moMh+ipVqpQcOXJE0uvUqVPmr7YmeZo1a5YUKVJEatasabrczp07l+I6EhMTJSEhwesCAADwt1uAIiMjkw0WeoDEokWLpmeV5rhCAwcOlEaNGpmg46ItSuXKlZOSJUuarrdhw4aZcULa3ZbSuKJRo0alqwwAAMAOEY4e1TCNHn74YTlx4oR89NFHprVGg4nO/tJBzNp1pdPk06pv377yxRdfyNdffy2lS5dO8X4rVqww44/27t0rlSpVSrYFSC8uGtT0AI3aupQ/f/40lwsAgHBRfvhnAV/ngfH/OxZgoOn3d1RUVIZ9f6erC+zVV1+VM2fOmPE658+fl9tvv90cC0jHAz3//PNpXp8eT+jTTz+VlStXXjX8qHr16pm/GoBSap3SivK8AAAA/O0uME1kS5culXXr1sl3331nwtBNN90kLVq0SNN6tPHp8ccflwULFsiqVaukQoUKqT5m27Zt5m+JEiXSU3QAAID0BSAXHa+jl/TSKfB6EtVFixaZ1iPXAGoNWLly5ZJ9+/aZ23V2WeHChU1X26BBg0w3mx6MEQAA4Jp1gT3xxBPyxhtvJFn+1ltvmYHM/poyZYrp29ODHWqLjuvy4Ycfmttz5MhhTraqxwaqWrWq/POf/5ROnTrJ4sWL01NsAACA9LcAzZs3Tz755JMky/U8YOPHj/d7EHRq46918LIeLBEAACDoLUA6A0y7qXzpgOPjx48HolwAAAChFYB0xpfvUZyVTmOvWLFiIMoFAAAQeqfC0Knrv//+uzRr1swsW758uZken55jAAEAAIR8AHrooYfMwQb1mD9jxowxy8qXL28GNXfv3j3QZQQAAAiNafB65Ga9aCuQTlnPmzdvYEsGAAAQiscBUuk99xcAAECmGgR99OhReeCBB8wJSrNly2bOA+Z5AQAACLsWoJ49e8rBgwdlxIgR5sCFERERgS8ZAABAKAUgPWP72rVr5YYbbgh8iQAAAEKxC0yP0JzaUZwBAADCKgDpsX6GDx8uBw4cCHyJAAAAQrELrHPnznLu3DmpVKmS5M6dW7Jnz+51+x9//BGo8gEAAIRGAOJozwAAwLoA1KNHj8CXBAAAIJTHAKl9+/bJM888I127dpVjx465T4b6ww8/BLJ8AAAAoRGAVq9eLbVq1ZJvvvlG5s+fL2fOnDHLv/vuOxk5cmSgywgAABD8AKQzwMaOHStLly6VHDlyuJfrmeE3bNgQyPIBAACERgDasWOHdOjQIcnyYsWKyfHjxwNRLgAAgNAKQAUKFJDDhw8nWb5161YpVapUIMoFAAAQWgGoS5cuMmzYMDly5Ig5D9iVK1dk3bp18uSTT0r37t0DX0oAAIBgB6AXXnhBqlatak6JoQOgq1evLk2aNJGGDRuamWEAAABhdRwgPQeYtvy88cYb8uyzz5rxQBqCbrzxRqlSpUrGlBIAACDYAahy5crmeD8aeLQVCAAAIKy7wLJkyWKCz4kTJzKmRAAAAKE4Bmj8+PEyZMgQ+f777wNfIgAAgFA8F5jO9NKzwdepU8ccCDFXrlxet3M2eAAAEMo4GzwAALBOmgPQpUuXzLnARowYIRUqVMiYUgEAAITSGKDs2bPLvHnzMqY0AAAAoToIun379rJw4cLAlwYAACBUxwDpNPjRo0eb01/UrVtX8uTJ43X7E088EajyAQAAhEYAmjp1qjkh6ubNm83Fk54bjAAEAADCrgts//79KV5+/vlnv9czbtw4ueWWWyRfvnxSrFgx07W2e/dur/tcuHBB+vXrJ4ULF5a8efNKp06d5OjRo+kpNgAAQPoDUKDobDINNxs2bJClS5eaGWZ33nmnnD171n2fQYMGyeLFi2Xu3Lnm/r/99pt07NgxmMUGAAA2doE99NBDV7192rRpfq1nyZIlXtdnzJhhWoK0W03PLn/q1CnT3fbBBx9Is2bNzH2mT58u1apVM6Gpfv366Sk+AACwXLoC0MmTJ72ua8uNnhYjPj7eHVTSQwOPKlSokPmrQUjX3aJFC/d9qlatKmXLlpW4uLhkA1BiYqK5uCQkJKS7PAAAIDylKwAtWLAgybIrV65I3759pVKlSukqiD5+4MCB0qhRI6lZs6ZZduTIEXOqDR1w7al48eLmtpTGFY0aNSpdZQAAAHYI2BggPUv84MGDZeLEiel6vI4F0lakOXPm/K1yxMbGmpYk1+XQoUN/a30AACD8pKsFKCX79u2TP//8M82P69+/v3z66aeyZs0aKV26tHt5dHS0XLx40XStebYC6SwwvS05kZGR5gIAABDQAKQtPZ4cx5HDhw/LZ599Jj169PB7Pfq4xx9/3HSprVq1Ksm5xfQgi3rqjeXLl5vp70qnyR88eFAaNGiQnqIDAACkLwBt3bo1SfdX0aJF5dVXX011hphvt5fO8Fq0aJE5FpBrXE9UVJTkypXL/O3Vq5cJXDowOn/+/CYwafhhBhgAALimAWjlypUSCFOmTDF/mzZt6rVcp7r37NnT/F/HFGnA0hYgnd0VExMjkydPDsjzAwAAO6UrAOkRn3Wsj54TzNOePXtMl1X58uX97gJLTc6cOWXSpEnmAgAAELRZYNo6s379+iTLv/nmG3fLDQAAQFgFIB0DpMfr8aXjcrZt2xaIcgEAAIRWANIzvp8+fTrJcj3uzuXLlwNRLgAAgNAKQHqeLj3ismfY0f/rssaNGweyfAAAAKExCPrFF180Iej666+X2267zSxbu3atOe/WihUrAl1GAACA4LcAVa9eXbZv3y733XefHDt2zHSHde/eXXbt2uU+jxcAAEDYnQqjZMmS8sILLwS2NAAAAKHaAqQHKpw7d26S5brsvffeC0S5AAAAQisA6WDnIkWKJFlerFgxWoUAAEB4BiA9GanviUtVuXLlzG0AAABhF4C0pUcHQfv67rvvpHDhwoEoFwAAQGgFoK5du8oTTzxhToqqx//Ri05/HzBggHTp0iXwpQQAAAj2LLAxY8bIgQMHpHnz5pIt2/9WoSGoR48ejAECAADhGYBy5MghH374oTz55JMmCOXKlUtq1aplxgABAACEXQCKj4+Xp59+2gSgkydPmmUFCxY0XV9jx46VAgUKZEQ5AQAAghOA/vjjD2nQoIH8+uuv0q1bN6lWrZpZvnPnTpkxY4YsX75c1q9fbwIRAABAWASg0aNHm+6vffv2SfHixZPcduedd5q/EydODHQ5AQAAgjMLbOHChfLKK68kCT8qOjpaXnrpJVmwYEHgSgcAABDsAHT48GGpUaNGirfriVCPHDkSiHIBAACERgDS01/orK+U7N+/XwoVKhSIcgEAAIRGAIqJiTEzwC5evJjktsTERBkxYoTcddddgSwfAABA8AdB33zzzVKlShXp16+fVK1aVRzHkR9//FEmT55sQtDMmTMDX0oAAIBgBaDSpUtLXFycPPbYYxIbG2vCj4qIiJCWLVvKW2+9JWXKlAlk+QAAAIJ/IEQ9C/wXX3xhDoK4Z88es6xy5cqM/QEAAOF9KgylBzu89dZbA1saAACAUD0bPAAAQGZGAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsE5QA9CaNWukbdu2UrJkSXM06YULF3rd3rNnT7Pc88K5xgAAwN8V1AB09uxZqVOnjkyaNCnF+2jgOXz4sPsye/bsa1pGAAAQftJ9JOhAaNWqlblcTWRkpERHR1+zMgEAgPAX8mOAVq1aJcWKFZPrr79e+vbtKydOnLjq/fWM9AkJCV4XAACATBOAtPvr/fffl+XLl8uLL74oq1evNi1Gly9fTvEx48aNk6ioKPeFs9MDAICQ6gJLTZcuXdz/r1WrltSuXVsqVapkWoWaN2+e7GNiY2Nl8ODB7uvaAkQIAgAAmaYFyFfFihWlSJEisnfv3quOGcqfP7/XBQAAINMGoP/+979mDFCJEiWCXRQAAJCJBbUL7MyZM16tOfv375dt27ZJoUKFzGXUqFHSqVMnMwts3759MnToUKlcubLExMQEs9gAACCTC2oA2rRpk9xxxx3u666xOz169JApU6bI9u3b5b333pP4+HhzsMQ777xTxowZY7q5AAAAMmUAatq0qTiOk+LtX3755TUtDwAAsEOmGgMEAAAQCAQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1ghqA1qxZI23btpWSJUtKRESELFy40Ot2x3Hk2WeflRIlSkiuXLmkRYsWsmfPnqCVFwAAhIegBqCzZ89KnTp1ZNKkScne/tJLL8kbb7whb7/9tnzzzTeSJ08eiYmJkQsXLlzzsgIAgPCRLZhP3qpVK3NJjrb+vPbaa/LMM89Iu3btzLL3339fihcvblqKunTpco1LCwAAwkXIjgHav3+/HDlyxHR7uURFRUm9evUkLi4uxcclJiZKQkKC1wUAACBkWoCuRsOP0hYfT3rddVtyxo0bJ6NGjcrw8gEAwlf54Z9lyHoPjG8T1OdCJmgBSq/Y2Fg5deqU+3Lo0KFgFwkAAISYkA1A0dHR5u/Ro0e9lut1123JiYyMlPz583tdAAAAMkUAqlChggk6y5cvdy/T8Tw6G6xBgwZBLRsAAMjcgjoG6MyZM7J3716vgc/btm2TQoUKSdmyZWXgwIEyduxYqVKliglEI0aMMMcMat++fTCLDQAAMrmgBqBNmzbJHXfc4b4+ePBg87dHjx4yY8YMGTp0qDlW0COPPCLx8fHSuHFjWbJkieTMmTOIpQYAAJldUANQ06ZNzfF+UqJHhx49erS5AAAAhP0YIAAAgIxCAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADrEIAAAIB1CEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWCekA9Bzzz0nERERXpeqVasGu1gAACCTyyYhrkaNGrJs2TL39WzZQr7IAAAgxIV8mtDAEx0dHexiAACAMBLSXWBqz549UrJkSalYsaJ069ZNDh48eNX7JyYmSkJCgtcFAAAg0wSgevXqyYwZM2TJkiUyZcoU2b9/v9x2221y+vTpFB8zbtw4iYqKcl/KlClzTcsMAABCX0gHoFatWsk//vEPqV27tsTExMjnn38u8fHx8tFHH6X4mNjYWDl16pT7cujQoWtaZgAAEPpCfgyQpwIFCsh1110ne/fuTfE+kZGR5gIAAJApW4B8nTlzRvbt2yclSpQIdlEAAEAmFtIB6Mknn5TVq1fLgQMHZP369dKhQwfJmjWrdO3aNdhFAwAAmVhId4H997//NWHnxIkTUrRoUWncuLFs2LDB/B8AACAsA9CcOXOCXQQAABCGQroLDAAAICMQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdUL6VBgAgNBXfvhnAV/ngfFtrsnzpPRcCH+0AAEAAOsQgAAAgHUIQAAAwDoEIAAAYB0CEAAAsA4BCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWyRbsAmRm5Yd/liHrPTC+Dc8V4OcJ1+fK7O9VuD5XsLeLa/1cQGZECxAAALAOAQgAAFiHAAQAAKyTKQLQpEmTpHz58pIzZ06pV6+efPvtt8EuEgAAyMRCPgB9+OGHMnjwYBk5cqRs2bJF6tSpIzExMXLs2LFgFw0AAGRSIR+AJkyYIL1795YHH3xQqlevLm+//bbkzp1bpk2bFuyiAQCATCqkA9DFixdl8+bN0qJFC/eyLFmymOtxcXFBLRsAAMi8Qvo4QMePH5fLly9L8eLFvZbr9V27diX7mMTERHNxOXXqlPmbkJAQ8PJdSTwnGSG5svJcf+95wvW5Mvt7Fa7PFeztIlyfK7NvF+H6XAkZ8P3quV7HcTJk/brikPXrr7/qq3bWr1/vtXzIkCHOrbfemuxjRo4caR7DhQsXLly4cMn8l0OHDmVIxgjpFqAiRYpI1qxZ5ejRo17L9Xp0dHSyj4mNjTWDpl2uXLkif/zxhxQuXFgiIiICkkjLlCkjhw4dkvz584vNqIu/UBd/oS7+Ql38hbr4C3XhX11oy8/p06elZMmSkhFCOgDlyJFD6tatK8uXL5f27du7A41e79+/f7KPiYyMNBdPBQoUCHjZ9I2yfcN1oS7+Ql38hbr4C3XxF+riL9RF6nURFRUlGSWkA5DS1pwePXrIzTffLLfeequ89tprcvbsWTMrDAAAICwDUOfOneX333+XZ599Vo4cOSI33HCDLFmyJMnAaAAAgLAJQEq7u1Lq8rrWtHtND8ro281mI+riL9TFX6iLv1AXf6Eu/kJdhEZdROhI6Gv+rAAAAEEU0gdCBAAAyAgEIAAAYB0CEAAAsA4BCAAAWMe6ADRp0iQpX7685MyZU+rVqyfffvvtVe+vxx26/vrrJVeuXOZolYMGDZILFy64bx83bpzccsstki9fPilWrJg5YOPu3bu91tG0aVNzFGrPS58+fSTc6uK5555L8jqrVq3qtQ69f79+/cyRufPmzSudOnVKcqTvcKgLXZdvXehFX3s4bReXLl2S0aNHS6VKlcz969SpYw5TkdZ1hsN24U9d2LK/8KcubNlf+FMXmXV/sWbNGmnbtq05UrOWZ+HChak+ZtWqVXLTTTeZWV+VK1eWGTNmBG9/4Vhkzpw5To4cOZxp06Y5P/zwg9O7d2+nQIECztGjR5O9/6xZs5zIyEjzd//+/c6XX37plChRwhk0aJD7PjExMc706dOd77//3tm2bZvTunVrp2zZss6ZM2fc97n99tvNcx0+fNh9OXXqlBNudaHnYatRo4bX6/z999+91tOnTx+nTJkyzvLly51NmzY59evXdxo2bOiEW10cO3bMqx6WLl1qzmmzcuXKsNouhg4d6pQsWdL57LPPnH379jmTJ092cubM6WzZsiVN6wyH7cKfurBlf+FPXdiyv/CnLjLr/uLzzz93nn76aWf+/PmmvAsWLLjq/X/++Wcnd+7czuDBg52dO3c6b775ppM1a1ZnyZIlQdlfWBWA9ASq/fr1c1+/fPmy2TDHjRuX7P31vs2aNfNapm9co0aNUnwO3ZB1Q1i9erXXhjtgwAAn3OtCd2h16tRJ8Tnj4+Od7NmzO3PnznUv+/HHH019xcXFOeG8Xej7X6lSJefKlSthtV1o8Hvrrbe8lnXs2NHp1q2b3+sMl+3Cn7qwZX/hT13Ysr9Iz3aRWfYXnvwJQBoGNfR66ty5s/lhEIz9hTVdYBcvXpTNmzdLixYt3MuyZMlirsfFxSX7mIYNG5rHuJrffv75Z/n888+ldevWKT7PqVOnzN9ChQp5LZ81a5Y5uWvNmjXNCVvPnTsn4VgXe/bsMc2hFStWlG7dusnBgwfdt+njtTnY83m1ybts2bIpPm84bBf6HP/5z3/koYceSnJC3sy+XSQmJppmak/aLfj111/7vc5w2S5Sqwub9hf+1oUN+4u0bheZZX+RHlpHnnWnYmJi3HV3rfcXmeJI0IFw/PhxuXz5cpJTaOj1Xbt2JfuY+++/3zyucePG5qy0f/75p+lzfeqpp5K9v56odeDAgdKoUSOzgXqup1y5cuaDvn37dhk2bJjp958/f76EU11oX6325+rYmMOHD8uoUaPktttuk++//96MedBTmegJbn1PTqvPq7eF63ah/eLx8fHSs2fPJOvJ7NuF7rwmTJggTZo0MWMc9ETFWn5dj7/rDJftIrW6sGl/4U9d2LK/SOt2kVn2F+mh71tydadnhD9//rycPHnymu4vrAlA6aGDtV544QWZPHmy+bDu3btXBgwYIGPGjJERI0Ykub8OytIPr2+yf+SRR9z/r1WrlpQoUUKaN28u+/btMx+IcKmLVq1aue9fu3Ztcz/9wH700UfSq1cvsXW7mDp1qqkb3XGF23bx+uuvS+/evc0vMP21quXWExVPmzZNbJPWugjn/YU/dWHL/iKt20U47y9CjTVdYNpsmDVr1iQjxfV6dHR0so/RL7MHHnhAHn74YbPBdejQwXzx6UwO/fXmSc9V9umnn8rKlSuldOnSVy2LftCVfnGGY124aEK/7rrr3K9T161NnPrrxt/nzex18csvv8iyZcvMfVOTGbeLokWLml+sZ8+eNa9Vf6XprAzt0vB3neGyXaRWFzbtL9JSF+G+v0hLXWSm/UV6aB0lV3f58+c33YLXen9hTQDSJrO6deua5kcX/bLS6w0aNEj2Mdq/qv2PnvTNUa5TqOlf3ZktWLBAVqxYIRUqVEi1LNu2bTN/NcGHU134OnPmjPl14nqd+pzZs2f3el5twtV+/5SeN7PXxfTp08105zZt2oTlduGiYxxKlSplugPnzZsn7dq183ud4bJdpFYXNu0v/KkLW/YXaamLzLS/SA+tI8+6U0uXLnXX3TXfXzgW0el1On15xowZZgreI488YqbXHTlyxNz+wAMPOMOHD/eapZAvXz5n9uzZZvreV199ZUbm33fffe779O3b14mKinJWrVrlNT3x3Llz5va9e/c6o0ePNlP1dMr0okWLnIoVKzpNmjRxwq0u/vnPf5p60Ne5bt06p0WLFk6RIkXMTBfP6Ys67XfFihWmTho0aGAu4VYXrtkL+lqHDRuW5DnDZbvYsGGDM2/ePDO9d82aNWZ2XIUKFZyTJ0/6vc5w2S78qQtb9hf+1IUt+wt/6iKz7i9Onz7tbN261Vw0TkyYMMH8/5dffjG3az1offhOgx8yZIiZuTVp0qRkp8Ffq/2FVQFI6XEHtOL0OAM63U43Ts9phj169HBfv3TpkvPcc8+ZLzc9boMed+Cxxx7z2nD1TU/uosf6UAcPHjQbaaFChcybWrlyZfPmB/v4DRlRFzqdUad86vpKlSplrusH19P58+fN4woWLGg+CB06dDBfAOFWF0qPD6Tbwu7du5M8X7hsF/oFVq1aNfMaChcubHZ2v/76a5rWGS7bhT91Ycv+wp+6sGV/4e9nJDPuL1auXJns9ux6/fpX68P3MTfccIOpOw1xrm0/GPuLCP0nXW1ZAAAAmZQ1Y4AAAABcCEAAAMA6BCAAAGAdAhAAALAOAQgAAFiHAAQAAKxDAAIAANYhAAFAgOlJL/X8TwBCFwEICGM9e/Y0X8Z60fPsVK5cWUaPHm3OR5RZES4ABEK2gKwFQMi66667zEkWExMT5fPPP5d+/fqZkwnGxsameV2XL182AcT3ZLCZ0aVLl0w9ALBT5t+LAbiqyMhIiY6OlnLlyknfvn2lRYsW8sknn5jbJkyYILVq1ZI8efJImTJl5LHHHjNn5XaZMWOGFChQwNy/evXqZl161uWNGzdKy5YtpUiRIhIVFSW33367bNmyxet5NSi98847cvfdd0vu3LmlWrVqEhcXJ3v37pWmTZua52zYsKE5A7inRYsWyU033WTOnl2xYkUZNWqUu8WqfPny5m+HDh3M+l3XU3ucqzxTpkyRe+65xzz3888/n6SunnrqKalXr16S5XXq1DEtZ8qf1+5p1apV5rnj4+O9zuStyw4cOOBe9vXXX8ttt90muXLlMu/FE088IWfPnk1xvQD+HgIQYBn9gr148aL5v7bkvPHGG/LDDz/Ie++9JytWrJChQ4d63f/cuXPy4osvyrvvvmvuV6xYMTl9+rT06NHDfGlv2LBBqlSpIq1btzbLPY0ZM0a6d+9uvvCrVq0q999/vzz66KOm9WnTpk16Mmbp37+/+/5r16419x8wYIDs3LnTBCgNYa6wouFDaYvW4cOH3ddTe5zLc889Z8LTjh075KGHHkpSN926dZNvv/3WK5Tpa96+fbspu/L3taeFPp+21HXq1Mk814cffmjW71k3AAIszadPBZBp6NmY27VrZ/5/5coVZ+nSpeZs0k8++WSy9587d645Y7WLnqlZdxPbtm276vNcvnzZyZcvn7N48WL3Mn3cM888474eFxdnlk2dOtW9bPbs2U7OnDnd15s3b+688MILXuueOXOmOWu453oXLFjgdR9/Hzdw4EAnNXXq1HFGjx7tvh4bG+vUq1cvza/dVUbXGbNPnjzpvn3r1q1m2f79+831Xr16OY888ojXeteuXetkyZLFnPkaQOAxBggIc59++qnkzZvXjHm5cuWKacnQlhC1bNkyGTdunOzatUsSEhJMl9GFCxdMq492WykdPF27dm2vdR49elSeeeYZ071z7NgxMzZIH6PdY548H1e8eHHzV7vcPJfp8+lz58+fX7777jtZt26dV8uNrtu3TL78fdzNN9+can1pK9C0adNkxIgRpoVq9uzZMnjw4DS/9rTQ8mvLz6xZs9zL9Ln1/dq/f7/pPgQQWAQgIMzdcccdZuyLBpmSJUtKtmz/+9jr+BMdn6PjgjQ4FCpUyHS79OrVy3SRuUKDdpnpeBVP2gV04sQJef31183YIh0b1KBBA3fXmovnIGPXOpJbpl/0Sscf6didjh07JnkdOrYnJf4+Tsf+pKZr164ybNgwM67n/PnzcujQIencuXOaX7uLa8D4/xqG/kfDqG/5tWtQx/34Klu2bKplBpB2BCAgzOmXvk5/97V582YTPF599VX3l/RHH33k1zq1tWXy5Mlm7IvSkHD8+PG/XVYdxLx79+5ky+uiAUpbXdL6OH+VLl3aDGzW1hgNQDrgWcc9pfe1Fy1a1PzVMUsFCxY0/9cxUb7l17FLgSg/AP8QgABL6ZettkS8+eab0rZtW/PF/vbbb/v1WB34O3PmTNOlpN1XQ4YMMS1Ff9ezzz5rWqW01ePee+81wUy7h77//nsZO3asuY/O/Fq+fLk0atTItL5oqPDncWmh3WAjR440rToTJ078W69d61lndWm3o7a0/fTTTyZ0etIWp/r165tBzw8//LAJrRqIli5dKm+99Vaayw8gdcwCAyylU7t1GrzO8KpZs6Zp8dDxQP6YOnWqnDx50rRcPPDAA6brxrOVJL1iYmLMmKWvvvpKbrnlFhMKNIBoV5OLhgcNBhoqbrzxRr8flxYaorSbS8f2tG/f/m+9dm2x0nFEOs5Kx0RpffuGMl2+evVqE450Kry+Lg112mUJIGNE6EjoDFo3AABASKIFCAAAWIcABAAArEMAAgAA1iEAAQAA6xCAAACAdQhAAADAOgQgAABgHQIQAACwDgEIAABYhwAEAACsQwACAADWIQABAADr/D8Hz8gRf4RhTgAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ "clf = RandomClassifier()\n", "\n", "if metric == \"precision\":\n", - " theoretical_value = p\n", + " if target_level <= p:\n", + " actual_valid_parameters = predict_params\n", + " else:\n", + " actual_valid_parameters = []\n", "elif metric == \"recall\":\n", - " theoretical_value = 1 - clf.threshold\n", - "\n", - "all_valid_parameters = []\n", + " actual_valid_parameters = predict_params[predict_params <= 1-target_level]\n", "\n", "for _ in range(n_repeats):\n", "\n", @@ -133,7 +127,7 @@ "\n", " controller = BinaryClassificationController(\n", " fitted_binary_classifier=clf,\n", - " metric=\"precision\",\n", + " metric=metric,\n", " target_level=target_level,\n", " confidence_level=confidence_level,\n", " )\n", From 10768c5c59a4eb66061d6d519a9ad9a21d201855 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Mon, 28 Jul 2025 18:42:18 +0200 Subject: [PATCH 08/18] ENH: theoretical tests notebook --- ...risk_control_theoretical_tests_proto.ipynb | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 3b7210699..3d4798513 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -85,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 4, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -96,6 +96,14 @@ "outputId": "b15146cf-518e-4a93-8128-6c1865a08b01" }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/faustinpulveric/.pyenv/versions/3.10.17/envs/mapie_dev/lib/python3.10/site-packages/mapie/risk_control_draft.py:117: UserWarning: No valid thresholds found\n", + " warnings.warn(\"No valid thresholds found\", UserWarning)\n" + ] + }, { "ename": "NameError", "evalue": "name 'all_valid_parameters' is not defined", @@ -103,7 +111,7 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[11], line 26\u001b[0m\n\u001b[1;32m 23\u001b[0m controller\u001b[38;5;241m.\u001b[39mcalibrate(X_calibrate, y_calibrate)\n\u001b[1;32m 25\u001b[0m valid_parameters \u001b[38;5;241m=\u001b[39m controller\u001b[38;5;241m.\u001b[39mvalid_thresholds\n\u001b[0;32m---> 26\u001b[0m \u001b[43mall_valid_parameters\u001b[49m\u001b[38;5;241m.\u001b[39mappend(valid_parameters)\n\u001b[1;32m 28\u001b[0m all_valid_parameters \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate([x \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters \u001b[38;5;28;01mif\u001b[39;00m x\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m]) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28many\u001b[39m(x\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters) \u001b[38;5;28;01melse\u001b[39;00m np\u001b[38;5;241m.\u001b[39marray([])\n\u001b[1;32m 30\u001b[0m nb_actual_valid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msum\u001b[39m(\u001b[38;5;241m1\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters \u001b[38;5;28;01mif\u001b[39;00m theoretical_value \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m target_level)\n", + "Cell \u001b[0;32mIn[4], line 26\u001b[0m\n\u001b[1;32m 23\u001b[0m controller\u001b[38;5;241m.\u001b[39mcalibrate(X_calibrate, y_calibrate)\n\u001b[1;32m 25\u001b[0m valid_parameters \u001b[38;5;241m=\u001b[39m controller\u001b[38;5;241m.\u001b[39mvalid_thresholds\n\u001b[0;32m---> 26\u001b[0m \u001b[43mall_valid_parameters\u001b[49m\u001b[38;5;241m.\u001b[39mappend(valid_parameters)\n\u001b[1;32m 28\u001b[0m all_valid_parameters \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate([x \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters \u001b[38;5;28;01mif\u001b[39;00m x\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m]) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28many\u001b[39m(x\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters) \u001b[38;5;28;01melse\u001b[39;00m np\u001b[38;5;241m.\u001b[39marray([])\n\u001b[1;32m 30\u001b[0m nb_actual_valid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msum\u001b[39m(\u001b[38;5;241m1\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters \u001b[38;5;28;01mif\u001b[39;00m theoretical_value \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m target_level)\n", "\u001b[0;31mNameError\u001b[0m: name 'all_valid_parameters' is not defined" ] } @@ -117,7 +125,7 @@ " else:\n", " actual_valid_parameters = []\n", "elif metric == \"recall\":\n", - " actual_valid_parameters = predict_params[predict_params <= 1-target_level]\n", + " actual_valid_parameters = predict_params[predict_params <= np.round(1-target_level, 2)]\n", "\n", "for _ in range(n_repeats):\n", "\n", From 37ddfe35d429d2c35e3a53138703e34ad6743e30 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Tue, 29 Jul 2025 12:05:17 +0200 Subject: [PATCH 09/18] ENH: theoretical tests notebook --- ...risk_control_theoretical_tests_proto.ipynb | 61 +++++++++---------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 3d4798513..26b49997d 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 116, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 117, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -66,26 +66,26 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 126, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" }, "outputs": [], "source": [ - "N = 100 # size of the calibration set\n", + "N = 200 # size of the calibration set\n", "p = 0.5 # proportion of positives in the calibration set\n", "metric = \"precision\"\n", - "target_level = 0.8\n", + "target_level = 0.7\n", "predict_params = np.linspace(0, 0.99, 100)\n", - "confidence_level = 0.7\n", + "confidence_level = 0.5\n", "\n", "n_repeats = 10000" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 127, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -97,27 +97,21 @@ }, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "/Users/faustinpulveric/.pyenv/versions/3.10.17/envs/mapie_dev/lib/python3.10/site-packages/mapie/risk_control_draft.py:117: UserWarning: No valid thresholds found\n", - " warnings.warn(\"No valid thresholds found\", UserWarning)\n" - ] - }, - { - "ename": "NameError", - "evalue": "name 'all_valid_parameters' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[4], line 26\u001b[0m\n\u001b[1;32m 23\u001b[0m controller\u001b[38;5;241m.\u001b[39mcalibrate(X_calibrate, y_calibrate)\n\u001b[1;32m 25\u001b[0m valid_parameters \u001b[38;5;241m=\u001b[39m controller\u001b[38;5;241m.\u001b[39mvalid_thresholds\n\u001b[0;32m---> 26\u001b[0m \u001b[43mall_valid_parameters\u001b[49m\u001b[38;5;241m.\u001b[39mappend(valid_parameters)\n\u001b[1;32m 28\u001b[0m all_valid_parameters \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate([x \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters \u001b[38;5;28;01mif\u001b[39;00m x\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m]) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28many\u001b[39m(x\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters) \u001b[38;5;28;01melse\u001b[39;00m np\u001b[38;5;241m.\u001b[39marray([])\n\u001b[1;32m 30\u001b[0m nb_actual_valid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msum\u001b[39m(\u001b[38;5;241m1\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m all_valid_parameters \u001b[38;5;28;01mif\u001b[39;00m theoretical_value \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m target_level)\n", - "\u001b[0;31mNameError\u001b[0m: name 'all_valid_parameters' is not defined" + "Mean number of valid thresholds found per iteration: 0.8022\n", + "Proportion of times LTT finds no valid threshold: 0.4875\n", + "Proportion of times the risk is not controlled: 0.5125\n", + "Risk not controlled\n" ] } ], "source": [ "clf = RandomClassifier()\n", + "nb_errors = 0 # number of iterations where the risk is not controlled (i.e., not all the valid thresholds found by LTT are actually valid)\n", + "no_valid_params = 0 # number of iterations where LTT finds no valid threshold\n", + "nb_valid_params = 0 # total number of valid thresholds LTT finds over all iterations\n", "\n", "if metric == \"precision\":\n", " if target_level <= p:\n", @@ -140,23 +134,24 @@ " confidence_level=confidence_level,\n", " )\n", " controller.calibrate(X_calibrate, y_calibrate)\n", - "\n", " valid_parameters = controller.valid_thresholds\n", - " all_valid_parameters.append(valid_parameters)\n", "\n", - "all_valid_parameters = np.concatenate([x for x in all_valid_parameters if x.size > 0]) if any(x.size > 0 for x in all_valid_parameters) else np.array([])\n", + " nb_valid_params += len(valid_parameters)\n", "\n", - "nb_actual_valid = sum(1 for x in all_valid_parameters if theoretical_value >= target_level)\n", + " if len(valid_parameters) == 0:\n", + " no_valid_params += 1\n", + " \n", + " if not all(x in actual_valid_parameters for x in valid_parameters):\n", + " nb_errors += 1\n", "\n", - "print(f\"Number of valid thresholds according to LTT across all iterations: {len(all_valid_parameters)}\")\n", - "print(f\"Number of actual valid thresholds across all iterations: {nb_actual_valid}\")\n", + "print(f\"Mean number of valid thresholds found per iteration: {nb_valid_params/n_repeats}\")\n", + "print(f\"Proportion of times LTT finds no valid threshold: {no_valid_params/n_repeats}\")\n", + "print(f\"Proportion of times the risk is not controlled: {nb_errors/n_repeats}\")\n", "\n", - "counter = Counter(all_valid_parameters)\n", - "plt.bar(counter.keys(), counter.values(), width=0.008)\n", - "plt.xlabel('Parameter value')\n", - "plt.ylabel('Occurrences')\n", - "plt.title('Occurrences of each parameter value')\n", - "plt.show()" + "if nb_errors/n_repeats <= 1 - confidence_level:\n", + " print(\"Risk controlled\")\n", + "else:\n", + " print(\"Risk not controlled\")" ] }, { From 7f4e1a7f0d1e15839f0a955295ed8f1657c94a7c Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Tue, 29 Jul 2025 16:44:47 +0200 Subject: [PATCH 10/18] ENH: theoretical tests notebook + n_prime --- mapie/control_risk/ltt.py | 3 ++- ...risk_control_theoretical_tests_proto.ipynb | 24 +++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/mapie/control_risk/ltt.py b/mapie/control_risk/ltt.py index e19d3b849..d4146b0c8 100644 --- a/mapie/control_risk/ltt.py +++ b/mapie/control_risk/ltt.py @@ -64,7 +64,8 @@ def ltt_procedure( "Invalid delta: delta cannot be None while" + " controlling precision with LTT. " ) - p_values = compute_hoeffdding_bentkus_p_value(r_hat, n_obs, alpha_np, binary) + n_prime = int(n_obs/2) + p_values = compute_hoeffdding_bentkus_p_value(r_hat, n_prime, alpha_np, binary) N = len(p_values) valid_index = [] for i in range(len(alpha_np)): diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 26b49997d..137d9ef82 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 3, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 4, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 26, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -76,16 +76,16 @@ "N = 200 # size of the calibration set\n", "p = 0.5 # proportion of positives in the calibration set\n", "metric = \"precision\"\n", - "target_level = 0.7\n", - "predict_params = np.linspace(0, 0.99, 100)\n", - "confidence_level = 0.5\n", + "target_level = 0.98\n", + "predict_params = np.linspace(0, 0.99, 1)\n", + "confidence_level = 0.1\n", "\n", - "n_repeats = 10000" + "n_repeats = 100" ] }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 27, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -100,10 +100,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "Mean number of valid thresholds found per iteration: 0.8022\n", - "Proportion of times LTT finds no valid threshold: 0.4875\n", - "Proportion of times the risk is not controlled: 0.5125\n", - "Risk not controlled\n" + "Mean number of valid thresholds found per iteration: 0.0\n", + "Proportion of times LTT finds no valid threshold: 1.0\n", + "Proportion of times the risk is not controlled: 0.0\n", + "Risk controlled\n" ] } ], From 38ef7be55760f6af06a275184ec0dd2069c7fff6 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Tue, 29 Jul 2025 16:52:10 +0200 Subject: [PATCH 11/18] ENH: theoretical tests notebook + n_prime --- mapie/control_risk/ltt.py | 3 ++ ...risk_control_theoretical_tests_proto.ipynb | 30 ++++++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/mapie/control_risk/ltt.py b/mapie/control_risk/ltt.py index d4146b0c8..cf199951b 100644 --- a/mapie/control_risk/ltt.py +++ b/mapie/control_risk/ltt.py @@ -132,3 +132,6 @@ def find_lambda_control_star( l_r_star.append(r_hat[valid_index[i][idx]]) return l_lambda_star, l_r_star + +def test12(): + print("test") diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 137d9ef82..8691a382b 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 4, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -76,16 +76,16 @@ "N = 200 # size of the calibration set\n", "p = 0.5 # proportion of positives in the calibration set\n", "metric = \"precision\"\n", - "target_level = 0.98\n", - "predict_params = np.linspace(0, 0.99, 1)\n", - "confidence_level = 0.1\n", + "target_level = 0.7\n", + "predict_params = np.linspace(0, 0.99, 100)\n", + "confidence_level = 0.8\n", "\n", "n_repeats = 100" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 5, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -96,14 +96,22 @@ "outputId": "b15146cf-518e-4a93-8128-6c1865a08b01" }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/faustinpulveric/.pyenv/versions/3.10.17/envs/mapie_dev/lib/python3.10/site-packages/mapie/risk_control_draft.py:117: UserWarning: No valid thresholds found\n", + " warnings.warn(\"No valid thresholds found\", UserWarning)\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Mean number of valid thresholds found per iteration: 0.0\n", - "Proportion of times LTT finds no valid threshold: 1.0\n", - "Proportion of times the risk is not controlled: 0.0\n", - "Risk controlled\n" + "Mean number of valid thresholds found per iteration: 0.69\n", + "Proportion of times LTT finds no valid threshold: 0.49\n", + "Proportion of times the risk is not controlled: 0.51\n", + "Risk not controlled\n" ] } ], From 3c83f60c0c50095178bcc5e8cd512a336d35f0d5 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Tue, 29 Jul 2025 17:27:46 +0200 Subject: [PATCH 12/18] ENH: theoretical tests notebook --- mapie/control_risk/ltt.py | 1 + ...risk_control_theoretical_tests_proto.ipynb | 48 ++++++++++--------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/mapie/control_risk/ltt.py b/mapie/control_risk/ltt.py index cf199951b..ae942e1e1 100644 --- a/mapie/control_risk/ltt.py +++ b/mapie/control_risk/ltt.py @@ -133,5 +133,6 @@ def find_lambda_control_star( return l_lambda_star, l_r_star + def test12(): print("test") diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 8691a382b..ffa436f1d 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "id": "1c564c4f-1e63-4c2f-bdd5-d84029c1473a", "metadata": {}, "outputs": [], @@ -23,24 +23,36 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" }, - "outputs": [], + "outputs": [ + { + "ename": "ImportError", + "evalue": "cannot import name 'test12' from 'mapie.control_risk.ltt' (/Users/faustinpulveric/.pyenv/versions/3.10.17/envs/mapie_dev/lib/python3.10/site-packages/mapie/control_risk/ltt.py)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[4], line 6\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m pyplot \u001b[38;5;28;01mas\u001b[39;00m plt\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mmapie\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mrisk_control_draft\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m BinaryClassificationController\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mmapie\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcontrol_risk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mltt\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m test12\n", + "\u001b[0;31mImportError\u001b[0m: cannot import name 'test12' from 'mapie.control_risk.ltt' (/Users/faustinpulveric/.pyenv/versions/3.10.17/envs/mapie_dev/lib/python3.10/site-packages/mapie/control_risk/ltt.py)" + ] + } + ], "source": [ "import numpy as np\n", "import itertools\n", "from matplotlib import pyplot as plt\n", - "from collections import Counter\n", "\n", - "from mapie.risk_control_draft import BinaryClassificationController" + "from mapie.risk_control_draft import BinaryClassificationController\n", + "from mapie.control_risk.ltt import test12" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -66,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 20, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -76,16 +88,16 @@ "N = 200 # size of the calibration set\n", "p = 0.5 # proportion of positives in the calibration set\n", "metric = \"precision\"\n", - "target_level = 0.7\n", + "target_level = 0.95\n", "predict_params = np.linspace(0, 0.99, 100)\n", - "confidence_level = 0.8\n", + "confidence_level = 0.9\n", "\n", "n_repeats = 100" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 21, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -96,22 +108,14 @@ "outputId": "b15146cf-518e-4a93-8128-6c1865a08b01" }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/faustinpulveric/.pyenv/versions/3.10.17/envs/mapie_dev/lib/python3.10/site-packages/mapie/risk_control_draft.py:117: UserWarning: No valid thresholds found\n", - " warnings.warn(\"No valid thresholds found\", UserWarning)\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "Mean number of valid thresholds found per iteration: 0.69\n", - "Proportion of times LTT finds no valid threshold: 0.49\n", - "Proportion of times the risk is not controlled: 0.51\n", - "Risk not controlled\n" + "Mean number of valid thresholds found per iteration: 0.0\n", + "Proportion of times LTT finds no valid threshold: 1.0\n", + "Proportion of times the risk is not controlled: 0.0\n", + "Risk controlled\n" ] } ], From 6dc2d8c75ed262aa9e83249c0a8b1991cf11a703 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Tue, 29 Jul 2025 17:33:59 +0200 Subject: [PATCH 13/18] ENH: theoretical tests notebook --- mapie/control_risk/ltt.py | 4 -- ...risk_control_theoretical_tests_proto.ipynb | 37 ++++++------------- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/mapie/control_risk/ltt.py b/mapie/control_risk/ltt.py index ae942e1e1..d4146b0c8 100644 --- a/mapie/control_risk/ltt.py +++ b/mapie/control_risk/ltt.py @@ -132,7 +132,3 @@ def find_lambda_control_star( l_r_star.append(r_hat[valid_index[i][idx]]) return l_lambda_star, l_r_star - - -def test12(): - print("test") diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index ffa436f1d..712e95d1c 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "1c564c4f-1e63-4c2f-bdd5-d84029c1473a", "metadata": {}, "outputs": [], @@ -23,36 +23,23 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" }, - "outputs": [ - { - "ename": "ImportError", - "evalue": "cannot import name 'test12' from 'mapie.control_risk.ltt' (/Users/faustinpulveric/.pyenv/versions/3.10.17/envs/mapie_dev/lib/python3.10/site-packages/mapie/control_risk/ltt.py)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[4], line 6\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m pyplot \u001b[38;5;28;01mas\u001b[39;00m plt\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mmapie\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mrisk_control_draft\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m BinaryClassificationController\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mmapie\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcontrol_risk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mltt\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m test12\n", - "\u001b[0;31mImportError\u001b[0m: cannot import name 'test12' from 'mapie.control_risk.ltt' (/Users/faustinpulveric/.pyenv/versions/3.10.17/envs/mapie_dev/lib/python3.10/site-packages/mapie/control_risk/ltt.py)" - ] - } - ], + "outputs": [], "source": [ "import numpy as np\n", "import itertools\n", "from matplotlib import pyplot as plt\n", "\n", - "from mapie.risk_control_draft import BinaryClassificationController\n", - "from mapie.control_risk.ltt import test12" + "from mapie.risk_control_draft import BinaryClassificationController" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -78,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 7, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -88,7 +75,7 @@ "N = 200 # size of the calibration set\n", "p = 0.5 # proportion of positives in the calibration set\n", "metric = \"precision\"\n", - "target_level = 0.95\n", + "target_level = 0.8\n", "predict_params = np.linspace(0, 0.99, 100)\n", "confidence_level = 0.9\n", "\n", @@ -97,7 +84,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 8, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -112,10 +99,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "Mean number of valid thresholds found per iteration: 0.0\n", - "Proportion of times LTT finds no valid threshold: 1.0\n", - "Proportion of times the risk is not controlled: 0.0\n", - "Risk controlled\n" + "Mean number of valid thresholds found per iteration: 0.69\n", + "Proportion of times LTT finds no valid threshold: 0.43\n", + "Proportion of times the risk is not controlled: 0.57\n", + "Risk not controlled\n" ] } ], From 0388177a1a66c7803e787c495469860c0c991977 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Wed, 30 Jul 2025 11:01:57 +0200 Subject: [PATCH 14/18] ENH: theoretical tests notebook - accuracy --- mapie/control_risk/ltt.py | 3 +- ...risk_control_theoretical_tests_proto.ipynb | 46 ++++++++++--------- mapie/risk_control_draft.py | 20 +++++++- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/mapie/control_risk/ltt.py b/mapie/control_risk/ltt.py index d4146b0c8..e19d3b849 100644 --- a/mapie/control_risk/ltt.py +++ b/mapie/control_risk/ltt.py @@ -64,8 +64,7 @@ def ltt_procedure( "Invalid delta: delta cannot be None while" + " controlling precision with LTT. " ) - n_prime = int(n_obs/2) - p_values = compute_hoeffdding_bentkus_p_value(r_hat, n_prime, alpha_np, binary) + p_values = compute_hoeffdding_bentkus_p_value(r_hat, n_obs, alpha_np, binary) N = len(p_values) valid_index = [] for i in range(len(alpha_np)): diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 712e95d1c..71912b0b2 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "1c564c4f-1e63-4c2f-bdd5-d84029c1473a", "metadata": {}, "outputs": [], @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 3, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -74,17 +74,18 @@ "source": [ "N = 200 # size of the calibration set\n", "p = 0.5 # proportion of positives in the calibration set\n", - "metric = \"precision\"\n", - "target_level = 0.8\n", + "metric = \"accuracy\"\n", + "target_level = 0.45\n", "predict_params = np.linspace(0, 0.99, 100)\n", - "confidence_level = 0.9\n", + "# predict_params = np.array([0])\n", + "confidence_level = 0.1\n", "\n", - "n_repeats = 100" + "n_repeats = 1000" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 4, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -95,14 +96,22 @@ "outputId": "b15146cf-518e-4a93-8128-6c1865a08b01" }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/faustinpulveric/MAPIE/MAPIE/mapie/risk_control_draft.py:117: UserWarning: No valid thresholds found\n", + " warnings.warn(\"No valid thresholds found\", UserWarning)\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Mean number of valid thresholds found per iteration: 0.69\n", - "Proportion of times LTT finds no valid threshold: 0.43\n", - "Proportion of times the risk is not controlled: 0.57\n", - "Risk not controlled\n" + "Mean number of valid thresholds found per iteration: 8.883\n", + "Proportion of times LTT finds no valid threshold: 0.514\n", + "Proportion of times the risk is not controlled: 0.0\n", + "Risk controlled\n" ] } ], @@ -112,14 +121,6 @@ "no_valid_params = 0 # number of iterations where LTT finds no valid threshold\n", "nb_valid_params = 0 # total number of valid thresholds LTT finds over all iterations\n", "\n", - "if metric == \"precision\":\n", - " if target_level <= p:\n", - " actual_valid_parameters = predict_params\n", - " else:\n", - " actual_valid_parameters = []\n", - "elif metric == \"recall\":\n", - " actual_valid_parameters = predict_params[predict_params <= np.round(1-target_level, 2)]\n", - "\n", "for _ in range(n_repeats):\n", "\n", " X_calibrate = list(range(1, N+1))\n", @@ -132,6 +133,7 @@ " target_level=target_level,\n", " confidence_level=confidence_level,\n", " )\n", + " controller._thresholds = predict_params\n", " controller.calibrate(X_calibrate, y_calibrate)\n", " valid_parameters = controller.valid_thresholds\n", "\n", @@ -140,7 +142,7 @@ " if len(valid_parameters) == 0:\n", " no_valid_params += 1\n", " \n", - " if not all(x in actual_valid_parameters for x in valid_parameters):\n", + " if target_level > p and len(valid_parameters) >= 1:\n", " nb_errors += 1\n", "\n", "print(f\"Mean number of valid thresholds found per iteration: {nb_valid_params/n_repeats}\")\n", diff --git a/mapie/risk_control_draft.py b/mapie/risk_control_draft.py index 1c3b7d7b8..cacd82032 100644 --- a/mapie/risk_control_draft.py +++ b/mapie/risk_control_draft.py @@ -101,7 +101,7 @@ def calibrate(self, X_calibrate: ArrayLike, y_calibrate: ArrayLike) -> None: predictions_proba = self._classifier.predict_proba(X_calibrate)[:, 1] - risk_per_threshold = 1 - self._compute_precision( + risk_per_threshold = 1 - self._compute_accuracy( predictions_proba, y_calibrate_ ) @@ -197,3 +197,21 @@ def _compute_recall( ) return recall_per_threshold + + def _compute_accuracy( + self, predictions_proba: NDArray[np.float32], y_cal: NDArray[np.float32] + ) -> NDArray[np.float32]: + """ + Compute the accuracy for each threshold. + """ + predictions_per_threshold = ( + predictions_proba[:, np.newaxis] >= self._thresholds + ).astype(int) + + correct_predictions = ( + predictions_per_threshold == y_cal[:, np.newaxis] + ).astype(int) + + accuracy_per_threshold = np.mean(correct_predictions, axis=0) + + return accuracy_per_threshold From e330e30cd40882f83f27e21ff9bd1db8967a2bb3 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Wed, 30 Jul 2025 16:14:41 +0200 Subject: [PATCH 15/18] ENH: theoretical tests notebook - recall --- ...risk_control_theoretical_tests_proto.ipynb | 53 ++++++++++--------- mapie/risk_control_draft.py | 7 ++- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 71912b0b2..1038de99c 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -30,11 +30,12 @@ }, "outputs": [], "source": [ + "from sklearn.datasets import make_classification\n", "import numpy as np\n", "import itertools\n", "from matplotlib import pyplot as plt\n", "\n", - "from mapie.risk_control_draft import BinaryClassificationController" + "from mapie.risk_control_draft import BinaryClassificationController, test23" ] }, { @@ -65,27 +66,26 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 12, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" }, "outputs": [], "source": [ - "N = 200 # size of the calibration set\n", + "N = 100 # size of the calibration set\n", "p = 0.5 # proportion of positives in the calibration set\n", - "metric = \"accuracy\"\n", - "target_level = 0.45\n", - "predict_params = np.linspace(0, 0.99, 100)\n", - "# predict_params = np.array([0])\n", - "confidence_level = 0.1\n", + "metric = \"recall\"\n", + "target_level = 0.8\n", + "predict_params = np.linspace(0, 0.99, 10)\n", + "confidence_level = 0.7\n", "\n", - "n_repeats = 1000" + "n_repeats = 100" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 14, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -96,20 +96,12 @@ "outputId": "b15146cf-518e-4a93-8128-6c1865a08b01" }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/faustinpulveric/MAPIE/MAPIE/mapie/risk_control_draft.py:117: UserWarning: No valid thresholds found\n", - " warnings.warn(\"No valid thresholds found\", UserWarning)\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "Mean number of valid thresholds found per iteration: 8.883\n", - "Proportion of times LTT finds no valid threshold: 0.514\n", + "Mean number of valid thresholds found per iteration: 1.33\n", + "Proportion of times LTT finds no valid threshold: 0.0\n", "Proportion of times the risk is not controlled: 0.0\n", "Risk controlled\n" ] @@ -123,9 +115,19 @@ "\n", "for _ in range(n_repeats):\n", "\n", - " X_calibrate = list(range(1, N+1))\n", - " y_calibrate = [1] * int(p*N) + [0] * (N - int(p*N))\n", - " np.random.shuffle(y_calibrate)\n", + " X_calibrate, y_calibrate = make_classification(\n", + " n_samples=N,\n", + " n_features=1,\n", + " n_informative=1,\n", + " n_redundant=0,\n", + " n_repeated=0,\n", + " n_classes=2,\n", + " n_clusters_per_class=1,\n", + " weights=[1 - p, p],\n", + " flip_y=0.5,\n", + " random_state=None\n", + " )\n", + " X_calibrate = X_calibrate.squeeze()\n", "\n", " controller = BinaryClassificationController(\n", " fitted_binary_classifier=clf,\n", @@ -142,7 +144,8 @@ " if len(valid_parameters) == 0:\n", " no_valid_params += 1\n", " \n", - " if target_level > p and len(valid_parameters) >= 1:\n", + " # if target_level > p and len(valid_parameters) >= 1:\n", + " if any(x < 0 or x > np.round(1-target_level, 2) for x in valid_parameters) and len(valid_parameters) >= 1:\n", " nb_errors += 1\n", "\n", "print(f\"Mean number of valid thresholds found per iteration: {nb_valid_params/n_repeats}\")\n", @@ -158,7 +161,7 @@ { "cell_type": "code", "execution_count": null, - "id": "1c1f49d5-234a-4a88-b030-b210bb16af6b", + "id": "104c7232-c8b1-432e-94dd-3f65e730483f", "metadata": {}, "outputs": [], "source": [] diff --git a/mapie/risk_control_draft.py b/mapie/risk_control_draft.py index cacd82032..952e4f905 100644 --- a/mapie/risk_control_draft.py +++ b/mapie/risk_control_draft.py @@ -101,7 +101,7 @@ def calibrate(self, X_calibrate: ArrayLike, y_calibrate: ArrayLike) -> None: predictions_proba = self._classifier.predict_proba(X_calibrate)[:, 1] - risk_per_threshold = 1 - self._compute_accuracy( + risk_per_threshold = 1 - self._compute_recall( predictions_proba, y_calibrate_ ) @@ -109,7 +109,7 @@ def calibrate(self, X_calibrate: ArrayLike, y_calibrate: ArrayLike) -> None: risk_per_threshold, np.array([self._alpha]), self._delta, - len(y_calibrate_), + int(len(y_calibrate_)/2), True, ) self.valid_thresholds = self._thresholds[valid_thresholds_index[0]] @@ -215,3 +215,6 @@ def _compute_accuracy( accuracy_per_threshold = np.mean(correct_predictions, axis=0) return accuracy_per_threshold + +def test2(): + print("test") From 7c6ccc7f1a7af0c520ab0bf71a0aff6ec198fde2 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Wed, 30 Jul 2025 16:19:18 +0200 Subject: [PATCH 16/18] ENH: theoretical tests notebook - recall --- .../risk_control_theoretical_tests_proto.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 1038de99c..7f64d06aa 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -35,7 +35,7 @@ "import itertools\n", "from matplotlib import pyplot as plt\n", "\n", - "from mapie.risk_control_draft import BinaryClassificationController, test23" + "from mapie.risk_control_draft import BinaryClassificationController, test2" ] }, { @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 3, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -85,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 4, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -100,9 +100,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Mean number of valid thresholds found per iteration: 1.33\n", + "Mean number of valid thresholds found per iteration: 1.37\n", "Proportion of times LTT finds no valid threshold: 0.0\n", - "Proportion of times the risk is not controlled: 0.0\n", + "Proportion of times the risk is not controlled: 0.01\n", "Risk controlled\n" ] } From 2108d64ebab0a563ac09abf14537b02d8c11cdab Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Wed, 30 Jul 2025 16:39:04 +0200 Subject: [PATCH 17/18] ENH: theoretical tests notebook - recall --- ...risk_control_theoretical_tests_proto.ipynb | 27 +++++----- mapie/risk_control_draft.py | 49 ++++++++++++++----- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 7f64d06aa..1b14dbe98 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 8, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -35,12 +35,12 @@ "import itertools\n", "from matplotlib import pyplot as plt\n", "\n", - "from mapie.risk_control_draft import BinaryClassificationController, test2" + "from mapie.risk_control_draft import BinaryClassificationController" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 9, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 10, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -77,7 +77,7 @@ "p = 0.5 # proportion of positives in the calibration set\n", "metric = \"recall\"\n", "target_level = 0.8\n", - "predict_params = np.linspace(0, 0.99, 10)\n", + "predict_params = np.linspace(0, 0.99, 100)\n", "confidence_level = 0.7\n", "\n", "n_repeats = 100" @@ -85,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 11, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -100,9 +100,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Mean number of valid thresholds found per iteration: 1.37\n", + "Mean number of valid thresholds found per iteration: 5.82\n", "Proportion of times LTT finds no valid threshold: 0.0\n", - "Proportion of times the risk is not controlled: 0.01\n", + "Proportion of times the risk is not controlled: 0.0\n", "Risk controlled\n" ] } @@ -143,10 +143,13 @@ "\n", " if len(valid_parameters) == 0:\n", " no_valid_params += 1\n", - " \n", - " # if target_level > p and len(valid_parameters) >= 1:\n", - " if any(x < 0 or x > np.round(1-target_level, 2) for x in valid_parameters) and len(valid_parameters) >= 1:\n", - " nb_errors += 1\n", + "\n", + " if metric == \"precision\" or metric == \"accuracy\":\n", + " if target_level > p and len(valid_parameters) >= 1:\n", + " nb_errors += 1\n", + " elif metric == \"recall\":\n", + " if any(x < 0 or x > np.round(1-target_level, 2) for x in valid_parameters) and len(valid_parameters) >= 1:\n", + " nb_errors += 1\n", "\n", "print(f\"Mean number of valid thresholds found per iteration: {nb_valid_params/n_repeats}\")\n", "print(f\"Proportion of times LTT finds no valid threshold: {no_valid_params/n_repeats}\")\n", diff --git a/mapie/risk_control_draft.py b/mapie/risk_control_draft.py index 952e4f905..7188f9377 100644 --- a/mapie/risk_control_draft.py +++ b/mapie/risk_control_draft.py @@ -70,6 +70,7 @@ def __init__( self._n_jobs = n_jobs # TODO : use this in the class or delete self._random_state = random_state # TODO : use this in the class or delete self._verbose = verbose # TODO : use this in the class or delete + self._metric = metric self._thresholds: NDArray[np.float32] = np.arange(0, 1, 0.01) # TODO: add a _is_calibrated attribute to check at prediction time @@ -101,17 +102,42 @@ def calibrate(self, X_calibrate: ArrayLike, y_calibrate: ArrayLike) -> None: predictions_proba = self._classifier.predict_proba(X_calibrate)[:, 1] - risk_per_threshold = 1 - self._compute_recall( - predictions_proba, y_calibrate_ - ) + if self._metric == "precision": + risk_per_threshold = 1 - self._compute_precision( + predictions_proba, y_calibrate_ + ) + valid_thresholds_index, _ = ltt_procedure( + risk_per_threshold, + np.array([self._alpha]), + self._delta, + int(len(y_calibrate_)/2), + True, + ) + + elif self._metric == "recall": + risk_per_threshold = 1 - self._compute_recall( + predictions_proba, y_calibrate_ + ) + valid_thresholds_index, _ = ltt_procedure( + risk_per_threshold, + np.array([self._alpha]), + self._delta, + int(len(y_calibrate_)/2), + True, + ) + + elif self._metric == "accuracy": + risk_per_threshold = 1 - self._compute_accuracy( + predictions_proba, y_calibrate_ + ) + valid_thresholds_index, _ = ltt_procedure( + risk_per_threshold, + np.array([self._alpha]), + self._delta, + len(y_calibrate_), + True, + ) - valid_thresholds_index, _ = ltt_procedure( - risk_per_threshold, - np.array([self._alpha]), - self._delta, - int(len(y_calibrate_)/2), - True, - ) self.valid_thresholds = self._thresholds[valid_thresholds_index[0]] if len(self.valid_thresholds) == 0: warnings.warn("No valid thresholds found", UserWarning) @@ -215,6 +241,3 @@ def _compute_accuracy( accuracy_per_threshold = np.mean(correct_predictions, axis=0) return accuracy_per_threshold - -def test2(): - print("test") From 9245ed7909d5f9b3aca3aa18889dd5eea934d1d3 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Thu, 31 Jul 2025 10:59:41 +0200 Subject: [PATCH 18/18] ENH: theoretical tests notebook - fix precision --- ...risk_control_theoretical_tests_proto.ipynb | 29 ++++++++++--------- mapie/risk_control_draft.py | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb index 1b14dbe98..3bf9e9d47 100644 --- a/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb +++ b/mapie/control_risk/risk_control_theoretical_tests_proto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "id": "f1c2e64a", "metadata": { "id": "f1c2e64a" @@ -34,13 +34,14 @@ "import numpy as np\n", "import itertools\n", "from matplotlib import pyplot as plt\n", + "from sklearn.dummy import DummyClassifier\n", "\n", "from mapie.risk_control_draft import BinaryClassificationController" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "id": "6c0b5e81-81f1-4688-a4d7-57c6adba44b4", "metadata": {}, "outputs": [], @@ -66,7 +67,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 75, "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0", "metadata": { "id": "8da2839a-3f14-4054-acf1-c60b1d02b7d0" @@ -74,9 +75,9 @@ "outputs": [], "source": [ "N = 100 # size of the calibration set\n", - "p = 0.5 # proportion of positives in the calibration set\n", - "metric = \"recall\"\n", - "target_level = 0.8\n", + "p = 0.5 # proportion of positives in the data generator\n", + "metric = \"precision\"\n", + "target_level = 0.6\n", "predict_params = np.linspace(0, 0.99, 100)\n", "confidence_level = 0.7\n", "\n", @@ -85,7 +86,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 76, "id": "03383363-b86d-4593-adf4-80215b6f1dcf", "metadata": { "colab": { @@ -100,10 +101,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "Mean number of valid thresholds found per iteration: 5.82\n", - "Proportion of times LTT finds no valid threshold: 0.0\n", - "Proportion of times the risk is not controlled: 0.0\n", - "Risk controlled\n" + "Mean number of valid thresholds found per iteration: 1.37\n", + "Proportion of times LTT finds no valid threshold: 0.63\n", + "Proportion of times the risk is not controlled: 0.37\n", + "Risk level: 0.30000000000000004\n", + "Risk not controlled\n" ] } ], @@ -124,7 +126,7 @@ " n_classes=2,\n", " n_clusters_per_class=1,\n", " weights=[1 - p, p],\n", - " flip_y=0.5,\n", + " flip_y=0,\n", " random_state=None\n", " )\n", " X_calibrate = X_calibrate.squeeze()\n", @@ -144,7 +146,7 @@ " if len(valid_parameters) == 0:\n", " no_valid_params += 1\n", "\n", - " if metric == \"precision\" or metric == \"accuracy\":\n", + " if metric == \"precision\" or metric == \"accuracy\": # vérifier que l'accuracy ne dépend pas de p\n", " if target_level > p and len(valid_parameters) >= 1:\n", " nb_errors += 1\n", " elif metric == \"recall\":\n", @@ -154,6 +156,7 @@ "print(f\"Mean number of valid thresholds found per iteration: {nb_valid_params/n_repeats}\")\n", "print(f\"Proportion of times LTT finds no valid threshold: {no_valid_params/n_repeats}\")\n", "print(f\"Proportion of times the risk is not controlled: {nb_errors/n_repeats}\")\n", + "print(f\"Risk level: {1-confidence_level}\")\n", "\n", "if nb_errors/n_repeats <= 1 - confidence_level:\n", " print(\"Risk controlled\")\n", diff --git a/mapie/risk_control_draft.py b/mapie/risk_control_draft.py index 7188f9377..334f157df 100644 --- a/mapie/risk_control_draft.py +++ b/mapie/risk_control_draft.py @@ -186,7 +186,7 @@ def _compute_precision( # TODO: use sklearn or MAPIE ? positive_predictions = true_positives + false_positives # Avoid division by zero - precision_per_threshold = np.ones_like(self._thresholds, dtype=float) + precision_per_threshold = np.zeros_like(self._thresholds, dtype=float) nonzero_mask = positive_predictions > 0 precision_per_threshold[nonzero_mask] = ( true_positives[nonzero_mask] / positive_predictions[nonzero_mask]