invalid_target data_check added

Author: Nabil Fayak

checkmates/objectives/__init__.py

@@ -11,3 +11,6 @@
 
 from checkmates.objectives.standard_metrics import RootMeanSquaredLogError
 from checkmates.objectives.standard_metrics import MeanSquaredLogError
+
+from checkmates.objectives.binary_classification_objective import BinaryClassificationObjective
+from checkmates.objectives.multiclass_classification_objective import MulticlassClassificationObjective

checkmates/objectives/binary_classification_objective.py

@@ -0,0 +1,84 @@
+"""Base class for all binary classification objectives."""
+import numpy as np
+from scipy.optimize import differential_evolution
+
+from checkmates.objectives.objective_base import ObjectiveBase
+from checkmates.problem_types import ProblemTypes
+
+
+class BinaryClassificationObjective(ObjectiveBase):
+    """Base class for all binary classification objectives."""
+
+    problem_types = [ProblemTypes.BINARY, ProblemTypes.TIME_SERIES_BINARY]
+
+    """[ProblemTypes.BINARY, ProblemTypes.TIME_SERIES_BINARY]"""
+
+    @property
+    def can_optimize_threshold(cls):
+        """Returns a boolean determining if we can optimize the binary classification objective threshold.
+
+        This will be false for any objective that works directly with
+        predicted probabilities, like log loss and AUC. Otherwise, it
+        will be true.
+
+        Returns:
+            bool: Whether or not an objective can be optimized.
+        """
+        return not cls.score_needs_proba
+
+    def optimize_threshold(self, ypred_proba, y_true, X=None):
+        """Learn a binary classification threshold which optimizes the current objective.
+
+        Args:
+            ypred_proba (pd.Series): The classifier's predicted probabilities
+            y_true (pd.Series): The ground truth for the predictions.
+            X (pd.DataFrame, optional): Any extra columns that are needed from training data.
+
+        Returns:
+            Optimal threshold for this objective.
+
+        Raises:
+            RuntimeError: If objective cannot be optimized.
+        """
+        ypred_proba = self._standardize_input_type(ypred_proba)
+        y_true = self._standardize_input_type(y_true)
+        if X is not None:
+            X = self._standardize_input_type(X)
+
+        if not self.can_optimize_threshold:
+            raise RuntimeError("Trying to optimize objective that can't be optimized!")
+
+        def cost(threshold):
+            y_predicted = self.decision_function(
+                ypred_proba=ypred_proba,
+                threshold=threshold[0],
+                X=X,
+            )
+            cost = self.objective_function(y_true, y_predicted, X=X)
+            return -cost if self.greater_is_better else cost
+
+        optimal = differential_evolution(cost, bounds=[(0, 1)], seed=0, maxiter=250)
+
+        return optimal.x[0]
+
+    def decision_function(self, ypred_proba, threshold=0.5, X=None):
+        """Apply a learned threshold to predicted probabilities to get predicted classes.
+
+        Args:
+            ypred_proba (pd.Series, np.ndarray): The classifier's predicted probabilities
+            threshold (float, optional): Threshold used to make a prediction. Defaults to 0.5.
+            X (pd.DataFrame, optional): Any extra columns that are needed from training data.
+
+        Returns:
+            predictions
+        """
+        ypred_proba = self._standardize_input_type(ypred_proba)
+        return ypred_proba > threshold
+
+    def validate_inputs(self, y_true, y_predicted):
+        """Validate inputs for scoring."""
+        super().validate_inputs(y_true, y_predicted)
+        if len(np.unique(y_true)) > 2:
+            raise ValueError("y_true contains more than two unique values")
+        if len(np.unique(y_predicted)) > 2 and not self.score_needs_proba:
+            raise ValueError("y_predicted contains more than two unique values")

checkmates/objectives/multiclass_classification_objective.py

@@ -0,0 +1,10 @@
+"""Base class for all multiclass classification objectives."""
+from checkmates.objectives.objective_base import ObjectiveBase
+from checkmates.problem_types import ProblemTypes
+
+
+class MulticlassClassificationObjective(ObjectiveBase):
+    """Base class for all multiclass classification objectives."""
+
+    problem_types = [ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS]
+    """[ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS]"""

checkmates/objectives/standard_metrics.py

@@ -5,6 +5,127 @@
 
 from checkmates.objectives.regression_objective import RegressionObjective
 from checkmates.utils import classproperty
+from checkmates.objectives.binary_classification_objective import BinaryClassificationObjective
+from checkmates.objectives.multiclass_classification_objective import MulticlassClassificationObjective
+
+
+class LogLossBinary(BinaryClassificationObjective):
+    """Log Loss for binary classification.
+
+    Example:
+        >>> y_true = pd.Series([0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1])
+        >>> y_pred = pd.Series([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1])
+        >>> np.testing.assert_almost_equal(LogLossBinary().objective_function(y_true, y_pred), 19.6601745)
+    """
+
+    name = "Log Loss Binary"
+    greater_is_better = False
+    score_needs_proba = True
+    perfect_score = 0.0
+    is_bounded_like_percentage = False  # Range [0, Inf)
+    expected_range = [0, 1]
+
+    def objective_function(
+        self,
+        y_true,
+        y_predicted,
+        y_train=None,
+        X=None,
+        sample_weight=None,
+    ):
+        """Objective function for log loss for binary classification."""
+        return metrics.log_loss(y_true, y_predicted, sample_weight=sample_weight)
+
+class LogLossMulticlass(MulticlassClassificationObjective):
+    """Log Loss for multiclass classification.
+
+    Example:
+        >>> y_true = [0, 1, 2, 0, 2, 1]
+        >>> y_pred = [[0.7, 0.2, 0.1],
+        ...           [0.3, 0.5, 0.2],
+        ...           [0.1, 0.3, 0.6],
+        ...           [0.9, 0.1, 0.0],
+        ...           [0.3, 0.1, 0.6],
+        ...           [0.5, 0.5, 0.0]]
+        >>> np.testing.assert_almost_equal(LogLossMulticlass().objective_function(y_true, y_pred), 0.4783301)
+    """
+
+    name = "Log Loss Multiclass"
+    greater_is_better = False
+    score_needs_proba = True
+    perfect_score = 0.0
+    is_bounded_like_percentage = False  # Range [0, Inf)
+    expected_range = [0, 1]
+
+    def objective_function(
+        self,
+        y_true,
+        y_predicted,
+        y_train=None,
+        X=None,
+        sample_weight=None,
+    ):
+        """Objective function for log loss for multiclass classification."""
+        return metrics.log_loss(y_true, y_predicted, sample_weight=sample_weight)
+
+class R2(RegressionObjective):
+    """Coefficient of determination for regression.
+
+    Example:
+        >>> y_true = pd.Series([1.5, 2, 3, 1, 0.5, 1, 2.5, 2.5, 1, 0.5, 2])
+        >>> y_pred = pd.Series([1.5, 2.5, 2, 1, 0.5, 1, 3, 2.25, 0.75, 0.25, 1.75])
+        >>> np.testing.assert_almost_equal(R2().objective_function(y_true, y_pred), 0.7638036)
+    """
+
+    name = "R2"
+    greater_is_better = True
+    score_needs_proba = False
+    perfect_score = 1
+    is_bounded_like_percentage = False  # Range (-Inf, 1]
+    expected_range = [-1, 1]
+
+    def objective_function(
+        self,
+        y_true,
+        y_predicted,
+        y_train=None,
+        X=None,
+        sample_weight=None,
+    ):
+        """Objective function for coefficient of determination for regression."""
+        return metrics.r2_score(y_true, y_predicted, sample_weight=sample_weight)
+
+class MedianAE(RegressionObjective):
+    """Median absolute error for regression.
+
+    Example:
+        >>> y_true = pd.Series([1.5, 2, 3, 1, 0.5, 1, 2.5, 2.5, 1, 0.5, 2])
+        >>> y_pred = pd.Series([1.5, 2.5, 2, 1, 0.5, 1, 3, 2.25, 0.75, 0.25, 1.75])
+        >>> np.testing.assert_almost_equal(MedianAE().objective_function(y_true, y_pred), 0.25)
+    """
+
+    name = "MedianAE"
+    greater_is_better = False
+    score_needs_proba = False
+    perfect_score = 0.0
+    is_bounded_like_percentage = False  # Range [0, Inf)
+    expected_range = [0, float("inf")]
+
+    def objective_function(
+        self,
+        y_true,
+        y_predicted,
+        y_train=None,
+        X=None,
+        sample_weight=None,
+    ):
+        """Objective function for median absolute error for regression."""
+        return metrics.median_absolute_error(
+            y_true,
+            y_predicted,
+            sample_weight=sample_weight,
+        )
+
 
 
 class RootMeanSquaredLogError(RegressionObjective):

checkmates/objectives/utils.py

@@ -1,4 +1,4 @@
-"""Utility methods for EvalML objectives."""
+"""Utility methods for CheckMates objectives."""
 from checkmates import objectives
 from checkmates.exceptions import ObjectiveCreationError, ObjectiveNotFoundError
 from checkmates.objectives.objective_base import ObjectiveBase
@@ -20,12 +20,20 @@ def get_non_core_objectives():
         objectives.RootMeanSquaredLogError,
     ]
 
+def get_all_objective_names():
+    """Get a list of the names of all objectives.
+
+    Returns:
+        list (str): Objective names
+    """
+    all_objectives_dict = _all_objectives_dict()
+    return list(all_objectives_dict.keys())
 
 def _all_objectives_dict():
     all_objectives = _get_subclasses(ObjectiveBase)
     objectives_dict = {}
     for objective in all_objectives:
-        if "evalml.objectives" not in objective.__module__:
+        if "checkmates.objectives" not in objective.__module__:
             continue
         objectives_dict[objective.name.lower()] = objective
     return objectives_dict
@@ -63,7 +71,7 @@ def get_objective(objective, return_instance=False, **kwargs):
     if objective.lower() not in all_objectives_dict:
         raise ObjectiveNotFoundError(
             f"{objective} is not a valid Objective! "
-            "Use evalml.objectives.get_all_objective_names() "
+            "Use checkmates.objectives.get_all_objective_names() "
             "to get a list of all valid objective names. ",
         )