Extract early stopping message constants and helpers to OSS (#4751)

Summary: Pull Request resolved: #4751

Differential Revision: D90404643

Author: shrutipatel31

ax/analysis/healthcheck/early_stopping_healthcheck.py

@@ -17,15 +17,20 @@
 )
 from ax.analysis.utils import validate_experiment
 from ax.core.base_trial import TrialStatus
-from ax.core.data import MAP_KEY
 from ax.core.experiment import Experiment
 from ax.core.map_metric import MapMetric
 from ax.core.optimization_config import MultiObjectiveOptimizationConfig
 from ax.early_stopping.dispatch import get_default_ess_or_none
 from ax.early_stopping.experiment_replay import replay_experiment
 from ax.early_stopping.strategies.base import BaseEarlyStoppingStrategy
 from ax.early_stopping.strategies.percentile import PercentileEarlyStoppingStrategy
-from ax.early_stopping.utils import estimate_early_stopping_savings
+from ax.early_stopping.utils import (
+    EARLY_STOPPING_NUDGE_MSG,
+    EARLY_STOPPING_NUDGE_TITLE,
+    EARLY_STOPPING_SAVINGS_TITLE,
+    estimate_early_stopping_savings,
+    format_early_stopping_savings_message,
+)
 from ax.generation_strategy.generation_strategy import GenerationStrategy
 from ax.service.utils.early_stopping import get_early_stopping_metrics
 from pyre_extensions import none_throws, override
@@ -350,26 +355,10 @@ def _report_early_stopping_status(
         n_running = len(experiment.trial_indices_by_status[TrialStatus.RUNNING])
         n_ran = n_stopped + n_completed + n_failed + n_running
 
-        ess_msg = (
-            f"Throughout this experiment, {n_stopped} trials were early stopped, "
-            f"out of a total of {n_ran} trials. "
-        )
-
-        if savings > 0:
-            ess_msg += (
-                f"The capacity savings (computed using {MAP_KEY}) are "
-                f"estimated to be {savings * 100:.0f}%."
-            )
-        else:
-            ess_msg += (
-                "Capacity savings are not yet available. Either no trials have "
-                "been early stopped, or no trials have completed (which is "
-                "required to estimate savings). Check back once more trials are "
-                "completed and/or early stopped."
-            )
-
         # Add multi-objective note if applicable
-        subtitle = ess_msg
+        subtitle = format_early_stopping_savings_message(
+            n_stopped=n_stopped, n_ran=n_ran, savings=savings
+        )
         if len(target_ess_metric_names) > 1:
             subtitle += (
                 f"\n\nNote: Although {len(target_ess_metric_names)} metrics are "
@@ -397,7 +386,7 @@ def _report_early_stopping_status(
         )
 
         return self._create_card(
-            title="Capacity savings due to early stopping",
+            title=EARLY_STOPPING_SAVINGS_TITLE,
             subtitle=subtitle,
             df=df,
             status=HealthcheckStatus.PASS,
@@ -437,13 +426,8 @@ def _report_early_stopping_nudge(
         best_metric_name = max(metric_to_savings, key=metric_to_savings.get)
         best_savings = metric_to_savings[best_metric_name]
 
-        subtitle = (
-            "This sweep uses metrics that are **compatible with early stopping**! "
-            "Using early stopping could have saved you both capacity and "
-            "optimization wall time. For example, we estimate that using early "
-            f"stopping on the '{best_metric_name}' metric could have provided "
-            f"{best_savings:.0f}% capacity savings, with no regression in "
-            "optimization performance."
+        subtitle = EARLY_STOPPING_NUDGE_MSG.format(
+            metric_name=best_metric_name, savings=best_savings
         )
 
         # Append additional info if provided
@@ -462,10 +446,7 @@ def _report_early_stopping_nudge(
         ]
         df = pd.DataFrame(metric_rows)
 
-        title = (
-            f"{best_savings:.0f}% potential capacity savings if you turn on "
-            f"early stopping feature"
-        )
+        title = EARLY_STOPPING_NUDGE_TITLE.format(savings=best_savings)
 
         return self._create_card(
             title=title,

ax/early_stopping/utils.py

@@ -19,6 +19,65 @@
 
 logger: Logger = get_logger(__name__)
 
+# Early stopping message constants for use in analysis and reporting
+EARLY_STOPPING_STATUS_MSG = (
+    "Throughout this experiment, {n_stopped} trials were early stopped, out "
+    "of a total of {n_ran} trials. "
+)
+
+EARLY_STOPPING_SAVINGS_TITLE = "Capacity savings due to early stopping"
+
+EARLY_STOPPING_SAVINGS_MSG = (
+    "The capacity savings (computed using {map_key}) are estimated to be "
+    "{savings:.0f}%."
+)
+
+EARLY_STOPPING_SAVINGS_TBD = (
+    "Capacity savings are not yet available. Either no trials have been early "
+    "stopped, or no trials have completed (which is required to estimate "
+    "savings). Check back once more trials are completed and/or early stopped."
+)
+
+EARLY_STOPPING_NUDGE_MSG = (
+    "This sweep uses metrics that are **compatible with early stopping**! "
+    "Using early stopping could have saved you both capacity and optimization "
+    "wall time. For example, we estimate that using early stopping on the "
+    "'{metric_name}' metric could have provided {savings:.0f}% capacity "
+    "savings, with no regression in optimization performance."
+)
+
+EARLY_STOPPING_NUDGE_TITLE = (
+    "{savings:.0f}% potential capacity savings if you turn on early stopping feature"
+)
+
+
+def format_early_stopping_savings_message(
+    n_stopped: int,
+    n_ran: int,
+    savings: float,
+) -> str:
+    """Format a message describing early stopping status and savings.
+
+    Args:
+        n_stopped: Number of trials that were early stopped.
+        n_ran: Total number of trials that ran.
+        savings: Resource savings as a fraction (0.0 to 1.0). For example, 0.11
+            indicates 11% savings.
+
+    Returns:
+        A formatted message string describing the early stopping status and
+        either the estimated savings percentage or a note that savings are
+        not yet available.
+    """
+    msg = EARLY_STOPPING_STATUS_MSG.format(n_stopped=n_stopped, n_ran=n_ran)
+
+    if savings > 0:
+        msg += EARLY_STOPPING_SAVINGS_MSG.format(map_key=MAP_KEY, savings=savings * 100)
+    else:
+        msg += EARLY_STOPPING_SAVINGS_TBD
+
+    return msg
+
 
 def _is_worse(a: Any, b: Any, minimize: bool) -> Any:
     """Determine if value `a` is worse than value `b` based on optimization direction.