(5/6) Port helpers to OSS for the new Complexity Rating Healthcheck - Renaming and rewording for OSS (#4653)

Summary: Pull Request resolved: #4653

Reviewed By: bernardbeckerman

Differential Revision: D88902863

Author: shrutipatel31

ax/utils/common/complexity_utils.py

@@ -17,8 +17,10 @@
 
 WHEELHOUSE_TIER_MESSAGE = """This experiment is in tier 'Wheelhouse'.
 
-Experiments belonging to this tier should not run into any problems! If an issue \
-does occur, please post to our github issues page.
+Experiments in the 'Wheelhouse' tier use standard features that are \
+thoroughly tested and should work reliably. If you encounter any issues, \
+they are typically easy to diagnose and resolve. Otherwise, please post \
+to our github issues page for help.
 """
 
 ADVANCED_TIER_MESSAGE = """This experiment is in tier 'Advanced'.
@@ -33,25 +35,26 @@
 UNSUPPORTED_TIER_MESSAGE = """This experiment is in tier 'Unsupported'.
 
 You are pushing Ax beyond its limits. Please post to our github issues page for help \
-in improving/simplifying your configuration to conform to a more \
-well-supported usage tier if possible.
+with improving/simplifying your configuration to conform to a more \
+well-supported usage tier if possible. We strongly recommend simplifying your \
+configuration to fall within a supported tier.
 """
 
-WIKI_TIER_MESSAGE = "https://ax.dev/docs/why-ax"
-
 UNKNOWN_TIER_MESSAGE = """Failed to determine the tier of this experiment.
 
 Please post on our github issues page or reach out to the Ax user group \
 to determine the support tier of your workflow.
+This may indicate an issue with your experiment configuration or an internal \
+error during tier classification. Please review your configuration for any \
+unusual settings, or consult our github issues page documentation for guidance.
 """
 
-NOT_STANDARD_API_MESSAGE = (
-    "The experiment summary indicates that this workflow is not using a standard \
-    API (`uses_standard_api=False`). Tier classification works best when the full \
-    experiment configuration is known upfront. If you are building a tool on top \
-    of this function, ensure that `uses_standard_api` is set to `True` in the \
-    `OptimizationSummary` when your tool uses a standard API."
-)
+NOT_STANDARD_API_MESSAGE = """The experiment summary indicates that this workflow \
+is not using a standard API (`uses_standard_api=False`). Tier classification works \
+best when the full experiment configuration is known upfront. If you are building a \
+tool on top of this function, ensure that `uses_standard_api` is set to `True` in the \
+`OptimizationSummary` when your tool uses a standard API.
+"""
 
 
 @dataclass(frozen=True)
@@ -68,11 +71,11 @@ class OptimizationSummary:
         num_outcome_constraints: Number of outcome constraints.
         uses_early_stopping: Whether early stopping is enabled.
         uses_global_stopping: Whether global stopping is enabled.
-        all_inputs_are_configs: Whether all inputs are high-level configs
+        uses_standard_api: Whether all inputs are high-level configs
             (as opposed to low-level Ax abstractions).
 
     Optional Keys:
-        max_trials: Maximum number of trials (required if all_inputs_are_configs
+        max_trials: Maximum number of trials (required if uses_standard_api
             is True).
         tolerated_trial_failure_rate: Maximum tolerated trial failure rate
             (should be <= 0.9).
@@ -94,7 +97,7 @@ class OptimizationSummary:
     num_outcome_constraints: int
     uses_early_stopping: bool
     uses_global_stopping: bool
-    all_inputs_are_configs: bool
+    uses_standard_api: bool
     # Optional keys
     max_trials: int | None = None
     tolerated_trial_failure_rate: float | None = None
@@ -158,6 +161,11 @@ def summarize_ax_optimization_complexity(
             uses_merge_multiple_curves = True
             break
 
+    # Support both new key and old key for backward compatibility
+    uses_standard_api = tier_metadata.get("uses_standard_api")
+    if uses_standard_api is None:
+        uses_standard_api = tier_metadata.get("all_inputs_are_configs", False)
+
     return OptimizationSummary(
         max_trials=max_trials,
         num_params=num_params,
@@ -170,7 +178,7 @@ def summarize_ax_optimization_complexity(
         uses_early_stopping=uses_early_stopping,
         uses_global_stopping=uses_global_stopping,
         uses_merge_multiple_curves=uses_merge_multiple_curves,
-        all_inputs_are_configs=tier_metadata.get("all_inputs_are_configs", False),
+        uses_standard_api=uses_standard_api,
         tolerated_trial_failure_rate=options.tolerated_trial_failure_rate,
         max_pending_trials=options.max_pending_trials,
         min_failed_trials_for_failure_rate_check=(
@@ -331,7 +339,7 @@ def _check_if_is_in_wheelhouse_other_settings(
     """
     is_in_wheelhouse, is_supported = True, True
     max_trials = optimization_summary.max_trials
-    if not optimization_summary.all_inputs_are_configs:
+    if not optimization_summary.uses_standard_api:
         is_in_wheelhouse, is_supported = False, False
         why_not_supported.append(NOT_STANDARD_API_MESSAGE)
     elif max_trials is None:
@@ -417,7 +425,7 @@ def check_if_in_wheelhouse(
               num_categorical_6_inf, num_parameter_constraints
             - Optimization config: num_objectives, num_outcome_constraints
             - Other settings: max_trials, uses_early_stopping, uses_global_stopping,
-              all_inputs_are_configs, tolerated_trial_failure_rate, max_pending_trials,
+              uses_standard_api, tolerated_trial_failure_rate, max_pending_trials,
               min_failed_trials_for_failure_rate_check, non_default_advanced_options,
               uses_merge_multiple_curves
 
@@ -516,9 +524,4 @@ def format_tier_message(
                 f"\n{why_msg}\n"
             )
             msg += why_msg
-
-    msg += (
-        "\n\nFor more information about the definition of each tier and what "
-        f"level of support you can expect: {WIKI_TIER_MESSAGE}"
-    )
     return msg

ax/utils/common/tests/test_complexity_utils.py

@@ -86,7 +86,7 @@ def test_tier_metadata_extraction(self) -> None:
         test_cases = [
             (
                 "with_values",
-                {"user_supplied_max_trials": 50, "all_inputs_are_configs": True},
+                {"user_supplied_max_trials": 50, "uses_standard_api": True},
                 50,
                 True,
             ),
@@ -114,7 +114,7 @@ def test_tier_metadata_extraction(self) -> None:
 
                 # THEN the summary should reflect tier metadata values
                 self.assertEqual(summary.max_trials, expected_max_trials)
-                self.assertEqual(summary.all_inputs_are_configs, expected_all_configs)
+                self.assertEqual(summary.uses_standard_api, expected_all_configs)
 
     def test_orchestrator_options_extraction(self) -> None:
         # GIVEN custom orchestrator options
@@ -226,7 +226,7 @@ def test_unknown_tier_raises_error(self) -> None:
             )
 
 
-def get_experiment_summary(
+def get_optimization_summary(
     max_trials: int | None = 100,
     num_params: int = 10,
     num_binary: int = 0,
@@ -237,7 +237,7 @@ def get_experiment_summary(
     num_outcome_constraints: int = 0,
     uses_early_stopping: bool = False,
     uses_global_stopping: bool = False,
-    all_inputs_are_configs: bool = True,
+    uses_standard_api: bool = True,
     tolerated_trial_failure_rate: float | None = 0.5,
     max_pending_trials: int | None = 5,
     min_failed_trials_for_failure_rate_check: int | None = 5,
@@ -256,7 +256,7 @@ def get_experiment_summary(
         num_outcome_constraints=num_outcome_constraints,
         uses_early_stopping=uses_early_stopping,
         uses_global_stopping=uses_global_stopping,
-        all_inputs_are_configs=all_inputs_are_configs,
+        uses_standard_api=uses_standard_api,
         tolerated_trial_failure_rate=tolerated_trial_failure_rate,
         max_pending_trials=max_pending_trials,
         min_failed_trials_for_failure_rate_check=(
@@ -272,7 +272,7 @@ class TestCheckIfInWheelhouse(TestCase):
 
     def setUp(self) -> None:
         super().setUp()
-        self.base_summary = get_experiment_summary()
+        self.base_summary = get_optimization_summary()
 
     def test_wheelhouse_tier_for_simple_experiment(self) -> None:
         """Test that a simple experiment is classified as Wheelhouse tier."""
@@ -287,28 +287,28 @@ def test_wheelhouse_tier_for_simple_experiment(self) -> None:
     def test_advanced_tier_conditions(self) -> None:
         """Test conditions that result in Advanced tier."""
         test_cases: list[tuple[OptimizationSummary, str]] = [
-            (get_experiment_summary(max_trials=250), "250 total trials"),
-            (get_experiment_summary(num_params=60), "60 tunable parameter(s)"),
-            (get_experiment_summary(num_binary=75), "75 binary tunable parameter(s)"),
+            (get_optimization_summary(max_trials=250), "250 total trials"),
+            (get_optimization_summary(num_params=60), "60 tunable parameter(s)"),
+            (get_optimization_summary(num_binary=75), "75 binary tunable parameter(s)"),
             (
-                get_experiment_summary(num_categorical_3_5=1),
+                get_optimization_summary(num_categorical_3_5=1),
                 "1 unordered choice parameter(s)",
             ),
             (
-                get_experiment_summary(num_parameter_constraints=4),
+                get_optimization_summary(num_parameter_constraints=4),
                 "4 parameter constraints",
             ),
-            (get_experiment_summary(num_objectives=3), "3 objectives"),
+            (get_optimization_summary(num_objectives=3), "3 objectives"),
             (
-                get_experiment_summary(num_outcome_constraints=3),
+                get_optimization_summary(num_outcome_constraints=3),
                 "3 outcome constraints",
             ),
             (
-                get_experiment_summary(uses_early_stopping=True),
+                get_optimization_summary(uses_early_stopping=True),
                 "Early stopping is enabled",
             ),
             (
-                get_experiment_summary(uses_global_stopping=True),
+                get_optimization_summary(uses_global_stopping=True),
                 "Global stopping is enabled",
             ),
         ]
@@ -327,44 +327,47 @@ def test_advanced_tier_conditions(self) -> None:
     def test_unsupported_tier_conditions(self) -> None:
         """Test conditions that result in Unsupported tier."""
         test_cases: list[tuple[OptimizationSummary, str]] = [
-            (get_experiment_summary(max_trials=510), "510 total trials"),
-            (get_experiment_summary(num_params=201), "201 tunable parameter(s)"),
-            (get_experiment_summary(num_binary=101), "101 binary tunable parameter(s)"),
+            (get_optimization_summary(max_trials=510), "510 total trials"),
+            (get_optimization_summary(num_params=201), "201 tunable parameter(s)"),
             (
-                get_experiment_summary(num_categorical_3_5=6),
+                get_optimization_summary(num_binary=101),
+                "101 binary tunable parameter(s)",
+            ),
+            (
+                get_optimization_summary(num_categorical_3_5=6),
                 "unordered choice parameters with more than 3 options",
             ),
             (
-                get_experiment_summary(num_categorical_6_inf=2),
+                get_optimization_summary(num_categorical_6_inf=2),
                 "unordered choice parameters with more than 5 options",
             ),
             (
-                get_experiment_summary(num_parameter_constraints=6),
+                get_optimization_summary(num_parameter_constraints=6),
                 "6 parameter constraints",
             ),
-            (get_experiment_summary(num_objectives=5), "5 objectives"),
+            (get_optimization_summary(num_objectives=5), "5 objectives"),
             (
-                get_experiment_summary(num_outcome_constraints=6),
+                get_optimization_summary(num_outcome_constraints=6),
                 "6 outcome constraints",
             ),
             (
-                get_experiment_summary(all_inputs_are_configs=False),
+                get_optimization_summary(uses_standard_api=False),
                 "uses_standard_api=False",
             ),
             (
-                get_experiment_summary(tolerated_trial_failure_rate=0.99),
+                get_optimization_summary(tolerated_trial_failure_rate=0.99),
                 "tolerated_trial_failure_rate=0.99",
             ),
             (
-                get_experiment_summary(non_default_advanced_options=True),
+                get_optimization_summary(non_default_advanced_options=True),
                 "Non-default advanced_options",
             ),
             (
-                get_experiment_summary(uses_merge_multiple_curves=True),
+                get_optimization_summary(uses_merge_multiple_curves=True),
                 "merge_multiple_curves=True",
             ),
             (
-                get_experiment_summary(
+                get_optimization_summary(
                     max_pending_trials=3, min_failed_trials_for_failure_rate_check=7
                 ),
                 "min_failed_trials_for_failure_rate_check=7",
@@ -380,8 +383,8 @@ def test_unsupported_tier_conditions(self) -> None:
                 self.assertIn(expected_msg, why_not_supported[0])
 
     def test_max_trials_none_raises(self) -> None:
-        """Test max_trials=None with all_inputs_are_configs=True raises error."""
-        summary = get_experiment_summary(all_inputs_are_configs=True, max_trials=None)
+        """Test max_trials=None with uses_standard_api=True raises error."""
+        summary = get_optimization_summary(uses_standard_api=True, max_trials=None)
 
         with self.assertRaisesRegex(UserInputError, "`max_trials` should not be None!"):
             check_if_in_wheelhouse(summary)