Merge pull request #123 from automl/bug/warning_message_runs_not_comparable

Bug/warning message runs not comparable

Author: sarah-segel

.flake8

@@ -7,6 +7,6 @@ extend-exclude =
     build
 extend-ignore =
     # No whitespace before ':' in [x : y]
-    E203
+    E203,
     # No lambdas — too strict
-    E731
+    E731,

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Plugins
 - Add symbolic explanations plugin (#46).
+- It is now possible to view multiple unequal runs at once in Cost over Time and Pareto (#93).
+- Runs with unequal objectives cannot be displayed together.
+- Added an enum for displaying according warning messages.
 
 ## Enhancements
 - Fix lower bounds of dependency versions.
@@ -19,6 +22,9 @@
 - Reset inputs to fix error when subsequently selecting runs with different configspaces, objectives or budgets (#106).
 - Fix errors due to changing inputs before runselection (#64).
 - For fANOVA, remove constant hyperparameters from configspace (#9).
+- When getting budget, objectives etc from multiple runs in Cost over Time and Pareto Front:
+    - Instead of taking the first run as comparative value,
+    - take the one with the lowest budget, else the index for the budgets could be out of bounds.
 
 ## Documentation
 - Add How to Contribute section.
@@ -68,7 +74,7 @@
 - SMAC 2.0
 
 ## Dependencies
-- Remove SMAC dependency by adding required function directly
+- Remove SMAC dependency by adding required function directly.
 
 # Version 1.0.1
 

deepcave/plugins/__init__.py

@@ -917,7 +917,6 @@ def __call__(self, render_button: bool = False) -> List[Component]:
             ]
         else:
             components += [html.H1(self.name)]
-
         try:
             self.check_runs_compatibility(self.all_runs)
         except NotMergeableError as message:

deepcave/plugins/dynamic.py

@@ -95,6 +95,7 @@ def plugin_output_update(_: Any, *inputs_list: str) -> Any:
             runs = self.get_selected_runs(inputs)
 
             raw_outputs = {}
+            rc.clear()
             for run in runs:
                 run_outputs = rc.get(run, self.id, inputs_key)
                 if run_outputs is None:

deepcave/plugins/objective/cost_over_time.py

@@ -19,9 +19,11 @@
 from dash import dcc, html
 from dash.exceptions import PreventUpdate
 
+from deepcave import notification
 from deepcave.config import Config
 from deepcave.plugins.dynamic import DynamicPlugin
 from deepcave.runs import AbstractRun, check_equality
+from deepcave.runs.exceptions import NotMergeableError, RunInequality
 from deepcave.utils.layout import get_select_options, help_button
 from deepcave.utils.styled_plotty import (
     get_color,
@@ -56,6 +58,9 @@ def check_runs_compatibility(self, runs: List[AbstractRun]) -> None:
         Since this function is called before the layout is created,
         it can be also used to set common values for the plugin.
 
+        If the runs are not mergeable, they still should be displayed
+        but with a corresponding warning message.
+
         Parameters
         ----------
         runs : List[AbstractRun]
@@ -69,18 +74,36 @@ def check_runs_compatibility(self, runs: List[AbstractRun]) -> None:
             If the budgets of the runs are not equal.
             If the objective of the runs are not equal.
         """
-        check_equality(runs, objectives=True, budgets=True)
+        try:
+            check_equality(runs, objectives=True, budgets=True)
+        except NotMergeableError as e:
+            run_inequality = e.args[1]
+            if run_inequality == RunInequality.INEQ_BUDGET:
+                notification.update("The budgets of the runs are not equal.", color="warning")
+            elif run_inequality == RunInequality.INEQ_CONFIGSPACE:
+                notification.update(
+                    "The configuration spaces of the runs are not equal.", color="warning"
+                )
+            elif run_inequality == RunInequality.INEQ_META:
+                notification.update("The meta data of the runs is not equal.", color="warning")
+            elif run_inequality == RunInequality.INEQ_OBJECTIVE:
+                raise NotMergeableError("The objectives of the selected runs cannot be merged.")
 
         # Set some attributes here
-        run = runs[0]
-
-        objective_names = run.get_objective_names()
-        objective_ids = run.get_objective_ids()
-        self.objective_options = get_select_options(objective_names, objective_ids)
-
-        budgets = run.get_budgets(human=True)
-        budget_ids = run.get_budget_ids()
-        self.budget_options = get_select_options(budgets, budget_ids)
+        # It is necessary to get the run with the smallest budget and objective options
+        # as first comparative value, else there is gonna be an index problem
+        objective_options = []
+        budget_options = []
+        for run in runs:
+            objective_names = run.get_objective_names()
+            objective_ids = run.get_objective_ids()
+            objective_options.append(get_select_options(objective_names, objective_ids))
+
+            budgets = run.get_budgets(human=True)
+            budget_ids = run.get_budget_ids()
+            budget_options.append(get_select_options(budgets, budget_ids))
+        self.objective_options = min(objective_options, key=len)
+        self.budget_options = min(budget_options, key=len)
 
     @staticmethod
     def get_input_layout(register: Callable) -> List[dbc.Row]:

deepcave/plugins/objective/pareto_front.py

@@ -17,9 +17,11 @@
 import plotly.graph_objs as go
 from dash import dcc, html
 
+from deepcave import notification
 from deepcave.config import Config
 from deepcave.plugins.dynamic import DynamicPlugin
 from deepcave.runs import AbstractRun, Status, check_equality
+from deepcave.runs.exceptions import NotMergeableError, RunInequality
 from deepcave.utils.layout import get_select_options, help_button
 from deepcave.utils.styled_plot import plt
 from deepcave.utils.styled_plotty import (
@@ -55,6 +57,9 @@ def check_runs_compatibility(self, runs: List[AbstractRun]) -> None:
         Since this function is called before the layout is created,
         it can be also used to set common values for the plugin.
 
+        If the runs are not mergeable, they still should be displayed
+        but with a corresponding warning message
+
         Parameters
         ----------
         runs : List[AbstractRun]
@@ -68,18 +73,36 @@ def check_runs_compatibility(self, runs: List[AbstractRun]) -> None:
             If the budgets of the runs are not equal.
             If the objective of the runs are not equal.
         """
-        check_equality(runs, objectives=True, budgets=True)
+        try:
+            check_equality(runs, objectives=True, budgets=True)
+        except NotMergeableError as e:
+            run_inequality = e.args[1]
+            if run_inequality == RunInequality.INEQ_BUDGET:
+                notification.update("The budgets of the runs are not equal.", color="warning")
+            elif run_inequality == RunInequality.INEQ_CONFIGSPACE:
+                notification.update(
+                    "The configuration spaces of the runs are not equal.", color="warning"
+                )
+            elif run_inequality == RunInequality.INEQ_META:
+                notification.update("The meta data of the runs is not equal.", color="warning")
+            elif run_inequality == RunInequality.INEQ_OBJECTIVE:
+                raise NotMergeableError("The objectives of the selected runs cannot be merged.")
 
         # Set some attributes here
-        run = runs[0]
-
-        objective_names = run.get_objective_names()
-        objective_ids = run.get_objective_ids()
-        self.objective_options = get_select_options(objective_names, objective_ids)
-
-        budgets = run.get_budgets(human=True)
-        budget_ids = run.get_budget_ids()
-        self.budget_options = get_select_options(budgets, budget_ids)
+        # It is necessary to get the run with the smallest budget and objective options
+        # as first comparative value, else there is gonna be an index problem
+        objective_options = []
+        budget_options = []
+        for run in runs:
+            objective_names = run.get_objective_names()
+            objective_ids = run.get_objective_ids()
+            objective_options.append(get_select_options(objective_names, objective_ids))
+
+            budgets = run.get_budgets(human=True)
+            budget_ids = run.get_budget_ids()
+            budget_options.append(get_select_options(budgets, budget_ids))
+        self.objective_options = min(objective_options, key=len)
+        self.budget_options = min(budget_options, key=len)
 
     @staticmethod
     def get_input_layout(register: Callable) -> List[Any]:

deepcave/runs/__init__.py

@@ -32,7 +32,7 @@
     CONSTANT_VALUE,
     NAN_VALUE,
 )
-from deepcave.runs.exceptions import NotMergeableError
+from deepcave.runs.exceptions import NotMergeableError, RunInequality
 from deepcave.runs.objective import Objective
 from deepcave.runs.status import Status
 from deepcave.runs.trial import Trial
@@ -1236,68 +1236,79 @@ def check_equality(
     if len(runs) == 0:
         return result
 
-    # Check meta
-    if meta:
-        ignore = ["objectives", "budgets", "wallclock_limit"]
-
-        m1 = runs[0].get_meta()
+    # Check if objectives are mergeable
+    if objectives:
+        o1 = None
         for run in runs:
-            m2 = run.get_meta()
-
-            for k, v in m1.items():
-                # Don't check on objectives or budgets
-                if k in ignore:
-                    continue
+            o2 = run.get_objectives()
 
-                if k not in m2 or m2[k] != v:
-                    raise NotMergeableError("Meta data of runs are not equal.")
+            if o1 is None:
+                o1 = o2
+                continue
 
-        result["meta"] = m1
+            if len(o1) != len(o2):
+                raise NotMergeableError(
+                    "Objectives of runs are not equal.", RunInequality.INEQ_OBJECTIVE
+                )
 
-    # Make sure the same configspace is used
-    # Otherwise it does not make sense to merge
-    # the histories
-    if configspace:
-        cs1 = runs[0].configspace
-        for run in runs:
-            cs2 = run.configspace
-            if cs1 != cs2:
-                raise NotMergeableError("Configspace of runs are not equal.")
+            for o1_, o2_ in zip(o1, o2):
+                try:
+                    o1_.merge(o2_)
+                except NotMergeableError:
+                    raise NotMergeableError(
+                        "Objectives of runs are not equal.", RunInequality.INEQ_OBJECTIVE
+                    )
 
-        result["configspace"] = cs1
+        assert o1 is not None
+        serialized_objectives = [o.to_json() for o in o1]
+        result["objectives"] = serialized_objectives
+        if meta:
+            result["meta"]["objectives"] = serialized_objectives
 
     # Also check if budgets are the same
     if budgets:
         b1 = runs[0].get_budgets(include_combined=False)
         for run in runs:
             b2 = run.get_budgets(include_combined=False)
             if b1 != b2:
-                raise NotMergeableError("Budgets of runs are not equal.")
+                raise NotMergeableError("Budgets of runs are not equal.", RunInequality.INEQ_BUDGET)
 
         result["budgets"] = b1
         if meta:
             result["meta"]["budgets"] = b1
 
-    # And if objectives are the same
-    if objectives:
-        o1 = None
+    # Make sure the same configspace is used
+    # Otherwise it does not make sense to merge
+    # the histories
+    if configspace:
+        cs1 = runs[0].configspace
         for run in runs:
-            o2 = run.get_objectives()
+            cs2 = run.configspace
+            if cs1 != cs2:
+                raise NotMergeableError(
+                    "Configspace of runs are not equal.", RunInequality.INEQ_CONFIGSPACE
+                )
 
-            if o1 is None:
-                o1 = o2
-                continue
+        result["configspace"] = cs1
 
-            if len(o1) != len(o2):
-                raise NotMergeableError("Objectives of runs are not equal.")
+    # Check meta
+    if meta:
+        ignore = ["objectives", "budgets", "wallclock_limit"]
 
-            for o1_, o2_ in zip(o1, o2):
-                o1_.merge(o2_)
+        m1 = runs[0].get_meta()
+        for run in runs:
+            m2 = run.get_meta()
 
-        assert o1 is not None
-        serialized_objectives = [o.to_json() for o in o1]
-        result["objectives"] = serialized_objectives
-        if meta:
-            result["meta"]["objectives"] = serialized_objectives
+            for k, v in m1.items():
+                # Don't check on objectives or budgets
+                if k in ignore:
+                    continue
+
+                if k not in m2 or m2[k] != v:
+                    raise NotMergeableError(
+                        "Meta data of runs are not equal.", RunInequality.INEQ_META
+                    )
+
+        result["meta"] = m1
 
     return result

deepcave/runs/exceptions.py

@@ -12,6 +12,8 @@
     - NotMergeableError: Raised if two or more runs are not mergeable.
 """
 
+from enum import Enum
+
 
 class NotValidRunError(Exception):
     """Raised if directory is not a valid run."""
@@ -23,3 +25,12 @@ class NotMergeableError(Exception):
     """Raised if two or more runs are not mergeable."""
 
     pass
+
+
+class RunInequality(Enum):
+    """Check why runs were not compatible."""
+
+    INEQ_META = 1
+    INEQ_OBJECTIVE = 2
+    INEQ_BUDGET = 3
+    INEQ_CONFIGSPACE = 4

docs/plugins/cost_over_time.rst

@@ -9,9 +9,9 @@ Since multiple runs are supported, you directly see which run performs best to w
 If you decide to display groups (which are combined runs), you will see the mean and standard
 deviation too.
 
-.. note:: 
-    The configuration spaces of the selected runs have to be equal. Otherwise, a good comparison
-    is not possible.
+.. note::
+    The configuration spaces of the selected runs should be equal. Otherwise, a good comparison
+    is not possible. They can, however, still be displayed in the same graph.
 
 This plugin is capable of answering following questions:
 

docs/plugins/pareto_front.rst

@@ -9,7 +9,7 @@ configurations for two given objectives.
 
 .. note::
     You can enable or disable specific runs if you click on the name right to the plot.
-    If you click on a configuration you a redirected to the configuration plugin to see 
+    If you click on a configuration you are redirected to the configuration plugin to see 
     the configuration in detail.
 
 This plugin is capable of answering following questions: