Changes from review

Sarah Krebs · Sarah Krebs · commit 45547ea6325e · 2024-03-26T11:23:50.000+01:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,10 @@
 - Fix errors due to changing inputs before runselection (#64).
 - For fANOVA, remove constant hyperparameters from configspace (#9).
 
+## Documentation
+- Add How to Contribute section.
+- Expand documentation for partial dependence plugin.
+
 ## Version-Updates
 - Black version from 23.1.0 to 23.3.0
 - Mypy from 0.930 to 1.5.1
diff --git a/deepcave/plugins/hyperparameter/importances.py b/deepcave/plugins/hyperparameter/importances.py
@@ -147,7 +147,10 @@ def get_filter_layout(register: Callable) -> List[html.Div]:
             html.Div(
                 [
                     dbc.Label("Budgets"),
-                    help_button("The hyperparameters are sorted by the highest budget."),
+                    help_button(
+                        "Budget refers to the multi-fidelity budget. "
+                        "The hyperparameters are sorted by the highest budget."
+                    ),
                     dbc.Checklist(id=register("budget_ids", ["value", "options"]), inline=True),
                 ]
             ),
diff --git a/deepcave/plugins/hyperparameter/pdp.py b/deepcave/plugins/hyperparameter/pdp.py
@@ -85,9 +85,9 @@ def get_input_layout(register: Callable) -> List[dbc.Row]:
                         [
                             dbc.Label("Budget"),
                             help_button(
-                                "Budget refers to the Multi-Fidelity budget."
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest"
-                                " evaluated budget is used.\n\n"
+                                " evaluated budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if"
                                 " a time objective is used. Often, higher budget take "
                                 " longer to evaluate, which might negatively influence "
diff --git a/deepcave/plugins/hyperparameter/symbolic_explanations.py b/deepcave/plugins/hyperparameter/symbolic_explanations.py
@@ -89,9 +89,9 @@ def get_input_layout(register: Callable) -> List[Union[dbc.Row, html.Details]]:
                         [
                             dbc.Label("Budget"),
                             help_button(
-                                "Budget refers to the Multi-Fidelity budget."
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest"
-                                " evaluated budget is used.\n\n"
+                                " evaluated budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if"
                                 " a time objective is used. Often, higher budget take "
                                 " longer to evaluate, which might negatively influence "
diff --git a/deepcave/plugins/objective/configuration_cube.py b/deepcave/plugins/objective/configuration_cube.py
@@ -81,9 +81,9 @@ def get_input_layout(register: Callable) -> List[dbc.Row]:
                         [
                             dbc.Label("Budget"),
                             help_button(
-                                "Budget refers to the Multi-Fidelity budget."
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest"
-                                " evaluated budget is used.\n\n"
+                                " evaluated budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if"
                                 " a time objective is used. Often, higher budget take "
                                 " longer to evaluate, which might negatively influence "
diff --git a/deepcave/plugins/objective/cost_over_time.py b/deepcave/plugins/objective/cost_over_time.py
@@ -115,6 +115,7 @@ def get_input_layout(register: Callable) -> List[dbc.Row]:
                         [
                             dbc.Label("Budget"),
                             help_button(
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest evaluated "
                                 "budget is used."
                             ),
diff --git a/deepcave/plugins/objective/parallel_coordinates.py b/deepcave/plugins/objective/parallel_coordinates.py
@@ -74,8 +74,9 @@ def get_input_layout(register: Callable) -> List[Any]:
                         [
                             dbc.Label("Budget"),
                             help_button(
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest evaluated"
-                                " budget is used.\n\n"
+                                " budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if a time"
                                 " objective is used. Often, higher budget take longer to evaluate,"
                                 " which might negatively influence the results."
diff --git a/deepcave/plugins/objective/pareto_front.py b/deepcave/plugins/objective/pareto_front.py
@@ -127,8 +127,9 @@ def get_input_layout(register: Callable) -> List[Any]:
                 [
                     dbc.Label("Budget"),
                     help_button(
-                        "Combined budget means that the trial on the highest evaluated budget is "
-                        "used.\n\n"
+                        "Budget refers to the multi-fidelity budget. "
+                        "Combined budget means that the trial on the highest"
+                        " evaluated budget is used.  \n "
                         "Note: Selecting combined budget might be misleading if a time objective "
                         "is used. Often, higher budget take longer to evaluate, which might "
                         "negatively influence the results."
diff --git a/deepcave/plugins/summary/footprint.py b/deepcave/plugins/summary/footprint.py
@@ -74,9 +74,9 @@ def get_input_layout(register: Callable) -> List[Any]:
                         [
                             dbc.Label("Budget"),
                             help_button(
-                                "Budget refers to the Multi-Fidelity budget."
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest"
-                                " evaluated budget is used.\n\n"
+                                " evaluated budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if"
                                 " a time objective is used. Often, higher budget take "
                                 " longer to evaluate, which might negatively influence "
diff --git a/docs/how_to_contribute.rst b/docs/how_to_contribute.rst
@@ -1,7 +1,10 @@
 ==================
 How to Contribute
 ==================
-We would appreciate your help in making this project better. Whether it's a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from our community. If you think you have found a bug, or have a new feature idea, please check for `github issues <github_issues_>`_ and create a new issue if needed. For new feature ideas, open a new issue and setup a development workflow to work on the issue. If you have a correction or addition to the documentation, please submit a pull request.
+We would appreciate your help in making this project better.
+Whether it's a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from our community.
+If you think you have found a bug, or have a new feature idea, please check for `github issues <github_issues_>`_ and create a new issue if needed.
+If you'd like to work on a bug or new feature idea, or have a correction or addition to the documentation, please open a new issue and setup a development workflow to work on the issue and submit a pull request.
 
 .. _contribute_fork_project:
 
@@ -20,9 +23,10 @@ The development workflow is based on the "fork-and-pull" Git workflow. A full de
 
 .. _contribute_unit_tests:
 
-Extend Unit Tests 
+Extend Unit Tests
 ------------------
-When adding new features or making changes to the codebase, it is important to add unit tests and check for mistakes revealed by preexisting test cases. This project uses `unittest` for testing. To run the test suite, use the following command:
+When adding new features or making changes to the codebase, it is important to add unit tests and check for mistakes revealed by pre-existing test cases. This project uses `unittest` for testing.
+To run the test suite, you can use the following command:
 
 .. code-block:: bash
 
@@ -34,7 +38,8 @@ When adding new features or making changes to the codebase, it is important to a
 Update Documentation
 ---------------------
 
-The documentation is written in reStructuredText and uses Sphinx to generate HTML documentation. To build the documentation, use the following command:
+The documentation is written in reStructuredText and uses Sphinx to generate HTML documentation.
+To build the documentation, you can use the following command:
 
 .. code-block:: bash
 
@@ -45,20 +50,26 @@ The documentation is written in reStructuredText and uses Sphinx to generate HTM
 
 Pre-Commit Hook
 ---------------
-Before submitting a pull request, you should run the pre-commit hook to check for formatting errors and run the test suite. 
+Before submitting a pull request, you should run the pre-commit hook to check for formatting errors and run the test suite, e.g. via
+
+.. code-block:: bash
+
+    pre-commit run --all-files
 
 .. _contribute_pull_request:
 
 Create Pull Request
 --------------------
 
-After making changes to the codebase, you can submit a pull request. Your pull request will be reviewed by the maintainers, and if everything looks good, it will be merged. Before submitting a pull request, please make sure of the following:
+After making changes to the codebase, you can submit a pull request. Your pull request will be reviewed by the maintainers, and if everything looks good, it will be merged.
+Before submitting a pull request, please make sure of the following:
 
-- Make sure the you reference the issue in the pull request.
+- Make sure to reference the issue in the pull request.
 - The pull request should include tests for the new feature or bug fix.
 - The pull request should include documentation for the new feature or bug fix.
 - All tests should pass.
-- Make sure to select the correct source and destination repositories and according branches.
+- Make sure to select the correct source and destination repositories and according branches, usually your fork to the main repository's `development` branch.
+- Add the change to the CHANGELOG.md file.
 - The pull request should include a description of the change and the reason for the change.
 
 
diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst
@@ -23,7 +23,7 @@ This plugin is capable of answering following questions:
 * Is the configuration space as expected?
 * How many trials have been successful/unsuccessful?
 * Why did certain trials crash?
-* On which Multi Fidelity budgets were the configurations evaluated?
+* On which multi-fidelity budgets were the configurations evaluated?
 
 
 Quick Information
@@ -37,8 +37,8 @@ Meta
 ----
 Next we show some meta information regarding your runs. Notably, you can see information such as the
 possible **budget** allocations that were considered during the run. The **budget** is the amount of
-resources that were allocated to a configuration as it was beeing evaluated using a Multi-Fidelity 
-Approach.
+resources that were allocated to a configuration as it was being evaluated using a multi-fidelity
+approach.
 
 
 Objectives
@@ -50,10 +50,10 @@ information about it's name and the bounds they were in.
 Statuses
 --------
 To get a nice overview of the optimization run, we provide a barplot that details how the end status
-of the many trials that occured. You can also view these as a heatmap or get more information from
+of the many trials that occurred. You can also view these as a heatmap or get more information from
 the **Details** tab.
 
-The barplot shows which trial recieved which status and under which budget allocation. These are
+The barplot shows which trial received which status and under which budget allocation. These are
 grouped according to the status and coloured based on the status it exited with.
 
 To see details on how these configurations faired as they progressed in budget, you can refer to
@@ -64,9 +64,9 @@ to try. However if you faced many crashes at high budgets for example, you'll se
 from the highest budget onwards. In general, you can use this to understand how often your optimizer
 balances out between low budget evaluations and full budget evaluations.
 
-In the **Details** tab, you'll recieve an overview of configurations that were not successful.
-Here you'll get some breif information about the config id number, the budget it was evaluated under
-and the status it recieved. If there is an associated error we can provide, such as when it recieved
+In the **Details** tab, you'll receive an overview of configurations that were not successful.
+Here you'll get some brief information about the config id number, the budget it was evaluated under
+and the status it received. If there is an associated error we can provide, such as when it received
 the **CRASHED** status, we will display it in the error column.
 
 
diff --git a/docs/plugins/partial_dependencies.rst b/docs/plugins/partial_dependencies.rst
@@ -1,31 +1,38 @@
 Partial Dependencies
 ====================
 
-The quality of a machine learning algorithm is highly dependent on the choice of hyperparameters. PdP plots aim to visualize how chaning one or two hyperparameters affects the objective function. To reduce comptutational complexity the plugin utilizes a random forest surrogate model.
+The quality of a machine learning algorithm is highly dependent on the choice of hyperparameters.
+Partial dependence plots (PDP) aim to visualize how changing one or two hyperparameters affects the objective function.
+The PDP is calculated using a random forest surrogate model trained on the run data, i.e. the evaluated hyperparameter configurations and corresponding objective values.
 
-* In the one hyperparameter case, random samples or the selected hyperparameters are combined with each combination of the othre hyperparameters present in the dataset resulting in a multitude of ice curves. The resulting ice curves are than averaged to a PD plot. The hyperparameter is plotted along the x axis, and the respective objective function value along the y-axis
+* In the **one hyperparameter case**, random samples or the selected hyperparameters are combined with each combination of the other hyperparameters present in the dataset, resulting in a multitude of individual conditional expectation (ICE) curves.
+  The resulting ICE curves are than averaged to a PDP.
+  The hyperparameter is plotted along the x axis, and the respective objective function value along the y-axis.
 
-* The two hyperparameter case works as the one hyperparameter case, but the then two dimensional ice-curves are disregarded in the plot. The resulting plot has one target hyperparameter plotted along the x-axis and the other hyperparameter plotted along the y-axis. The color in the plot indicates the respective objective function value.
+* The **two hyperparameter case** works as the one hyperparameter case, but the then two dimensional ICE-curves are disregarded in the plot.
+  The resulting plot has one target hyperparameter plotted along the x-axis and the other hyperparameter plotted along the y-axis.
+  The color in the heatmap plot indicates the respective objective function value.
 
 
-This plugin is capable of answering following questions:
+This plugin is capable of answering the following questions:
 
 * How does the objective change with respect to one or two hyperparameters? For example, does the
   accuracy increase if the learning rate decreases?
 * Do multiple trials show similar behavior?
+
 Options
 -------
-* **Objective**: Here you can select the objective you wish to analyze for. In the one hyperparameter case the objective is plotted on the y-axis, in the two hyperparameter case it is represented by the color.
+* **Objective**: Here you can select the objective you wish to analyze. In the one hyperparameter case the objective is plotted on the y-axis, in the two hyperparameter case it is represented by the color.
 
-* **Budget**: Here you can select the budget used to compute ICE. TODO 
+* **Budget**: Here you can select the multi-fidelity budget to be used. The plugin will only consider trials evaluated on the selected budget.
 
 * **Hyperparameter 1**: Here you can select the first hyperparameter. It is plotted along the x-axis.
 
 * **Hyperparameter 2**: Here you can optionally select the second hyperparameter.
 
-* **Show Confidence**: Selection for whether or not to show certainty estimates (based on the difference of ice curves).
+* **Show Confidence**: Selection for whether or not to show confidence estimates (based on the surrogate model's uncertainty estimate).
 
-* **Show ICE Curves**: Selection for whether or not to to show the ICE curves used during computation
+* **Show ICE Curves**: Selection for whether or not to to show the ICE curves used during computation (only for the one hyperparameter case).
 
 
 .. image:: ../images/plugins/partial_dependencies.png
diff --git a/docs/sg_execution_times.rst b/docs/sg_execution_times.rst

Original file line number	Diff line number	Diff line change
`@@ -147,7 +147,10 @@ def get_filter_layout(register: Callable) -> List[html.Div]:`
`147`	`147`	`html.Div(`
`148`	`148`	`[`
`149`	`149`	`dbc.Label("Budgets"),`
`150`		`- help_button("The hyperparameters are sorted by the highest budget."),`
	`150`	`+ help_button(`
	`151`	`+ "Budget refers to the multi-fidelity budget. "`
	`152`	`+ "The hyperparameters are sorted by the highest budget."`
	`153`	`+ ),`
`151`	`154`	`dbc.Checklist(id=register("budget_ids", ["value", "options"]), inline=True),`
`152`	`155`	`]`
`153`	`156`	`),`
Original file line number	Diff line number	Diff line change
`@@ -115,6 +115,7 @@ def get_input_layout(register: Callable) -> List[dbc.Row]:`
`115`	`115`	`[`
`116`	`116`	`dbc.Label("Budget"),`
`117`	`117`	`help_button(`
	`118`	`+ "Budget refers to the multi-fidelity budget. "`
`118`	`119`	`"Combined budget means that the trial on the highest evaluated "`
`119`	`120`	`"budget is used."`
`120`	`121`	`),`