Merge pull request #131 from automl/docs/documentation-review

sarah-segel · web-flow · commit 25fb549bb1df · 2024-03-26T11:33:29.000+01:00
Docs/documentation review
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,10 @@
 - Fix errors due to changing inputs before runselection (#64).
 - For fANOVA, remove constant hyperparameters from configspace (#9).
 
+## Documentation
+- Add How to Contribute section.
+- Expand documentation for partial dependence plugin.
+
 ## Version-Updates
 - Black version from 23.1.0 to 23.3.0
 - Mypy from 0.930 to 1.5.1
diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ Following features are provided:
 - A large collection of plugins to explore multiple areas like performance, hyperparameter and
 budget analysis.
 - Save your runs using DeepCAVE's native recorder.
-- Support for many optimizers using converter (e.g., DeepCAVE, SMAC and BOHB).
+- Support for many optimizers using converters (e.g., DeepCAVE, SMAC and BOHB).
 - Select runs directly from a working directory in the interface.
 - Select groups of runs for combined analysis.
 - Modularized plugin structure with access to selected runs/groups to provide maximal flexibility.
@@ -43,6 +43,7 @@ pip install DeepCAVE
 If you want to contribute to DeepCAVE use the following steps instead:
 ```bash
 git clone https://github.com/automl/DeepCAVE.git
+cd DeepCAVE
 conda create -n DeepCAVE python=3.9
 conda activate DeepCAVE
 conda install -c anaconda swig
diff --git a/deepcave/plugins/hyperparameter/importances.py b/deepcave/plugins/hyperparameter/importances.py
@@ -147,7 +147,10 @@ def get_filter_layout(register: Callable) -> List[html.Div]:
             html.Div(
                 [
                     dbc.Label("Budgets"),
-                    help_button("The hyperparameters are sorted by the highest budget."),
+                    help_button(
+                        "Budget refers to the multi-fidelity budget. "
+                        "The hyperparameters are sorted by the highest budget."
+                    ),
                     dbc.Checklist(id=register("budget_ids", ["value", "options"]), inline=True),
                 ]
             ),
diff --git a/deepcave/plugins/hyperparameter/pdp.py b/deepcave/plugins/hyperparameter/pdp.py
@@ -85,8 +85,9 @@ def get_input_layout(register: Callable) -> List[dbc.Row]:
                         [
                             dbc.Label("Budget"),
                             help_button(
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest"
-                                " evaluated budget is used.\n\n"
+                                " evaluated budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if"
                                 " a time objective is used. Often, higher budget take "
                                 " longer to evaluate, which might negatively influence "
diff --git a/deepcave/plugins/hyperparameter/symbolic_explanations.py b/deepcave/plugins/hyperparameter/symbolic_explanations.py
@@ -89,8 +89,9 @@ def get_input_layout(register: Callable) -> List[Union[dbc.Row, html.Details]]:
                         [
                             dbc.Label("Budget"),
                             help_button(
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest"
-                                " evaluated budget is used.\n\n"
+                                " evaluated budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if"
                                 " a time objective is used. Often, higher budget take "
                                 " longer to evaluate, which might negatively influence "
diff --git a/deepcave/plugins/objective/configuration_cube.py b/deepcave/plugins/objective/configuration_cube.py
@@ -81,8 +81,9 @@ def get_input_layout(register: Callable) -> List[dbc.Row]:
                         [
                             dbc.Label("Budget"),
                             help_button(
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest"
-                                " evaluated budget is used.\n\n"
+                                " evaluated budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if"
                                 " a time objective is used. Often, higher budget take "
                                 " longer to evaluate, which might negatively influence "
diff --git a/deepcave/plugins/objective/cost_over_time.py b/deepcave/plugins/objective/cost_over_time.py
@@ -115,6 +115,7 @@ def get_input_layout(register: Callable) -> List[dbc.Row]:
                         [
                             dbc.Label("Budget"),
                             help_button(
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest evaluated "
                                 "budget is used."
                             ),
diff --git a/deepcave/plugins/objective/parallel_coordinates.py b/deepcave/plugins/objective/parallel_coordinates.py
@@ -74,8 +74,9 @@ def get_input_layout(register: Callable) -> List[Any]:
                         [
                             dbc.Label("Budget"),
                             help_button(
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest evaluated"
-                                " budget is used.\n\n"
+                                " budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if a time"
                                 " objective is used. Often, higher budget take longer to evaluate,"
                                 " which might negatively influence the results."
diff --git a/deepcave/plugins/objective/pareto_front.py b/deepcave/plugins/objective/pareto_front.py
@@ -127,8 +127,9 @@ def get_input_layout(register: Callable) -> List[Any]:
                 [
                     dbc.Label("Budget"),
                     help_button(
-                        "Combined budget means that the trial on the highest evaluated budget is "
-                        "used.\n\n"
+                        "Budget refers to the multi-fidelity budget. "
+                        "Combined budget means that the trial on the highest"
+                        " evaluated budget is used.  \n "
                         "Note: Selecting combined budget might be misleading if a time objective "
                         "is used. Often, higher budget take longer to evaluate, which might "
                         "negatively influence the results."
diff --git a/deepcave/plugins/summary/footprint.py b/deepcave/plugins/summary/footprint.py
@@ -74,8 +74,9 @@ def get_input_layout(register: Callable) -> List[Any]:
                         [
                             dbc.Label("Budget"),
                             help_button(
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest"
-                                " evaluated budget is used.\n\n"
+                                " evaluated budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if"
                                 " a time objective is used. Often, higher budget take "
                                 " longer to evaluate, which might negatively influence "
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
@@ -39,6 +39,9 @@ to see the dashboard in action:
 .. image:: images/plugins/general.png
 
 
+Displaying the data is done via plugins. You can select the plugins you want to use in the menu on the left.
+This is further explained in the :ref:`plugins<Displaying Data with Plugins>` section.
+
 DeepCAVE uses runs to interprete data. A run is a collection of trials
 (a configuration with associated costs). Once valid runs (see :ref:`converter<Converters>`) are
 found, you can select them to analyze them further. After selecting runs, you can also add them to
diff --git a/docs/how_to_contribute.rst b/docs/how_to_contribute.rst
@@ -0,0 +1,77 @@
+==================
+How to Contribute
+==================
+We would appreciate your help in making this project better.
+Whether it's a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from our community.
+If you think you have found a bug, or have a new feature idea, please check for `github issues <github_issues_>`_ and create a new issue if needed.
+If you'd like to work on a bug or new feature idea, or have a correction or addition to the documentation, please open a new issue and setup a development workflow to work on the issue and submit a pull request.
+
+.. _contribute_fork_project:
+
+Setup Development Workflow
+---------------------------
+The development workflow is based on the "fork-and-pull" Git workflow. A full description of the workflow can be found `here <fork_and_pull_>`_. To contribute to the project, follow these steps:
+
+1. Fork the project.
+2. Install the project in development mode as described in the :ref:`installation` section.
+3. Create a new branch for the new feature on your own fork, note that all changes should be based on the `development` branch.
+4. Make your changes and commit them to your branch.
+5. Push your changes to your fork.
+6. Open a pull request from your fork to the main repository's `development` branch.
+
+
+
+.. _contribute_unit_tests:
+
+Extend Unit Tests
+------------------
+When adding new features or making changes to the codebase, it is important to add unit tests and check for mistakes revealed by pre-existing test cases. This project uses `unittest` for testing.
+To run the test suite, you can use the following command:
+
+.. code-block:: bash
+
+    python -m unittest discover -s tests
+
+
+.. _contribute_update_documentation:
+
+Update Documentation
+---------------------
+
+The documentation is written in reStructuredText and uses Sphinx to generate HTML documentation.
+To build the documentation, you can use the following command:
+
+.. code-block:: bash
+
+    make docs
+
+
+.. _contribute_pre_commit_hook:
+
+Pre-Commit Hook
+---------------
+Before submitting a pull request, you should run the pre-commit hook to check for formatting errors and run the test suite, e.g. via
+
+.. code-block:: bash
+
+    pre-commit run --all-files
+
+.. _contribute_pull_request:
+
+Create Pull Request
+--------------------
+
+After making changes to the codebase, you can submit a pull request. Your pull request will be reviewed by the maintainers, and if everything looks good, it will be merged.
+Before submitting a pull request, please make sure of the following:
+
+- Make sure to reference the issue in the pull request.
+- The pull request should include tests for the new feature or bug fix.
+- The pull request should include documentation for the new feature or bug fix.
+- All tests should pass.
+- Make sure to select the correct source and destination repositories and according branches, usually your fork to the main repository's `development` branch.
+- Add the change to the CHANGELOG.md file.
+- The pull request should include a description of the change and the reason for the change.
+
+
+.. _github_issues: https://github.com/automl/DeepCAVE/issues
+.. _fork_and_pull: https://reflectoring.io/github-fork-and-pull/
diff --git a/docs/index.rst b/docs/index.rst
@@ -14,6 +14,7 @@ Home
    examples/index
    plugins/index
    converters
+   how_to_contribute
    api
    glossary
    faq
diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst
@@ -1,5 +1,5 @@
-Plugins
-=======
+Displaying Data with Plugins
+============================
 
 .. toctree::
    :hidden:
diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst
@@ -23,7 +23,7 @@ This plugin is capable of answering following questions:
 * Is the configuration space as expected?
 * How many trials have been successful/unsuccessful?
 * Why did certain trials crash?
-* On which budgets were the configurations evaluated?
+* On which multi-fidelity budgets were the configurations evaluated?
 
 
 Quick Information
@@ -36,7 +36,9 @@ the runs were last updated and a handy quick link to the best **Configuration**.
 Meta
 ----
 Next we show some meta information regarding your runs. Notably, you can see information such as the
-possible **budget** allocations that were considered during the run.
+possible **budget** allocations that were considered during the run. The **budget** is the amount of
+resources that were allocated to a configuration as it was being evaluated using a multi-fidelity
+approach.
 
 
 Objectives
@@ -48,10 +50,10 @@ information about it's name and the bounds they were in.
 Statuses
 --------
 To get a nice overview of the optimization run, we provide a barplot that details how the end status
-of the many trials that occured. You can also view these as a heatmap or get more information from
+of the many trials that occurred. You can also view these as a heatmap or get more information from
 the **Details** tab.
 
-The barplot shows which trial recieved which status and under which budget allocation. These are
+The barplot shows which trial received which status and under which budget allocation. These are
 grouped according to the status and coloured based on the status it exited with.
 
 To see details on how these configurations faired as they progressed in budget, you can refer to
@@ -62,9 +64,9 @@ to try. However if you faced many crashes at high budgets for example, you'll se
 from the highest budget onwards. In general, you can use this to understand how often your optimizer
 balances out between low budget evaluations and full budget evaluations.
 
-In the **Details** tab, you'll recieve an overview of configurations that were not successful.
-Here you'll get some breif information about the config id number, the budget it was evaluated under
-and the status it recieved. If there is an associated error we can provide, such as when it recieved
+In the **Details** tab, you'll receive an overview of configurations that were not successful.
+Here you'll get some brief information about the config id number, the budget it was evaluated under
+and the status it received. If there is an associated error we can provide, such as when it received
 the **CRASHED** status, we will display it in the error column.
 
 
diff --git a/docs/plugins/partial_dependencies.rst b/docs/plugins/partial_dependencies.rst
@@ -1,15 +1,38 @@
 Partial Dependencies
 ====================
 
-This plugin is capable of answering following questions:
+The quality of a machine learning algorithm is highly dependent on the choice of hyperparameters.
+Partial dependence plots (PDP) aim to visualize how changing one or two hyperparameters affects the objective function.
+The PDP is calculated using a random forest surrogate model trained on the run data, i.e. the evaluated hyperparameter configurations and corresponding objective values.
+
+* In the **one hyperparameter case**, random samples or the selected hyperparameters are combined with each combination of the other hyperparameters present in the dataset, resulting in a multitude of individual conditional expectation (ICE) curves.
+  The resulting ICE curves are than averaged to a PDP.
+  The hyperparameter is plotted along the x axis, and the respective objective function value along the y-axis.
+
+* The **two hyperparameter case** works as the one hyperparameter case, but the then two dimensional ICE-curves are disregarded in the plot.
+  The resulting plot has one target hyperparameter plotted along the x-axis and the other hyperparameter plotted along the y-axis.
+  The color in the heatmap plot indicates the respective objective function value.
+
+
+This plugin is capable of answering the following questions:
 
 * How does the objective change with respect to one or two hyperparameters? For example, does the
   accuracy increase if the learning rate decreases?
 * Do multiple trials show similar behavior?
 
+Options
+-------
+* **Objective**: Here you can select the objective you wish to analyze. In the one hyperparameter case the objective is plotted on the y-axis, in the two hyperparameter case it is represented by the color.
+
+* **Budget**: Here you can select the multi-fidelity budget to be used. The plugin will only consider trials evaluated on the selected budget.
+
+* **Hyperparameter 1**: Here you can select the first hyperparameter. It is plotted along the x-axis.
+
+* **Hyperparameter 2**: Here you can optionally select the second hyperparameter.
+
+* **Show Confidence**: Selection for whether or not to show confidence estimates (based on the surrogate model's uncertainty estimate).
 
-.. warning::
-    This page is under construction.
+* **Show ICE Curves**: Selection for whether or not to to show the ICE curves used during computation (only for the one hyperparameter case).
 
 
 .. image:: ../images/plugins/partial_dependencies.png

Original file line number	Diff line number	Diff line change
`@@ -147,7 +147,10 @@ def get_filter_layout(register: Callable) -> List[html.Div]:`
`147`	`147`	`html.Div(`
`148`	`148`	`[`
`149`	`149`	`dbc.Label("Budgets"),`
`150`		`- help_button("The hyperparameters are sorted by the highest budget."),`
	`150`	`+ help_button(`
	`151`	`+ "Budget refers to the multi-fidelity budget. "`
	`152`	`+ "The hyperparameters are sorted by the highest budget."`
	`153`	`+ ),`
`151`	`154`	`dbc.Checklist(id=register("budget_ids", ["value", "options"]), inline=True),`
`152`	`155`	`]`
`153`	`156`	`),`
Original file line number	Diff line number	Diff line change
`@@ -115,6 +115,7 @@ def get_input_layout(register: Callable) -> List[dbc.Row]:`
`115`	`115`	`[`
`116`	`116`	`dbc.Label("Budget"),`
`117`	`117`	`help_button(`
	`118`	`+ "Budget refers to the multi-fidelity budget. "`
`118`	`119`	`"Combined budget means that the trial on the highest evaluated "`
`119`	`120`	`"budget is used."`
`120`	`121`	`),`