support parameter searching with optuna

Signed-off-by: reiase <[email protected]>

Author: reiase

examples/optuna/README.md

@@ -0,0 +1,87 @@
+Hyper-Parameter Optimization
+============================
+
+This example is based on `optuna` [quick start example](https://optuna.org/#code_quickstart). [Optuna](https://optuna.org/) is an open-source [hyperparameter](https://github.com/reiase/hyperparameter) optimization framework which is easy to use:
+
+```python
+import optuna
+
+def objective(trial):
+    x = trial.suggest_float('x', -10, 10)
+    return (x - 2) ** 2
+
+study = optuna.create_study()
+study.optimize(objective, n_trials=100)
+
+study.best_params  # E.g. {'x': 2.002108042}
+```
+
+The above example creates a `study` object to search for the best parameter `x` that minimizes the objective function `(x-2)^2`.
+
+Parameter Searching with [`HyperParameter`](https://github.com/reiase/hyperparameter)
+-----------------------------------------
+
+Parameter searching can be much easier with [`HyperParameter`](https://github.com/reiase/hyperparameter):
+
+```python
+import optuna
+from hyperparameter import param_scope, auto_param, lazy_dispatch
+
+@auto_param
+def objective(x = 0.0):
+    return (x - 2) ** 2
+
+def wrapper(trial):
+    trial = lazy_dispatch(trial)
+    with param_scope(**{
+        "objective.x": trial.suggest_float('objective.x', -10, 10)
+    }):
+        return objective()
+
+study = optuna.create_study()
+study.optimize(wrapper, n_trials=100)
+
+study.best_params  # E.g. {'x': 2.002108042}
+```
+
+We directly apply [the `auto_param` decorator](https://reiase.github.io/hyperparameter/quick_start/#auto_param) to the objective function so that it accepts parameters from [`param_scope`](https://reiase.github.io/hyperparameter/quick_start/#param_scope). Then we define a wrapper function that adapts `param_scope` API to `optuna`'s `trial` API and starts the parameter experiment as suggested in `optuna`'s example.
+
+Put the Best Parameters into Production
+---------------------------------------
+
+To put the best parameters into production, we can directly pass them to `param_scope`. This is very convenient if you want to put a ML model into production.
+
+```python
+with param_scope(**study.best_params):
+    print(f"{study.best_params} => {objective()}")
+```
+
+- hyperparameters for nested functions:
+
+For complex problems with nested functions:
+
+```python
+@auto_param
+def objective_x(x = 0.0):
+    return (x - 2) ** 2
+
+@auto_param
+def objective_y(y = 0.0):
+    return (y - 1) ** 3
+
+def objective():
+    return objective_x() * objective_y()
+
+def wrapper(trial):
+    trial = lazy_dispatch(trial)
+    with param_scope(**{
+        "objective_x.x": trial.suggest_float('objective_x.x', -10, 10),
+        "objective_y.y": trial.suggest_float('objective_y.y', -10, 10)
+    }):
+        return objective()
+
+study = optuna.create_study()
+study.optimize(wrapper, n_trials=100)
+
+study.best_params  # E.g. {'x': 2.002108042}
+```

examples/optuna/example.py

@@ -0,0 +1,10 @@
+import optuna
+
+def objective(trial):
+    x = trial.suggest_float('x.x.y', -10, 10)
+    return (x - 2) ** 2
+
+study = optuna.create_study()
+study.optimize(objective, n_trials=100)
+
+print(study.best_params)  # E.g. {'x': 2.002108042}

examples/optuna/example_hp.py

@@ -0,0 +1,21 @@
+import optuna
+from hyperparameter import param_scope, auto_param, lazy_dispatch
+
+@auto_param
+def objective(x = 0.0):
+    return (x - 2) ** 2
+
+def wrapper(trial):
+    trial = lazy_dispatch(trial)
+    with param_scope(**{
+        "objective.x": trial.suggest_float('objective.x', -10, 10)
+    }):
+        return objective()
+
+study = optuna.create_study()
+study.optimize(wrapper, n_trials=100)
+
+study.best_params  # E.g. {'x': 2.002108042}
+
+with param_scope(**study.best_params):
+    print(f"{study.best_params} => {objective()}")

examples/optuna/example_hp_nested.py

@@ -0,0 +1,29 @@
+import optuna
+from hyperparameter import param_scope, auto_param, lazy_dispatch
+
+@auto_param
+def objective_x(x = 0.0):
+    return (x - 2) ** 2
+
+@auto_param
+def objective_y(y = 0.0):
+    return (y - 1) ** 4
+
+def objective():
+    return objective_x() + objective_y()
+
+def wrapper(trial):
+    trial = lazy_dispatch(trial)
+    with param_scope(**{
+        "objective_x.x": trial.suggest_float('objective_x.x', -10, 10),
+        "objective_y.y": trial.suggest_float('objective_y.y', -10, 10)
+    }):
+        return objective()
+
+study = optuna.create_study()
+study.optimize(wrapper, n_trials=100)
+
+study.best_params  # E.g. {'x': 2.002108042}
+
+with param_scope(**study.best_params):
+    print(f"{study.best_params} => {objective()}")

hyperparameter/__init__.py

@@ -1,11 +1,13 @@
 from .hyperparameter import HyperParameter
 from .hyperparameter import param_scope, reads, writes, all_params
 from .hyperparameter import auto_param, set_auto_param_callback
-from .hyperparameter import dynamic_dispatch
+from .hyperparameter import dynamic_dispatch, lazy_dispatch, suggest_from
 
 __all__ = [
     "HyperParameter",
     "dynamic_dispatch",
+    "suggest_from",
+    "lazy_dispatch",
     "param_scope",
     "reads",
     "writes",

hyperparameter/hyperparameter.py

@@ -1,6 +1,7 @@
 import inspect
+from operator import getitem
 import threading
-from typing import Any, Callable, Dict, Set
+from typing import Any, Callable, Dict, List, Optional, Set
 
 
 def _sorted_set(s):
@@ -40,10 +41,10 @@ def clear(self):
         self._get.clear()
         self._put.clear()
 
-    def read(self, key: str = None) -> Set[str]:
+    def read(self, key: Optional[str] = None) -> Optional[List[str]]:
         return self._get.add(key) if key else _sorted_set(self._get)
 
-    def write(self, key: str = None) -> Set[str]:
+    def write(self, key: Optional[str] = None) -> Optional[List[str]]:
         return self._put.add(key) if key else _sorted_set(self._put)
 
     def all(self):
@@ -65,6 +66,47 @@ def all_params():
     return _tracker.all()
 
 
+class Suggester:
+    def __init__(self, callback: Callable) -> None:
+        self._callback = callback
+
+    def __call__(self):
+        return self._callback()
+
+
+def suggest_from(callback: Callable) -> Suggester:
+    """ Suggest parameter from a callback function
+
+    Examples
+    --------
+    >>> class ValueWrapper:
+    ...     def __init__(self, lst):
+    ...         self._lst = lst
+    ...         self._offset = 0
+    ...     def __call__(self):
+    ...         index, self._offset = self._offset % len(self._lst), self._offset + 1
+    ...         return self._lst[index]
+
+    >>> with param_scope(suggested = suggest_from(ValueWrapper([1,2,3]))) as ps:
+    ...     ps().suggested()
+    ...     ps().suggested()
+    ...     ps().suggested()
+    1
+    2
+    3
+    """
+    return Suggester(callback)
+
+
+def _unwrap_suggester(func):
+    def wrapper(*args, **kwargs):
+        retval = func(*args, **kwargs)
+        if isinstance(retval, Suggester):
+            return retval()
+        return retval
+    return wrapper
+
+
 class _Accessor(dict):
     """Helper for accessing hyper-parameters."""
 
@@ -74,6 +116,7 @@ def __init__(self, root, path=None, scope=None):
         self._path = path
         self._scope = scope
 
+    @_unwrap_suggester
     def get_or_else(self, default: Any = None):
         """Get value for the parameter, or get default value if the parameter is not defined."""
         if self._scope is not None:
@@ -173,6 +216,46 @@ def dynamic_dispatch(func, name=None, index=None):
     return new_class(func, name, index)
 
 
+class LazyDispatch:
+    """Dynamic call dispatcher
+
+    Examples
+    --------
+
+    >>> class ExampleObj:
+    ...     def get_42(self, offset):
+    ...         return 42+offset
+
+    >>> lazy_obj = lazy_dispatch(ExampleObj())
+    >>> lazy_obj.get_42(0)()
+    42
+    """
+
+    def __init__(self, obj: Any, name=None, index=None):
+        self._obj = obj
+        self._name = name
+        self._index = index
+
+    def __call__(self, *args, **kws) -> Any:
+        obj = self._obj
+        for n in self._name.split("."):
+            obj = getattr(obj, n)
+        if self._index is not None:
+            obj = getitem(obj, self._index)
+        return Suggester(lambda: obj(*args, **kws))
+
+    def __getattr__(self, name: str) -> Any:
+        if self._name is not None:
+            name = f"{self._name}.{name}"
+        return lazy_dispatch(self._obj, name, self._index)
+
+    def __getitem__(self, index):
+        return lazy_dispatch(self._obj, self._name, index)
+
+def lazy_dispatch(obj, name=None, index=None):
+    """Wraps an object for lazy dispatch"""
+    return LazyDispatch(obj, name, index)
+
 class HyperParameter(dict):
     """HyperParameter is an extended dict designed for parameter storage.