Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,14 @@ If you don't need the substeps in post-processing, you can both improve simulati
experiment.engine = Engine(drop_substeps=True)
```

#### Including parameters in results

If you'd like to include each run's parameters directly in the experiment results (rather than have to manually construct the corresponding parameters for each run afterwards), you can specify to return parameters as part of the experiment results:

```python
experiment.engine = Engine(return_parameters=True)
```

#### Exception handling

radCAD allows you to choose whether to raise exceptions, ending the simulation, or to continue with the remaining runs and return the results along with the exceptions. Failed runs are returned as partial results - the part of the simulation result up until the timestep where the simulation failed.
Expand Down
5 changes: 5 additions & 0 deletions radcad/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def _single_run(
deepcopy: bool,
deepcopy_method: Callable,
drop_substeps: bool,
return_parameters: bool,
):
logger.info(f"Starting simulation {simulation} / run {run} / subset {subset}")

Expand Down Expand Up @@ -127,6 +128,7 @@ def single_run(
deepcopy: bool=True,
deepcopy_method: Callable=default_deepcopy_method,
drop_substeps: bool=False,
return_parameters: bool=False,
) -> Tuple[list, Exception, str]:
result = []

Expand All @@ -144,6 +146,7 @@ def single_run(
deepcopy,
deepcopy_method,
drop_substeps,
return_parameters,
),
None, # Error
None, # Traceback
Expand All @@ -164,6 +167,8 @@ def _single_run_wrapper(args):
if raise_exceptions and exception:
raise exception
else:
if run_args.return_parameters:
[substep.update(run_args.parameters) for timestep in results for substep in timestep]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One issue we might face including all parameters in the simulation results is that typically not all parameters are basic Python types, serialisable, or able to be stored in Pandas DataFrames efficiently.

As an alternative we do currently have access to each subset's parameters in the before_subset() and after_subset() hooks, should a user want to get access to these before or after a specific set of parameters is used in a simulation.

I understand the use case of storing parameters in the simulation results and having them end up in a DataFrame though - I have done this in the past for a subset of the parameters that are useful to have in post-processing and maybe additionally are basic Python types.

Let me sleep on this and see if there are other ways of implementing, open to ideas too :)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I 100% agree with the benefits of such a feature:

This feature saves time, reduces complexity, and avoids the risk of error when requiring users to manually construct a map of which parameters correspond to each run after the fact.

Copy link
Author

@mattyTokenomics mattyTokenomics Feb 13, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fair point about complex data types for params. Given all params are specified as values in a dict though, I believe it should cover every case to output a dict with keys corresponding to run # and subset # and a value of a dict of params:
{ run_i: {subset_j: {param_x: value_x }, subset_j+1: {param_x: value_x }} }

This approach would need to make experiment.run() return both the run results and the params dict at well. ie:
results, params = experiment.run()

return results, {
'exception': exception,
'traceback': traceback,
Expand Down
3 changes: 3 additions & 0 deletions radcad/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ def __init__(self, **kwargs):
**deepcopy (bool): Whether to enable deepcopy of State Variables to avoid unintended state mutation. Defaults to `True`.
**deepcopy_method (Callable): Method to use for deepcopy of State Variables. By default uses Pickle for improved performance, use `copy.deepcopy` for an alternative to Pickle.
**drop_substeps (bool): Whether to drop simulation result substeps during runtime to save memory and improve performance. Defaults to `False`.
**return_parameters (bool): Whether to include the parameters used for each run in the simulation results at expense of increased memory usage and reduced performance. Defaults to `False`.
**_run_generator (tuple_iterator): Generator to generate simulation runs, used to implement custom execution backends. Defaults to `iter(())`.
"""
self.executable = None
Expand All @@ -31,6 +32,7 @@ def __init__(self, **kwargs):
self.deepcopy = kwargs.pop("deepcopy", True)
self.deepcopy_method = kwargs.pop("deepcopy_method", core.default_deepcopy_method)
self.drop_substeps = kwargs.pop("drop_substeps", False)
self.return_parameters = kwargs.pop("return_parameters", False)
self._run_generator = iter(())

if kwargs:
Expand Down Expand Up @@ -136,6 +138,7 @@ def _run_stream(self, configs):
deepcopy=self.deepcopy,
deepcopy_method=self.deepcopy_method,
drop_substeps=self.drop_substeps,
return_parameters=self.return_parameters,
))
self.executable._after_subset(context=context)
self.executable._after_run(context=context)
Expand Down
4 changes: 4 additions & 0 deletions radcad/wrappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class RunArgs(NamedTuple):
deepcopy: bool = None
deepcopy_method: bool = None
drop_substeps: bool = None
return_parameters: bool = None


@dataclass
Expand Down Expand Up @@ -55,6 +56,7 @@ def __init__(self, initial_state={}, state_update_blocks=[], params={}):
self._deepcopy = True
self._deepcopy_method = default_deepcopy_method
self._drop_substeps = False
self._return_parameters = False

def __iter__(self):
while True:
Expand All @@ -71,6 +73,7 @@ def __iter__(self):
deepcopy = self._deepcopy,
deepcopy_method = self._deepcopy_method,
drop_substeps = self._drop_substeps,
return_parameters = self._return_parameters,
)
result, exception = _single_run_wrapper((run_args, self._raise_exceptions))
if exception: self.exceptions.append(exception)
Expand All @@ -83,6 +86,7 @@ def __call__(self, **kwargs):
self._deepcopy = kwargs.pop("deepcopy", True)
self._deepcopy_method = kwargs.pop("deepcopy_method", default_deepcopy_method)
self._drop_substeps = kwargs.pop("drop_substeps", False)
self._return_parameters = kwargs.pop("return_parameters", False)
return self


Expand Down