diff --git a/docs/migration_guide.md b/docs/migration_guide.md index 06d2b9f65c9..885e1cff47a 100644 --- a/docs/migration_guide.md +++ b/docs/migration_guide.md @@ -3,6 +3,37 @@ This guide contains breaking changes between major Mesa versions and how to reso Non-breaking changes aren't included, for those see our [Release history](https://github.com/projectmesa/mesa/releases). +## Mesa 3.4.0 + +### batch run +`batch_run` has been updated to offer explicit control over the random seeds that are used to run +multiple replications of a given experiment. For this a new keyword argument, `rng` has been +added and `iterations` will issue a `DeprecationWarning`. The new `rng` keyword argument +takes a valid value for seeding or a list of valid values. If you want to run multiple iterations/replications +of a given experiment, you need to pass the required seeds explicitly. + +Below is a simple example of the new recommended usage of `batch_run`. Note how we first +create 5 random integers which we then use as seed values for the new `rng` keyword argument. + +```python +import numpy as np +import sys + +# let's create 5 random integers +rng = np.random.default_rng(42) +rng_values = rng.integers(0, sys.maxsize, size=(5,)) + +results = mesa.batch_run( + MoneyModel, + parameters=params, + rng=rng_values.tolist(), # we pass the 5 seed values to rng + max_steps=100, + number_processes=1, + data_collection_period=1, + display_progress=True, +) +``` + ## Mesa 3.3.0 Mesa 3.3.0 is a visualization upgrade introducing a new and improved API, full support for both `altair` and `matplotlib` backends, and resolving several recurring issues from previous versions. diff --git a/docs/tutorials/9_batch_run.ipynb b/docs/tutorials/9_batch_run.ipynb index 98ee83e09bd..e4e79c2d37a 100644 --- a/docs/tutorials/9_batch_run.ipynb +++ b/docs/tutorials/9_batch_run.ipynb @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "collapsed": false, "jupyter": { @@ -94,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -162,7 +162,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -199,8 +199,8 @@ "* `number_processes`\n", " If not specified, defaults to 1. Set it to `None` to use all the available processors.\n", " Note: Multiprocessing does make debugging challenging. If your parameter sweeps are resulting in unexpected errors set `number_processes=1`.\n", - "* `iterations`\n", - " The number of iterations to run each parameter combination for. Optional. If not specified, defaults to 1.\n", + "* `rng`\n", + " a valid value or iterable of values for seeding the random number generator in the model. Defaults to a single None value meaning the model is ran once.\n", "* `data_collection_period`\n", " The length of the period (number of steps) after which the model and agent reporters collect data. Optional. If not specified, defaults to -1, i.e. only at the end of each episode.\n", "* `max_steps`\n", @@ -227,16 +227,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], + "source": [ + "import sys\n", + "\n", + "rng = np.random.default_rng(42)\n", + "seed_values = rng.integers(0, sys.maxsize, size=(5,))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2a208c5972c84a9ebc77fbc828763c94", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/100 [00:00 list[dict[str, Any]]: """Batch run a mesa model with a set of parameter values. @@ -62,6 +68,7 @@ def batch_run( data_collection_period (int, optional): Number of steps after which data gets collected, by default -1 (end of episode) max_steps (int, optional): Maximum number of model steps after which the model halts, by default 1000 display_progress (bool, optional): Display batch run process, by default True + rng : a valid value or iterable of values for seeding the random number generator in the model Returns: List[Dict[str, Any]] @@ -70,11 +77,33 @@ def batch_run( batch_run assumes the model has a `datacollector` attribute that has a DataCollector object initialized. """ + if iterations is not None and rng is not None: + raise ValueError( + "you cannot use both iterations and rng at the same time. Please only use rng." + ) + if iterations is not None: + warnings.warn( + "The `iterations` keyword argument is deprecated, please use `rng` instead." + "See https://mesa.readthedocs.io/latest/migration_guide.html#batch-run", + DeprecationWarning, + stacklevel=2, + ) + rng = [None] * iterations + if not isinstance(rng, Iterable): + rng = [rng] + + # establish to use seed or rng as name for parameter + model_parameters = inspect.signature(Model).parameters + rng_kwarg_name = "rng" + if "seed" in model_parameters: + rng_kwarg_name = "seed" + runs_list = [] run_id = 0 - for iteration in range(iterations): + for i, rng_i in enumerate(rng): for kwargs in _make_model_kwargs(parameters): - runs_list.append((run_id, iteration, kwargs)) + kwargs[rng_kwarg_name] = rng_i + runs_list.append((run_id, i, kwargs)) run_id += 1 process_func = partial( @@ -170,6 +199,7 @@ def _model_run_func( Return model_data, agent_data from the reporters """ run_id, iteration, kwargs = run + model = model_cls(**kwargs) while model.running and model.steps <= max_steps: model.step() diff --git a/tests/test_batch_run.py b/tests/test_batch_run.py index aec13bc4ccc..92d7c487963 100644 --- a/tests/test_batch_run.py +++ b/tests/test_batch_run.py @@ -1,5 +1,7 @@ """Test Batchrunner.""" +import pytest + import mesa from mesa.agent import Agent from mesa.batchrunner import _make_model_kwargs @@ -130,7 +132,41 @@ def step(self): # noqa: D102 def test_batch_run(): # noqa: D103 - result = mesa.batch_run(MockModel, {}, number_processes=2) + result = mesa.batch_run(MockModel, {}, number_processes=2, rng=42) + assert result == [ + { + "RunId": 0, + "iteration": 0, + "Step": 1000, + "reported_model_param": 42, + "AgentID": 1, + "agent_id": 1, + "agent_local": 250.0, + "seed": 42, + }, + { + "RunId": 0, + "iteration": 0, + "Step": 1000, + "reported_model_param": 42, + "AgentID": 2, + "agent_id": 2, + "agent_local": 250.0, + "seed": 42, + }, + { + "RunId": 0, + "iteration": 0, + "Step": 1000, + "reported_model_param": 42, + "AgentID": 3, + "agent_id": 3, + "agent_local": 250.0, + "seed": 42, + }, + ] + + result = mesa.batch_run(MockModel, {}, number_processes=2, iterations=1) assert result == [ { "RunId": 0, @@ -140,6 +176,7 @@ def test_batch_run(): # noqa: D103 "AgentID": 1, "agent_id": 1, "agent_local": 250.0, + "seed": None, }, { "RunId": 0, @@ -149,6 +186,7 @@ def test_batch_run(): # noqa: D103 "AgentID": 2, "agent_id": 2, "agent_local": 250.0, + "seed": None, }, { "RunId": 0, @@ -158,9 +196,81 @@ def test_batch_run(): # noqa: D103 "AgentID": 3, "agent_id": 3, "agent_local": 250.0, + "seed": None, }, ] + result = mesa.batch_run(MockModel, {}, number_processes=2, rng=[42, 31415]) + + # we use 2 processes, so we are not guaranteed the order of the return + result = sorted(result, key=lambda x: (x["RunId"], x["AgentID"])) + + assert result == [ + { + "RunId": 0, + "iteration": 0, + "Step": 1000, + "reported_model_param": 42, + "AgentID": 1, + "agent_id": 1, + "agent_local": 250.0, + "seed": 42, + }, + { + "RunId": 0, + "iteration": 0, + "Step": 1000, + "reported_model_param": 42, + "AgentID": 2, + "agent_id": 2, + "agent_local": 250.0, + "seed": 42, + }, + { + "RunId": 0, + "iteration": 0, + "Step": 1000, + "reported_model_param": 42, + "AgentID": 3, + "agent_id": 3, + "agent_local": 250.0, + "seed": 42, + }, + { + "RunId": 1, + "iteration": 1, + "Step": 1000, + "reported_model_param": 42, + "AgentID": 1, + "agent_id": 1, + "agent_local": 250.0, + "seed": 31415, + }, + { + "RunId": 1, + "iteration": 1, + "Step": 1000, + "reported_model_param": 42, + "AgentID": 2, + "agent_id": 2, + "agent_local": 250.0, + "seed": 31415, + }, + { + "RunId": 1, + "iteration": 1, + "Step": 1000, + "reported_model_param": 42, + "AgentID": 3, + "agent_id": 3, + "agent_local": 250.0, + "seed": 31415, + }, + ] + + with pytest.raises(ValueError): + mesa.batch_run(MockModel, {}, number_processes=2, rng=42, iterations=1) + def test_batch_run_with_params(): # noqa: D103 mesa.batch_run( @@ -185,6 +295,7 @@ def test_batch_run_no_agent_reporters(): # noqa: D103 "Step": 1000, "enable_agent_reporters": False, "reported_model_param": 42, + "seed": None, } ] @@ -208,6 +319,7 @@ def test_batch_run_unhashable_param(): # noqa: D103 "agent_local": 250.0, "n_agents": 2, "variable_model_params": {"key": "value"}, + "seed": None, } assert result == [