Merge pull request #797 from camsys/perf-fix

Performance monitoring fixes

Author: jpn--

activitysim/abm/models/input_checker.py

@@ -385,7 +385,7 @@ def input_checker(state: workflow.State):
     )
 
     data_model_dir = state.get_injectable("data_model_dir")[0]
-    logger.info("Data model directory:", data_model_dir)
+    logger.info("Data model directory: %s", data_model_dir)
 
     # FIXME: path doesn't recognize windows path object, so converting to string.
     # Is there a better way to get the data model directory than just adding it to the path?

activitysim/cli/run.py

@@ -22,6 +22,7 @@
     "configs_dir",
     "data_model_dir",
     "output_dir",
+    "cache_dir",
     "settings_file_name",
     "imported_extensions",
 ]
@@ -375,7 +376,13 @@ def run(args):
 
             from activitysim.core import mp_tasks
 
-            injectables = {k: state.get_injectable(k) for k in INJECTABLES}
+            injectables = {}
+            for k in INJECTABLES:
+                try:
+                    injectables[k] = state.get_injectable(k)
+                except KeyError:
+                    # if injectable is not set, just ignore it
+                    pass
             injectables["settings"] = state.settings
             # injectables["settings_package"] = state.settings.dict()
             mp_tasks.run_multiprocess(state, injectables)

activitysim/examples/external_example_manifest.yaml

@@ -20,6 +20,18 @@ prototype_mtc:
       url: https://github.com/ActivitySim/activitysim-prototype-mtc/releases/download/v1.3.1/prototype_mtc_reference_pipeline.zip
       sha256: 394e5b403d4c61d5214493cefe161432db840ba4967c23c999d914178d43a1f0
 
+prototype_mtc_extended:
+  url: https://github.com/ActivitySim/activitysim-prototype-mtc/archive/refs/tags/v1.3.2.tar.gz
+  sha256: b3f8f60f6354a0ffce80dd67f058c2ef81645d2c8bedddfcb989d3878651f045
+  assets:
+    mtc_data_full.tar.zst:
+      url: https://github.com/ActivitySim/activitysim-prototype-mtc/releases/download/v1.3.2/mtc_data_full.tar.zst
+      sha256: ffc22b34b4990a1459829b101b43180d0f5d41da0d4535d58588f232c98bd44e
+      unpack: data_full
+    test/prototype_mtc_reference_pipeline.zip:
+      url: https://github.com/ActivitySim/activitysim-prototype-mtc/releases/download/v1.3.2/prototype_mtc_extended_reference_pipeline.zip
+      sha256: 4d94b6a8a83225dda17e9ca19c9110bc1df2df5b4b362effa153d1c8d31524f5
+
 estimation_example:
   url: https://github.com/ActivitySim/activitysim-estimation-example/archive/refs/tags/v0.0.2.tar.gz
   sha256: 88c8208ee250a20e7d77036d77bc71122f21cbfeaba1eaf3b644120799f9d023

activitysim/examples/prototype_mtc_extended/data_model/input_checks.py

@@ -3,24 +3,26 @@
 
 Instructions: customize these example values for your own ActivitySim implementation
 """
-from typing import List, Optional
-import os, sys, logging
+from __future__ import annotations
 
-from pydantic import BaseModel, validator
-import pandera as pa
-import numpy as np
-import pandas as pd
-import openmatrix as omx
+import csv
+import logging
+import os
 
 # for skim name parsing
 import re
-import csv
-
-from activitysim.core import config
+import sys
+from typing import List, Optional
 
 import enums as e
+import numpy as np
+import openmatrix as omx
+import pandas as pd
+import pandera as pa
+from pydantic import BaseModel, validator
 
 from activitysim.abm.models.input_checker import TABLE_STORE, log_info
+from activitysim.core import config
 
 logger = logging.getLogger(__name__)
 
@@ -225,12 +227,13 @@ def check_hh_per_zone(cls, land_use: pd.DataFrame):
     def check_pop_per_zone(cls, land_use: pd.DataFrame):
         persons = TABLE_STORE["persons"]
         households = TABLE_STORE["households"]
-        pop = persons.groupby(
-            persons.household_id.map(
-                lambda hhid: households.set_index("household_id").home_zone_id[hhid]
-            )
-        ).person_id.nunique()
-        return (pop == land_use.set_index("zone_id").TOTPOP).reindex(land_use.index)
+        persons_per_household = persons.groupby("household_id").size()
+        hh = households[["household_id", "home_zone_id"]].merge(
+            persons_per_household.rename("persons_per_household"), on="household_id"
+        )
+        pop = hh.groupby(households.home_zone_id)["persons_per_household"].sum()
+        lu = land_use.set_index("zone_id")
+        return pop.reindex(lu.index) == lu.TOTPOP
 
 
 class NetworkLinks(pa.DataFrameModel):

activitysim/workflows/steps/wrapping.py

@@ -125,8 +125,13 @@ def run_step(context: Context = None) -> None:
                 reset_progress_step(description=progress_tag)
 
             return_type = _annotations.get("return", "<missing>")
-            _updates_context = updates_context or return_type in {dict, Context}
-            if return_type not in {None, dict, Context}:
+            _updates_context = updates_context or return_type in {
+                dict,
+                Context,
+                "dict",
+                "Context",
+            }
+            if return_type not in {None, dict, Context, "None", "dict", "Context"}:
                 if returns_names is None and not _updates_context:
                     context.assert_key_has_value(
                         key="report", caller=wrapped_func.__module__

conda-environments/activitysim-dev-base.yml

@@ -15,6 +15,7 @@ channels:
 dependencies:
 - python=3.10
 - pip
+- altair
 - asv  # for benchmarking
 - black >= 22.0,<23
 - bump2version  # for making a release

conda-environments/activitysim-dev.yml

@@ -11,6 +11,7 @@ channels:
 dependencies:
 - python=3.10
 - pip
+- altair
 - asv  # for benchmarking
 - black >= 22.0,<23
 - bump2version  # for making a release