Progress on documentation

Author: yamilbknsu

configuration/demos_config.toml

@@ -17,6 +17,17 @@ initialize_empty_tables = [
     "rebalanced_persons",
     "rebalanced_households"
 ]
+modules = [
+    "aging",
+    "employment",
+    "household_reorg",
+    # "kids_moving_model",
+    # "fatality_model",
+    # "birth_model",
+    # "education_model",
+    # "household_rebalancing",
+    # "update_income"
+]
 
 # Data sources
 ## Synthetic population data
@@ -33,7 +44,6 @@ filepath = "../data/custom_mpo_06197001_model_data.h5"
 h5_key = "households"
 
 ## Other static data tables
-## Hide it?
 [[tables]]
 file_type = "csv"
 table_name = "relational_adjustment_mapping"

demos/TODO.md

@@ -44,16 +44,23 @@ class MortalityModuleConfig(BaseModel):
 class BirthModuleConfig(BaseModel):
     calibration_procedure: Optional[CalibrationConfig] = None
 
+class AgingModuleConfig(BaseModel):
+    """
+    """
+    #: Age at which a person qualifies as senior
+    senior_age: int = 65
+
+
 class DEMOSConfig(BaseModel):
     """
     Global configuration for DEMOS. Individual fields in this class control the configuration of each module.
     """
     random_seed: int
 
-    #: Last year of simulation
-    forecast_year: int = 2020
     #: Year represented in synthetic population input
     base_year: int
+    #: Last year of simulation
+    forecast_year: int = 2020
     #: Path to DEMOS outputs
     output_dir: str = "../data/output"
     #: Name of output HDF5 file. Defaults to `demos_output_{forecast_year}.h5`.
@@ -66,19 +73,19 @@ class DEMOSConfig(BaseModel):
     inconsistent_persons_table_behavior: Literal["error", "fix", "ignore"] = "error"
     #: Name of tables to be initialized as empty
     initialize_empty_tables: list[str] = None
-    
-    # region_code: str
-    # calibrated_folder: str = "custom"
 
     #: List of tables to be loaded into orca
-    tables: list[DataSourceModel] = Field(default_factory=list)
+    tables: Optional[list[DataSourceModel]] = None
+    #: List of modules to be run
+    modules: Optional[list[str]] = None
 
     # Module-specific config
-    employment_module_config: EmploymentModuleConfig
-    mortality_module_config: MortalityModuleConfig
-    birth_module_config: BirthModuleConfig
-    hh_reorg_module_config: HHReorgModuleConfig
-    hh_rebalancing_module_config: HHRebalancingModuleConfig
+    aging_module_config: AgingModuleConfig = Field(default_factory=AgingModuleConfig)
+    employment_module_config: EmploymentModuleConfig = Field(default_factory=EmploymentModuleConfig)
+    hh_reorg_module_config: HHReorgModuleConfig = Field(default_factory=HHReorgModuleConfig)
+    mortality_module_config: MortalityModuleConfig = Field(default_factory=MortalityModuleConfig)
+    birth_module_config: BirthModuleConfig = Field(default_factory=BirthModuleConfig)
+    hh_rebalancing_module_config: HHRebalancingModuleConfig = Field(default_factory=HHRebalancingModuleConfig)
 
     def model_post_init(self, __context) -> None:
         if self.output_fname is None:
@@ -99,6 +106,19 @@ def model_post_init(self, __context) -> None:
         for n in self.initialize_empty_tables:
             orca.add_table(n, pd.DataFrame())
 
+        if self.modules is None:
+            self.modules = [
+                "aging",
+                "laborforce_model",
+                "households_reorg",
+                "kids_moving_model",
+                "fatality_model",
+                "birth_model",
+                "education_model",
+                "household_rebalancing",
+                "update_income"
+            ]
+        
 
     @model_validator(mode='after')
     def require_persons_and_households(self):
@@ -113,4 +133,6 @@ def load_config_file(dir: str) -> DEMOSConfig:
 
 def get_config():
     global CONFIG
+    if CONFIG is None:
+        CONFIG = DEMOSConfig()
     return CONFIG

demos/config.py

@@ -6,6 +6,7 @@
 
 
 class CSVTableSource(BaseModel):
+    """"""
     file_type: Literal['csv']
     #: Path to source file
     filepath: str
@@ -26,7 +27,9 @@ def load_into_orca(self):
 
 
 class H5TableSource(BaseModel):
+    """"""
     file_type: Literal['h5']
+    #: Path to source file
     filepath: str
     #: key in the source HDF5 to be loaded
     h5_key: str

demos/datasources.py

@@ -1,46 +1,29 @@
 import orca
 import time
 import pandas as pd
-import logging
-import inspect
 from loguru import logger
+import sys, contextlib
 
-class InterceptHandler(logging.Handler):
-    def __init__(self, target_name: str, prefix: str):
-        super().__init__()
-        self.target_name = target_name
-        self.prefix = prefix
-
-    def emit(self, record: logging.LogRecord) -> None:
-        # 1. map stdlib level to Loguru level
-        try:
-            level = logger.level(record.levelname).name
-        except ValueError:
-            level = record.levelno
-
-        # 2. compute correct stack depth so Loguru shows your caller, not the Handler
-        frame, depth = inspect.currentframe(), 2
-        while frame and frame.f_code.co_filename == logging.__file__:
-            frame = frame.f_back
-            depth += 1
-
-        # 3. optionally tag or prefix messages from your library
-        msg = record.getMessage()
-        if record.name == self.target_name:
-            msg = f"{self.prefix} {msg}"
-
-        # 4. re-emit through Loguru
-        logger.opt(depth=depth, exception=record.exc_info).log(level, msg)
-
-def capture_orca_logs():
-    LIB_NAME = "orca.orca"
-    PREFIX   = "[ORCA]"
-    handler  = InterceptHandler(target_name=LIB_NAME, prefix=PREFIX)
-
-    lib_logger = logging.getLogger(LIB_NAME)
-    lib_logger.handlers = [handler]
-    lib_logger.propagate = False
 
+class _StdoutToLoguru:
+    def __init__(self, level="INFO", prefix="[external] "):
+        self.level = level
+        self.prefix = prefix
+        self._buf = ""
+
+    def write(self, msg):
+        # buffer to handle partial writes / no trailing newline
+        self._buf += msg
+        while "\n" in self._buf:
+            line, self._buf = self._buf.split("\n", 1)
+            line = line.rstrip()
+            if line:
+                logger.opt(depth=1).log(self.level, f"{self.prefix}{line}")
+
+    def flush(self):
+        if self._buf.strip():
+            logger.opt(depth=1).log(self.level, f"{self.prefix}{self._buf.rstrip()}")
+        self._buf = ""
 
 def log_execution_time(start_time, year, module_name):
     now = time.time()

demos/logging_logic.py

@@ -1,6 +1,6 @@
 from .data_fix import *
-from .aging import *
-from .laborforce import *
+from .aging import REQUIRED_COLUMNS as AGING_COLUMNS
+from .employment import *
 from .household_reorg import *
 from .marriage import *
 from .kids_moving import *