Merge pull request #977 from Transport-for-the-North/pre-me-tour-props-fix

Fixes for time period splits

Author: BenTaylor-TfN

RELEASE.md

@@ -30,6 +30,9 @@ Below, a brief summary of patches made since the previous version can be found.
   - Renamed the incorrectly named "TemproExtractor" to "NTEM Extractor"
     - Updated tool to extract different versions of NTEM data
     - Updated tool to extract different scenarios of NTEM data
+  - TLD Builder
+    - Fixed a bug where demand on bounds was being dropped
+    - Added functionality to generate dynamic bands to create a log curve for demand
 - Core
   - Added Midlands Connect zoning systems as built-ins
   - Added functionality to convert matrices into SATURN and CUBE formats
@@ -38,6 +41,7 @@ Below, a brief summary of patches made since the previous version can be found.
   - Adapted NoTEM to optionally perform steps needed for Midlands Models
     -  Including optional localised trip end adjustments
   - Added a temporary front end script for `Midlands Distribution Model`
+  - Fixed a bug in the default distribution model run, running at the incorrect time format
   - Created a generic front end for all forecasting functionality
     - Can run NTEM based forecasts, or synthetic trip end based forecasts from 
       the same front end

normits_demand/logging.py

@@ -10,6 +10,8 @@
 File purpose:
 Initialiser for all logging in normits_demand
 """
+from __future__ import annotations
+
 # Builtins
 import logging
 from typing import Any, Dict
@@ -330,3 +332,35 @@ def capture_warnings(
     if file_handler_args is not None:
         warning_logger.addHandler(get_file_handler(**file_handler_args))
 
+
+class TemporaryLogFile:
+    """Add temporary log file to a logger."""
+
+    def __init__(self, logger: logging.Logger, log_file: nd.PathLike, **kwargs) -> None:
+        """Add temporary log file handler to `logger`.
+        Parameters
+        ----------
+        logger : logging.Logger
+            Logger to add FileHandler to.
+        log_file : nd.PathLike
+            Path to new log file to create.
+        kwargs : Keyword arguments, optional
+            Any arguments to pass to `get_file_handler`.
+        """
+        self.logger = logger
+        self.log_file = log_file
+        self.logger.debug('Creating temporary log file: "%s"', self.log_file)
+        self.handler = get_file_handler(log_file, **kwargs)
+        self.logger.addHandler(self.handler)
+
+    def __enter__(self) -> TemporaryLogFile:
+        """Initialise TemporaryLogFile."""
+        return self
+
+    def __exit__(self, excepType, excepVal, traceback) -> None:
+        """Close temporary log file."""
+        if excepType is not None or excepVal is not None or traceback is not None:
+            self.logger.critical("Oh no a critical error occurred", exc_info=True)
+
+        self.logger.removeHandler(self.handler)
+        self.logger.debug('Closed temporary log file: "%s"', self.log_file)

normits_demand/matrices/matrix_processing.py

@@ -2413,7 +2413,7 @@ def compile_matrices(
     compile_params_path: nd.PathLike,
     factor_pickle_path: str = None,
     round_dp: int = consts.DEFAULT_ROUNDING,
-    factors_fname: str = "od_compilation_factors.pickle",
+    factors_fname: str = "od_compilation_factors.pkl",
     avoid_zero_splits: bool = False,
     process_count: int = consts.PROCESS_COUNT,
     overwrite: bool = True,
@@ -2470,7 +2470,7 @@ def compile_matrices(
         if pathlib.Path(factor_pickle_path).suffix == "":
             print(
                 "WARNING: No filename was given for the pickle factors. "
-                "Defaulting to od_compilation_factors.pickle, but this is "
+                "Defaulting to od_compilation_factors.pkl, but this is "
                 "deprecated and will be removed in future!"
             )
             factor_pickle_path = os.path.join(factor_pickle_path, factors_fname)

normits_demand/matrices/utils.py

@@ -10,6 +10,8 @@
 File purpose:
 Utility functions specific to matrices
 """
+from __future__ import annotations
+
 # builtins
 import os
 import pathlib
@@ -258,3 +260,14 @@ def split_matrix_by_time_periods(
         )
 
     return tp_mats
+
+
+def apply_factor(
+    input_path: os.PathLike,
+    output_path: os.PathLike,
+    factor: int | float,
+) -> None:
+    """Apply a factor to a matrix"""
+    mat = file_ops.read_df(input_path, index_col=0)
+    mat *= factor
+    file_ops.write_df(mat, output_path)

normits_demand/models/distribution_model.py

@@ -13,6 +13,7 @@
 # Built-Ins
 import os
 import pathlib
+import itertools
 import functools
 
 from typing import Any
@@ -37,6 +38,7 @@
 from normits_demand.matrices import pa_to_od
 from normits_demand.matrices import utils as mat_utils
 from normits_demand.reports import matrix_reports
+from normits_demand.concurrency import multiprocessing
 
 from normits_demand.pathing.distribution_model import DistributionModelExportPaths
 from normits_demand.pathing.distribution_model import DMArgumentBuilderBase
@@ -742,7 +744,87 @@ def run_od_matrix_reports(self):
             col_name='destinations',
         )
 
-    def compile_to_assignment_format(self):
+    def _convert_matrix_time_format(
+        self,
+        import_dir: pathlib.Path,
+        export_dir: pathlib.Path,
+        from_time_format: nd.core.TimeFormat = None,
+        to_time_format: nd.core.TimeFormat = None,
+    ) -> None:
+        """Converts matrices between time formats"""
+        # TODO(BT): This function just assumes there's time periods.
+        #  Won't work otherwise
+        conversion_factors = from_time_format.get_conversion_factors(to_time_format)
+
+        # Build matrix naming templates
+        template = self.running_segmentation.generate_template_file_name(
+            file_desc='{matrix_format}',
+            trip_origin=self.trip_origin,
+            year=str(self.year),
+            compressed=True,
+        )
+
+        if self.trip_origin == nd.core.TripOrigin.HB.value:
+            matrix_formats = ["synthetic_od_from", "synthetic_od_to"]
+        elif self.trip_origin == nd.core.TripOrigin.NHB.value:
+            matrix_formats = ["synthetic_od"]
+        else:
+            raise ValueError(f"Trip origin '{self.trip_origin}' not recognised.")
+
+        # Build the multiprocessing kwargs
+        kwarg_list = list()
+
+        # BACKLOG(BT): This is all a really rough kludge to get this working
+        #  NOW. Need to come back and think how to do this properly.
+        if self.running_segmentation.has_time_period_segments():
+            iterator = itertools.product(self.running_segmentation, [-1])
+            naming_order = self.running_segmentation.naming_order
+            segment_types = self.running_segmentation.segment_types
+        else:
+            tps = [1, 2, 3, 4, 5, 6]
+            iterator = itertools.product(self.running_segmentation, tps)
+            naming_order = self.running_segmentation.naming_order + ['tp']
+            segment_types = self.running_segmentation.segment_types | {"tp": int}
+
+        for segment_params, tp in iterator:
+            # Generate filenames
+            tp_params = segment_params.copy()
+            if "tp" not in tp_params:
+                tp_params['tp'] = tp
+            segment_str = nd.core.SegmentationLevel.generate_template_segment_str(
+                naming_order=naming_order,
+                segment_params=tp_params,
+                segment_types=segment_types,
+            )
+
+            # Build the kwarg list
+            for mx_format in matrix_formats:
+                fname = template.format(segment_params=segment_str, matrix_format=mx_format)
+                kwarg_list.append({
+                    "input_path": import_dir / fname,
+                    "output_path": export_dir / fname,
+                    "factor": conversion_factors[tp_params["tp"]]
+                })
+
+        # MP running
+        self._logger.info(
+            f"Converting OD matrix time format from {from_time_format.value} "
+            f"to {to_time_format.value}."
+        )
+        pbar_kwargs = {'desc': "Converting OD matrix time format"}
+        multiprocessing.multiprocess(
+            fn=mat_utils.apply_factor,
+            kwargs=kwarg_list,
+            process_count=self.process_count,
+            pbar_kwargs=pbar_kwargs
+
+        )
+
+    def compile_to_assignment_format(
+        self,
+        from_time_format: nd.core.TimeFormat = None,
+        to_time_format: nd.core.TimeFormat = None,
+    ):
         """TfN Specific helper function to compile outputs into assignment format
 
         This should really be the job of NorMITs Matrix tools! Move there
@@ -752,18 +834,32 @@ def compile_to_assignment_format(self):
         -------
 
         """
-        # TODO(BT): NEED TO OUTPUT SPLITTING FACTORS
-
         # TODO(BT): UPDATE build_compile_params() to use segmentation levels
         m_needed = self.running_segmentation.segments['m'].unique()
 
         # NoHAM should be tp split
         tp_needed = [1, 2, 3, 4]
 
+        # Covert time periods if factors given
+        od_mat_dir = self.export_paths.full_od_dir
+        if (
+           (from_time_format is not None and to_time_format is not None)
+            and (from_time_format != to_time_format)
+           ):
+            new_od_mat_dir = pathlib.Path(self.export_paths.full_od_dir) / "converted time format"
+            new_od_mat_dir.mkdir(exist_ok=True)
+            self._convert_matrix_time_format(
+                import_dir=pathlib.Path(self.export_paths.full_od_dir),
+                export_dir=pathlib.Path(new_od_mat_dir),
+                from_time_format=from_time_format,
+                to_time_format=to_time_format,
+            )
+            od_mat_dir = new_od_mat_dir
+
         if self.running_mode in [nd.Mode.CAR, nd.Mode.BUS]:
             # Compile to NoHAM format
             compile_params_paths = matrix_processing.build_compile_params(
-                import_dir=self.export_paths.full_od_dir,
+                import_dir=od_mat_dir,
                 export_dir=self.export_paths.compiled_od_dir,
                 matrix_format=self._od_matrix_desc,
                 years_needed=[self.year],
@@ -772,9 +868,10 @@ def compile_to_assignment_format(self):
             )
 
             matrix_processing.compile_matrices(
-                mat_import=self.export_paths.full_od_dir,
+                mat_import=od_mat_dir,
                 mat_export=self.export_paths.compiled_od_dir,
                 compile_params_path=compile_params_paths[0],
+                factors_fname="od_compilation_factors.pkl",
             )
 
             # TODO(BT): Build in DM imports!
@@ -815,7 +912,7 @@ def compile_to_assignment_format(self):
             self._logger.info("Compiling NoRMS VDM Format")
             matrix_processing.compile_norms_to_vdm(
                 mat_pa_import=self.export_paths.full_tp_pa_dir,
-                mat_od_import=self.export_paths.full_od_dir,
+                mat_od_import=od_mat_dir,
                 mat_export=self.export_paths.compiled_pa_dir,  # TODO(BT): Rename to NoRMS
                 params_export=self.export_paths.compiled_pa_dir,
                 year=self.year,