88import os
99import warnings
1010from abc import ABC
11+ from collections import defaultdict
1112
1213import numpy as np
1314import openmatrix as omx
@@ -86,6 +87,7 @@ def __init__(self, state, skim_tag, network_los):
8687 self .offset_map_name = None
8788 self .offset_map = None
8889 self .omx_keys = None
90+ self .skim_conflicts = None
8991 self .base_keys = None
9092 self .block_offsets = None
9193
@@ -117,7 +119,12 @@ def load_skim_info(self, state, skim_tag):
117119 logger .debug (f"load_skim_info { skim_tag } { omx_file_path } )
118120
119121 with omx .open_file (omx_file_path , mode = "r" ) as omx_file :
120- # fixme call to omx_file.shape() failing in windows p3.5
122+
123+ # Check the shape of the skims. All skim files loaded within this
124+ # loop need to have the same shape. For the first file, the
125+ # shape is set to the omx_shape attribute, so we know what shape
126+ # to expect. For subsequent files, we check that the shape is the
127+ # same as the first file. If not, we raise an error.
121128 if self .omx_shape is None :
122129 self .omx_shape = tuple (
123130 int (i ) for i in omx_file .shape ()
@@ -127,13 +134,24 @@ def load_skim_info(self, state, skim_tag):
127134 int (i ) for i in omx_file .shape ()
128135 ), f"Mismatch shape { self .omx_shape } { omx_file .shape ()}
129136
137+ # Check that all the matrix names are unique across all the
138+ # omx files. This check is only looking at the name as stored in
139+ # the file, and is not processing any time period transformations.
140+ # If duplicate names are found, a warning is issued. This is not
141+ # a fatal error, but it is inefficient and may be symptom of a
142+ # deeper problem.
130143 for skim_name in omx_file .listMatrices ():
131144 if skim_name in self .omx_manifest :
132145 warnings .warn (
133146 f"duplicate skim '{ skim_name } { self .omx_manifest [skim_name ]} { omx_file .filename }
134147 )
135148 self .omx_manifest [skim_name ] = omx_file_path
136149
150+ # We load the offset map if it exists. This is expected to be
151+ # a 1D array of integers that that gives ID values for each TAZ
152+ # in the skims. ActivitySim expects there to be only one such
153+ # mapping, although it can appear multiple times (e.g. once in
154+ # each file).
137155 for m in omx_file .listMappings ():
138156 if self .offset_map is None :
139157 self .offset_map_name = m
@@ -146,21 +164,54 @@ def load_skim_info(self, state, skim_tag):
146164 f"Multiple mappings in omx file: { self .offset_map_name } { m }
147165 )
148166
149- # - omx_keys dict maps skim key to omx_key
150- # DISTWALK: DISTWALK
151- # ('DRV_COM_WLK_BOARDS', 'AM'): DRV_COM_WLK_BOARDS__AM, ...
167+ # Create the `omx_keys` mapping, which connects skim key to omx_key.
168+ # The skim key is either a single string that names a skim that is not
169+ # time-dependent, or a 2-tuple of strings which names a skim and a time
170+ # period. The omx_key is the original name of the skim in the omx file.
171+ # For non-time-dependent skims, the omx_key is the same as the skim key,
172+ # e.g. DISTWALK: DISTWALK. For time-dependent skims, the omx_key is the
173+ # skim key with the time period appended,
174+ # e.g. ('DRV_COM_WLK_BOARDS', 'AM'): DRV_COM_WLK_BOARDS__AM.
152175 self .omx_keys = dict ()
153176 for skim_name in self .omx_manifest .keys ():
154177 key1 , sep , key2 = skim_name .partition ("__" )
155-
156178 # - ignore composite tags not in dim3_tags_to_load
157179 if dim3_tags_to_load and sep and key2 not in dim3_tags_to_load :
180+ # If a skim is found that has a time period that is not one of
181+ # the known named time periods, a warning is issued, and that
182+ # skim is ignored. This is not a fatal error, but it may be a
183+ # symptom of a deeper problem.
184+ warnings .warn (f"skim '{ key1 } { key2 } )
158185 continue
159-
160186 skim_key = (key1 , key2 ) if sep else key1
161-
162187 self .omx_keys [skim_key ] = skim_name
163188
189+ # Create a skim_conflicts set, which identifies any skims that have both
190+ # time-dependent and time-agnostic versions. This condition in and of
191+ # itself is not a fatal error, as it is possible to have both types of skims
192+ # in the same data when using the legacy codebase. When using skim_dataset
193+ # instead of skim_dictionary (the former is required when using sharrow) this
194+ # condition is no longer allowed, although we can potentially recover if
195+ # the user has specified instructions that certain skim variables are not
196+ # to be loaded. The recovery option is checked later, in the skim_dataset
197+ # module, as that is where the skim variables are actually loaded.
198+ time_dependent_skims = defaultdict (set )
199+ time_agnostic_skims = set ()
200+ for k , v in self .omx_keys .items ():
201+ if isinstance (k , tuple ):
202+ time_dependent_skims [k [0 ]].add (v )
203+ else :
204+ time_agnostic_skims .add (k )
205+ self .skim_conflicts = {
206+ k : v for k , v in time_dependent_skims .items () if k in time_agnostic_skims
207+ }
208+ if self .skim_conflicts :
209+ msg = "some skims have both time-dependent and time-agnostic versions:"
210+ for k in self .skim_conflicts :
211+ msg += f"\n - { k }
212+ warnings .warn (msg )
213+
214+ # Count the number of skims in the omx file
164215 self .num_skims = len (self .omx_keys )
165216
166217 # - key1_subkeys dict maps key1 to dict of subkeys with that key1
0 commit comments