Rjf/investigate omf panic

rust/bambam-omf/src/app/network.rs

@@ -39,11 +39,13 @@ pub fn run(
     modes: &[NetworkEdgeListConfiguration],
     output_directory: &Path,
     local_source: Option<&Path>,
+    object_store: ObjectStoreSource,
+    batch_size: usize,
     write_json: bool,
 ) -> Result<(), OvertureMapsCollectionError> {
     let collection: TransportationCollection = match local_source {
         Some(src_path) => read_local(src_path),
-        None => run_collector(bbox),
+        None => run_collector(bbox, object_store, batch_size),
     }?;
 
     if write_json {
@@ -75,9 +77,9 @@ fn read_local(path: &Path) -> Result<TransportationCollection, OvertureMapsColle
 /// retrieve a TransportationCollection from a URL.
 fn run_collector(
     bbox_arg: Option<&CliBoundingBox>,
+    object_store: ObjectStoreSource,
+    batch_size: usize,
 ) -> Result<TransportationCollection, OvertureMapsCollectionError> {
-    let object_store = ObjectStoreSource::AmazonS3;
-    let batch_size = 128;
     let collector = OvertureMapsCollectorConfig::new(object_store, batch_size).build()?;
     let release = ReleaseVersion::Latest;
     let bbox = bbox_arg.ok_or_else(|| {

rust/bambam-omf/src/app/omf_app.rs

@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::{
     app::{cli_bbox::parse_bbox, network::NetworkEdgeListConfiguration, CliBoundingBox},
-    collection::OvertureMapsCollectionError,
+    collection::{ObjectStoreSource, OvertureMapsCollectionError},
 };
 
 /// Command line tool for batch downloading and summarizing of OMF (Overture Maps Foundation) data
@@ -37,6 +37,14 @@ pub enum OmfOperation {
         #[arg(short, long)]
         local_source: Option<String>,
 
+        /// size of each download batch. this may be modified for network performance
+        #[arg(long, default_value = "128")]
+        batch_size: usize,
+
+        /// location where the file is stored
+        #[arg(long, value_enum, default_value_t = ObjectStoreSource::AmazonS3)]
+        object_store: ObjectStoreSource,
+
         /// write the raw OMF dataset as a JSON blob to the output directory.
         #[arg(short, long)]
         store_raw: bool,
@@ -54,6 +62,8 @@ impl OmfOperation {
                 configuration_file,
                 output_directory,
                 local_source,
+                batch_size,
+                object_store,
                 store_raw,
                 bbox,
             } => {
@@ -78,7 +88,15 @@ impl OmfOperation {
                     None => Path::new(""),
                 };
                 let local = local_source.as_ref().map(Path::new);
-                crate::app::network::run(bbox.as_ref(), &network_config, outdir, local, *store_raw)
+                crate::app::network::run(
+                    bbox.as_ref(),
+                    &network_config,
+                    outdir,
+                    local,
+                    *object_store,
+                    *batch_size,
+                    *store_raw,
+                )
             }
         }
     }

rust/bambam-omf/src/collection/collector.rs

@@ -72,7 +72,7 @@ impl OvertureMapsCollector {
             })?;
 
         // Process all common prefixes to find latest date
-        let mut version_tuples: Vec<(NaiveDate, String)> = common_prefixes
+        let (_, latest) = common_prefixes
             .iter()
             .filter_map(|p| {
                 let clean_str = p.to_string().strip_prefix("release/")?.to_string();
@@ -82,16 +82,14 @@ impl OvertureMapsCollector {
 
                 Some((date_part, clean_str))
             })
-            .collect();
-
-        // Get the latest date from tuples
-        version_tuples.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
-        version_tuples
-            .pop()
-            .ok_or(OvertureMapsCollectionError::ConnectionError(String::from(
-                "No version tuples generated while getting latest version string",
-            )))
-            .map(|(_, v)| v)
+            .max_by_key(|(date, _)| *date)
+            .ok_or_else(|| {
+                OvertureMapsCollectionError::MetadataError(String::from(
+                    "latest /release not found",
+                ))
+            })?;
+
+        Ok(latest)
     }
 
     pub fn collect_from_path(

rust/bambam-omf/src/collection/object_source.rs

@@ -1,13 +1,17 @@
+use clap::ValueEnum;
 use object_store::{aws::AmazonS3Builder, ObjectStore};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 
 use super::OvertureMapsCollectionError;
 
-#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+#[derive(Debug, Serialize, Deserialize, Copy, Clone, ValueEnum)]
 pub enum ObjectStoreSource {
+    #[serde(rename = "s3")]
     AmazonS3,
+    #[serde(rename = "azure")]
     Azure,
+    #[serde(rename = "fs")]
     FileSystem,
 }
 
@@ -29,3 +33,14 @@ impl ObjectStoreSource {
         }
     }
 }
+
+impl std::fmt::Display for ObjectStoreSource {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let s = match self {
+            ObjectStoreSource::AmazonS3 => "s3",
+            ObjectStoreSource::Azure => "azure",
+            ObjectStoreSource::FileSystem => "fs",
+        };
+        write!(f, "{s}")
+    }
+}

rust/bambam-omf/src/collection/record/during_expression.rs

@@ -0,0 +1,28 @@
+use opening_hours_syntax::rules::OpeningHoursExpression;
+use serde::{Deserialize, Serialize};
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum DuringExpression {
+    #[serde(with = "opening_hours_codec")]
+    Osm(OpeningHoursExpression),
+    Unexpected(String),
+}
+
+mod opening_hours_codec {
+    use opening_hours_syntax::rules::OpeningHoursExpression;
+    use serde::Deserialize;
+    pub fn serialize<S>(t: &OpeningHoursExpression, s: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        s.serialize_str(&t.to_string())
+    }
+    pub fn deserialize<'de, D>(d: D) -> Result<OpeningHoursExpression, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let s = String::deserialize(d)?;
+        opening_hours_syntax::parse(&s).map_err(serde::de::Error::custom)
+    }
+}

rust/bambam-omf/src/collection/record/mod.rs

@@ -1,5 +1,6 @@
 mod building;
 mod common;
+mod during_expression;
 mod overture_record;
 mod place;
 mod record_type;

rust/bambam-omf/src/collection/record/transportation_segment.rs

@@ -1,13 +1,14 @@
 use std::fmt::{self, Debug};
 
 use geo::{Coord, Geometry, Haversine, InterpolatableLine, Length, LineString};
-use opening_hours_syntax::rules::OpeningHoursExpression;
 use routee_compass_core::model::unit::SpeedUnit;
 use serde::{Deserialize, Serialize};
 use uom::si::f64::Velocity;
 
 use super::{geometry_wkb_codec, OvertureMapsBbox, OvertureMapsNames, OvertureMapsSource};
-use crate::collection::{OvertureMapsCollectionError, OvertureRecord};
+use crate::collection::{
+    record::during_expression::DuringExpression, OvertureMapsCollectionError, OvertureRecord,
+};
 
 /// Represents a transportation segment record in the Overture Maps schema.
 /// This struct contains information about a segment of transportation infrastructure,
@@ -592,11 +593,11 @@ pub struct SegmentAccessRestrictionWhen {
     /// in the OSM opening hours specification:
     /// see <https://wiki.openstreetmap.org/wiki/Key:opening_hours/specification>
     #[serde(
-        with = "opening_hours_codec",
+        // with = "opening_hours_codec",
         skip_serializing_if = "Option::is_none",
         default = "default_none"
     )]
-    pub during: Option<OpeningHoursExpression>,
+    pub during: Option<DuringExpression>,
     /// Enumerates possible travel headings along segment geometry.
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub heading: Option<SegmentHeading>,
@@ -616,32 +617,32 @@ pub struct SegmentAccessRestrictionWhen {
     pub vehicle: Option<Vec<SegmentAccessRestrictionWhenVehicle>>,
 }
 
-mod opening_hours_codec {
-    use opening_hours_syntax::rules::OpeningHoursExpression;
-    use serde::Deserialize;
-
-    pub fn serialize<S>(t: &Option<OpeningHoursExpression>, s: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        match t {
-            Some(expr) => s.serialize_str(&expr.to_string()),
-            None => s.serialize_none(),
-        }
-    }
-    pub fn deserialize<'de, D>(d: D) -> Result<Option<OpeningHoursExpression>, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        let s: Option<&str> = Option::deserialize(d)?;
-        match s {
-            Some(text) => opening_hours_syntax::parse(text)
-                .map(Some)
-                .map_err(serde::de::Error::custom),
-            None => Ok(None),
-        }
-    }
-}
+// mod opening_hours_codec {
+//     use opening_hours_syntax::rules::OpeningHoursExpression;
+//     use serde::Deserialize;
+
+//     pub fn serialize<S>(t: &Option<OpeningHoursExpression>, s: S) -> Result<S::Ok, S::Error>
+//     where
+//         S: serde::Serializer,
+//     {
+//         match t {
+//             Some(expr) => s.serialize_str(&expr.to_string()),
+//             None => s.serialize_none(),
+//         }
+//     }
+//     pub fn deserialize<'de, D>(d: D) -> Result<Option<OpeningHoursExpression>, D::Error>
+//     where
+//         D: serde::Deserializer<'de>,
+//     {
+//         let s: Option<&str> = Option::deserialize(d)?;
+//         match s {
+//             Some(text) => opening_hours_syntax::parse(text)
+//                 .map(Some)
+//                 .map_err(serde::de::Error::custom),
+//             None => Ok(None),
+//         }
+//     }
+// }
 
 impl SegmentAccessRestrictionWhen {
     pub fn contains_mode(&self, mode: &SegmentMode) -> bool {

rust/bambam-omf/src/main.rs

@@ -0,0 +1,8 @@
+use bambam_omf::{app::OmfApp, collection::OvertureMapsCollectionError};
+use clap::Parser;
+
+fn main() -> Result<(), OvertureMapsCollectionError> {
+    env_logger::init();
+    let args = OmfApp::parse();
+    args.op.run()
+}