From 7b5af05914ec740a9ea3a0a2453880a6d197490d Mon Sep 17 00:00:00 2001 From: Yamil Essus Date: Tue, 16 Dec 2025 10:51:35 -0500 Subject: [PATCH 01/14] change `id` fields to required --- .../src/collection/record/transportation_connector.rs | 2 +- .../src/collection/record/transportation_segment.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/bambam-omf/src/collection/record/transportation_connector.rs b/rust/bambam-omf/src/collection/record/transportation_connector.rs index 455e847b..2a8581cf 100644 --- a/rust/bambam-omf/src/collection/record/transportation_connector.rs +++ b/rust/bambam-omf/src/collection/record/transportation_connector.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; /// identifier, geometry, bounding box, version, and data sources. #[derive(Debug, Serialize, Deserialize)] pub struct TransportationConnectorRecord { - id: Option, + pub id: String, #[serde(deserialize_with = "deserialize_geometry")] geometry: Option, bbox: OvertureMapsBbox, diff --git a/rust/bambam-omf/src/collection/record/transportation_segment.rs b/rust/bambam-omf/src/collection/record/transportation_segment.rs index 122e9ac5..22643b0e 100644 --- a/rust/bambam-omf/src/collection/record/transportation_segment.rs +++ b/rust/bambam-omf/src/collection/record/transportation_segment.rs @@ -12,7 +12,7 @@ use super::{OvertureMapsBbox, OvertureMapsNames, OvertureMapsSource}; /// and other attributes relevant to routing and mapping. #[derive(Debug, Serialize, Deserialize)] pub struct TransportationSegmentRecord { - id: Option, + id: String, #[serde(deserialize_with = "deserialize_geometry")] geometry: Option, bbox: OvertureMapsBbox, @@ -21,7 +21,7 @@ pub struct TransportationSegmentRecord { subtype: Option, class: Option, names: Option, - connectors: Option>, + pub connectors: Option>, routes: Option>, subclass_rules: Option>>, access_restrictions: Option>, @@ -50,7 +50,7 @@ impl TryFrom for TransportationSegmentRecord { } #[derive(Debug, Serialize, Deserialize)] -struct ConnectorReference { +pub struct ConnectorReference { connector_id: String, at: f64, } From f52e2b3385c0bd7224bf76bc4c44a91d90c94f2b Mon Sep 17 00:00:00 2001 From: Yamil Essus Date: Tue, 16 Dec 2025 16:07:38 -0500 Subject: [PATCH 02/14] first implementation of serialization of transport --- rust/bambam-omf/Cargo.toml | 6 + rust/bambam-omf/src/app/mod.rs | 3 + rust/bambam-omf/src/app/omf_app.rs | 56 +++++++ rust/bambam-omf/src/app/serialize_options.rs | 13 ++ rust/bambam-omf/src/bin/bambam_omf.rs | 8 + rust/bambam-omf/src/collection/error.rs | 8 + rust/bambam-omf/src/collection/mod.rs | 5 +- rust/bambam-omf/src/collection/record/mod.rs | 2 + .../record/transportation_collection.rs | 60 +++++++ .../record/transportation_connector.rs | 26 ++++ .../record/transportation_segment.rs | 29 +++- rust/bambam-omf/src/graph/mod.rs | 6 + rust/bambam-omf/src/graph/omf_graph.rs | 147 ++++++++++++++++++ rust/bambam-omf/src/graph/segment_split.rs | 65 ++++++++ rust/bambam-omf/src/graph/serialize_ops.rs | 50 ++++++ .../src/graph/vertex_serializable.rs | 19 +++ rust/bambam-omf/src/lib.rs | 2 + 17 files changed, 499 insertions(+), 6 deletions(-) create mode 100644 rust/bambam-omf/src/app/mod.rs create mode 100644 rust/bambam-omf/src/app/omf_app.rs create mode 100644 rust/bambam-omf/src/app/serialize_options.rs create mode 100644 rust/bambam-omf/src/bin/bambam_omf.rs create mode 100644 rust/bambam-omf/src/collection/record/transportation_collection.rs create mode 100644 rust/bambam-omf/src/graph/mod.rs create mode 100644 rust/bambam-omf/src/graph/omf_graph.rs create mode 100644 rust/bambam-omf/src/graph/segment_split.rs create mode 100644 rust/bambam-omf/src/graph/serialize_ops.rs create mode 100644 rust/bambam-omf/src/graph/vertex_serializable.rs diff --git a/rust/bambam-omf/Cargo.toml b/rust/bambam-omf/Cargo.toml index 15f295bf..fdb5d048 100644 --- a/rust/bambam-omf/Cargo.toml +++ b/rust/bambam-omf/Cargo.toml @@ -28,11 +28,16 @@ routee-compass-powertrain = { workspace = true } arrow = { workspace = true } chrono = { workspace = true } +clap = { workspace = true } csv = { workspace = true } +env_logger = { workspace = true } +flate2 = { workspace = true } futures = { workspace = true } geo = { workspace = true } geozero = { workspace = true } hex = { workspace = true } +itertools = { workspace = true } +kdam = { workspace = true } log = { workspace = true } object_store = { workspace = true } parquet = { workspace = true } @@ -43,3 +48,4 @@ serde_json = { workspace = true } serde_arrow = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +uom = { workspace = true } diff --git a/rust/bambam-omf/src/app/mod.rs b/rust/bambam-omf/src/app/mod.rs new file mode 100644 index 00000000..dbc554d8 --- /dev/null +++ b/rust/bambam-omf/src/app/mod.rs @@ -0,0 +1,3 @@ +mod omf_app; + +pub use omf_app::OmfApp; diff --git a/rust/bambam-omf/src/app/omf_app.rs b/rust/bambam-omf/src/app/omf_app.rs new file mode 100644 index 00000000..b5a90576 --- /dev/null +++ b/rust/bambam-omf/src/app/omf_app.rs @@ -0,0 +1,56 @@ +use std::path::Path; + +use clap::{Parser, Subcommand}; +use object_store::aws::AmazonS3; +use serde::{Deserialize, Serialize}; + +use crate::{ + collection::{ + ObjectStoreSource, OvertureMapsCollectionError, OvertureMapsCollectorConfig, + ReleaseVersion, RowFilterConfig, TransportationCollection, + }, + graph::OmfGraphVectorized, +}; + +/// command line tool for batch downloading and summarizing of GTFS archives +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +#[command(propagate_version = true)] +pub struct OmfApp { + #[command(subcommand)] + pub op: OmfOperation, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Subcommand)] +pub enum OmfOperation { + /// download all of the GTFS archives + Download, +} + +impl OmfOperation { + pub fn run(self) -> Result<(), OvertureMapsCollectionError> { + match self { + OmfOperation::Download {} => { + let collector = + OvertureMapsCollectorConfig::new(ObjectStoreSource::AmazonS3, 128).build()?; + let release = ReleaseVersion::Latest; + let row_filter_config = RowFilterConfig::Bbox { + xmin: -105.254, + xmax: -105.197, + ymin: 39.733, + ymax: 39.784, + }; + + let collection = TransportationCollection::try_from_collector( + collector, + release, + Some(row_filter_config), + )?; + let vectorized_graph = OmfGraphVectorized::try_from_collection(collection, 0)?; + vectorized_graph.write_compass(Path::new("./"), true)?; + + Ok(()) + } + } + } +} diff --git a/rust/bambam-omf/src/app/serialize_options.rs b/rust/bambam-omf/src/app/serialize_options.rs new file mode 100644 index 00000000..3b92408a --- /dev/null +++ b/rust/bambam-omf/src/app/serialize_options.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +struct OvertureSerializeOptions { + out_file: String, + scope: SerializeScope, +} + +#[derive(Debug, Serialize, Deserialize)] +enum SerializeScope { + Complete, + Compass, +} diff --git a/rust/bambam-omf/src/bin/bambam_omf.rs b/rust/bambam-omf/src/bin/bambam_omf.rs new file mode 100644 index 00000000..e76d49e8 --- /dev/null +++ b/rust/bambam-omf/src/bin/bambam_omf.rs @@ -0,0 +1,8 @@ +use bambam_omf::app::OmfApp; +use clap::Parser; + +fn main() { + env_logger::init(); + let args = OmfApp::parse(); + args.op.run().unwrap() +} diff --git a/rust/bambam-omf/src/collection/error.rs b/rust/bambam-omf/src/collection/error.rs index e496efa1..a3d129d7 100644 --- a/rust/bambam-omf/src/collection/error.rs +++ b/rust/bambam-omf/src/collection/error.rs @@ -30,4 +30,12 @@ pub enum OvertureMapsCollectionError { GroupMappingError(String), #[error("Processing records into opportunities failed: {0}")] ProcessingError(String), + #[error("Serializing record into compass format failed failed: {0}")] + SerializationError(String), + #[error("Segment connectors vector is invalid or not specified: {0}")] + InvalidSegmentConnectors(String), + #[error("Invalid or empty geometry: {0}")] + InvalidGeometry(String), + #[error("Error writing to csv: {0}")] + CsvWriteError(String), } diff --git a/rust/bambam-omf/src/collection/mod.rs b/rust/bambam-omf/src/collection/mod.rs index ba965a86..dc6e3c02 100644 --- a/rust/bambam-omf/src/collection/mod.rs +++ b/rust/bambam-omf/src/collection/mod.rs @@ -16,6 +16,9 @@ pub use filter::Bbox; pub use filter::RowFilter; pub use filter::RowFilterConfig; pub use object_source::ObjectStoreSource; -pub use record::{BuildingsRecord, OvertureRecord, OvertureRecordType, PlacesRecord}; +pub use record::{ + BuildingsRecord, OvertureRecord, OvertureRecordType, PlacesRecord, TransportationCollection, + TransportationConnectorRecord, TransportationSegmentRecord, +}; pub use taxonomy::{TaxonomyModel, TaxonomyModelBuilder}; pub use version::ReleaseVersion; diff --git a/rust/bambam-omf/src/collection/record/mod.rs b/rust/bambam-omf/src/collection/record/mod.rs index 703a357e..a145efa7 100644 --- a/rust/bambam-omf/src/collection/record/mod.rs +++ b/rust/bambam-omf/src/collection/record/mod.rs @@ -3,6 +3,7 @@ mod common; mod overture_record; mod place; mod record_type; +mod transportation_collection; mod transportation_connector; mod transportation_segment; @@ -10,6 +11,7 @@ pub use building::BuildingsRecord; pub use overture_record::OvertureRecord; pub use place::PlacesRecord; pub use record_type::OvertureRecordType; +pub use transportation_collection::TransportationCollection; pub use transportation_connector::TransportationConnectorRecord; pub use transportation_segment::TransportationSegmentRecord; diff --git a/rust/bambam-omf/src/collection/record/transportation_collection.rs b/rust/bambam-omf/src/collection/record/transportation_collection.rs new file mode 100644 index 00000000..e9ddece9 --- /dev/null +++ b/rust/bambam-omf/src/collection/record/transportation_collection.rs @@ -0,0 +1,60 @@ +use crate::collection::{ + OvertureMapsCollectionError, OvertureMapsCollector, OvertureRecord, OvertureRecordType, + ReleaseVersion, RowFilterConfig, TransportationConnectorRecord, TransportationSegmentRecord, +}; + +pub struct TransportationCollection { + pub connectors: Vec, + pub segments: Vec, +} + +impl TransportationCollection { + /// Use a pre-built collector and download configuration to + /// retrieve connectors and segments for a specified query + pub fn try_from_collector( + collector: OvertureMapsCollector, + release: ReleaseVersion, + row_filter_config: Option, + ) -> Result { + let connectors = collector + .collect_from_release( + release.clone(), + &OvertureRecordType::Connector, + row_filter_config.clone(), + )? + .into_iter() + .map(|record| match record { + OvertureRecord::Connector(transportation_connector_record) => { + Ok(transportation_connector_record) + } + _ => Err(OvertureMapsCollectionError::DeserializeTypeError(format!( + "expected connector type, got {:?}", + record + ))), + }) + .collect::, OvertureMapsCollectionError>>()?; + + let segments = collector + .collect_from_release( + release.clone(), + &OvertureRecordType::Segment, + row_filter_config.clone(), + )? + .into_iter() + .map(|record| match record { + OvertureRecord::Segment(transportation_connector_record) => { + Ok(transportation_connector_record) + } + _ => Err(OvertureMapsCollectionError::DeserializeTypeError(format!( + "expected segment type, got {:?}", + record + ))), + }) + .collect::, OvertureMapsCollectionError>>()?; + + Ok(Self { + connectors: connectors, + segments: segments, + }) + } +} diff --git a/rust/bambam-omf/src/collection/record/transportation_connector.rs b/rust/bambam-omf/src/collection/record/transportation_connector.rs index 2a8581cf..d024089d 100644 --- a/rust/bambam-omf/src/collection/record/transportation_connector.rs +++ b/rust/bambam-omf/src/collection/record/transportation_connector.rs @@ -3,6 +3,7 @@ use crate::collection::{OvertureMapsCollectionError, OvertureRecord}; use super::deserialize_geometry; use super::{OvertureMapsBbox, OvertureMapsSource}; use geo::Geometry; +use routee_compass_core::model::network::Vertex; use serde::{Deserialize, Serialize}; /// Represents a transportation connector record as defined in the Overture Maps Foundation schema. @@ -30,3 +31,28 @@ impl TryFrom for TransportationConnectorRecord { } } } + +impl TransportationConnectorRecord { + pub fn get_geometry(&self) -> Option<&Geometry> { + self.geometry.as_ref() + } + + pub fn try_to_vertex(&self, idx: usize) -> Result { + let geometry = + self.get_geometry() + .ok_or(OvertureMapsCollectionError::SerializationError(format!( + "Invalid or empty geometry {:?}", + self.get_geometry() + )))?; + + let (x, y) = match geometry { + Geometry::Point(point) => Ok(point.x_y()), + _ => Err(OvertureMapsCollectionError::SerializationError(format!( + "Incorrect geometry in ConnectorRecord: {:?}", + geometry + ))), + }?; + + Ok(Vertex::new(idx, x as f32, y as f32)) + } +} diff --git a/rust/bambam-omf/src/collection/record/transportation_segment.rs b/rust/bambam-omf/src/collection/record/transportation_segment.rs index 22643b0e..81f1ed62 100644 --- a/rust/bambam-omf/src/collection/record/transportation_segment.rs +++ b/rust/bambam-omf/src/collection/record/transportation_segment.rs @@ -1,4 +1,4 @@ -use geo::Geometry; +use geo::{Geometry, Haversine, Length}; use serde::{Deserialize, Serialize}; use crate::collection::{OvertureMapsCollectionError, OvertureRecord}; @@ -12,9 +12,9 @@ use super::{OvertureMapsBbox, OvertureMapsNames, OvertureMapsSource}; /// and other attributes relevant to routing and mapping. #[derive(Debug, Serialize, Deserialize)] pub struct TransportationSegmentRecord { - id: String, + pub id: String, #[serde(deserialize_with = "deserialize_geometry")] - geometry: Option, + pub geometry: Option, bbox: OvertureMapsBbox, version: i32, sources: Option>>, @@ -49,10 +49,29 @@ impl TryFrom for TransportationSegmentRecord { } } +impl TransportationSegmentRecord { + pub fn get_distance_at(&self, at: f64) -> Result { + let geometry = + self.geometry + .as_ref() + .ok_or(OvertureMapsCollectionError::InvalidGeometry(format!( + "empty geometry" + )))?; + + match geometry { + Geometry::LineString(line_string) => Ok(Haversine.length(line_string) * at), + _ => Err(OvertureMapsCollectionError::InvalidGeometry(format!( + "geometry was not a linestring {:?}", + geometry + ))), + } + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct ConnectorReference { - connector_id: String, - at: f64, + pub connector_id: String, + pub at: f64, } #[derive(Debug, Serialize, Deserialize)] diff --git a/rust/bambam-omf/src/graph/mod.rs b/rust/bambam-omf/src/graph/mod.rs new file mode 100644 index 00000000..7d0d2e04 --- /dev/null +++ b/rust/bambam-omf/src/graph/mod.rs @@ -0,0 +1,6 @@ +mod omf_graph; +mod segment_split; +mod serialize_ops; +mod vertex_serializable; + +pub use omf_graph::OmfGraphVectorized; diff --git a/rust/bambam-omf/src/graph/omf_graph.rs b/rust/bambam-omf/src/graph/omf_graph.rs new file mode 100644 index 00000000..6a706e32 --- /dev/null +++ b/rust/bambam-omf/src/graph/omf_graph.rs @@ -0,0 +1,147 @@ +use std::{collections::HashMap, fs::File, path::Path}; + +use csv::QuoteStyle; +use flate2::{write::GzEncoder, Compression}; +use kdam::tqdm; +use routee_compass_core::model::network::{Edge, EdgeConfig, EdgeId, Vertex}; + +use super::serialize_ops::get_connectors_mapping; +use crate::{ + collection::{OvertureMapsCollectionError, TransportationCollection}, + graph::{serialize_ops::get_connector_splits, vertex_serializable::VertexSerializable}, +}; + +#[derive(Debug)] +pub struct OmfGraphVectorized { + pub edge_list_id: usize, + pub vertices: Vec, + pub edges: Vec, + /// for each OMF ID, the vertex index + pub vertex_lookup: HashMap, +} + +impl OmfGraphVectorized { + pub fn try_from_collection( + collection: TransportationCollection, + edge_list_id: usize, + ) -> Result { + // Process initial set of connectors + let (vertices, vertex_mapping) = get_connectors_mapping(&collection.connectors)?; + + // Initialize result + let mut result = Self { + edge_list_id, + vertices, + edges: vec![], + vertex_lookup: vertex_mapping, + }; + + // Process segments + for segment in collection.segments.iter() { + // Compute all the splits + // Here is where we would define and run additional splits + get_connector_splits(segment)? + .iter() + .try_for_each(|split| split.split(&mut result, segment))?; + } + + Ok(result) + } + + pub fn add_vertex(&mut self, x: f32, y: f32) -> usize { + let current_vertex_id = self.vertices.len(); + // Should we just use UUID? + let new_name = format!("new-{}", current_vertex_id); + self.vertex_lookup.insert(new_name, current_vertex_id); + self.vertices + .push(Vertex::new(current_vertex_id, x as f32, y as f32)); + + current_vertex_id + } + + pub fn write_compass( + &self, + output_directory: &Path, + overwrite: bool, + ) -> Result<(), OvertureMapsCollectionError> { + let mut vertex_writer = create_writer( + output_directory, + "vertices-compass.csv.gz", + true, + QuoteStyle::Necessary, + overwrite, + ); + let mut edge_writer = create_writer( + output_directory, + "edges-compass.csv.gz", + true, + QuoteStyle::Necessary, + overwrite, + ); + + // Write vertices + let v_iter = tqdm!( + self.vertices.iter().enumerate(), + total = self.vertices.len(), + desc = "write vertex dataset" + ); + for (_, vertex) in v_iter { + if let Some(ref mut writer) = vertex_writer { + let vertex_ser = VertexSerializable::from(*vertex); + writer.serialize(vertex_ser).map_err(|e| { + OvertureMapsCollectionError::CsvWriteError(format!( + "Failed to write to vertices-compass.csv.gz: {}", + e + )) + })?; + } + } + + // Write Edges + let e_iter = tqdm!( + self.edges.iter().enumerate(), + total = self.edges.len(), + desc = "write edges dataset" + ); + for (edge_id, row) in e_iter { + if let Some(ref mut writer) = edge_writer { + let edge = EdgeConfig { + edge_id: EdgeId(edge_id), + src_vertex_id: row.src_vertex_id, + dst_vertex_id: row.dst_vertex_id, + distance: row.distance.get::(), + }; + writer.serialize(edge).map_err(|e| { + OvertureMapsCollectionError::CsvWriteError(format!( + "Failed to write to edges-compass.csv.gz: {}", + e + )) + })?; + } + } + Ok(()) + } +} + +/// helper function to build a filewriter for writing either .csv.gz or +/// .txt.gz files for compass datasets while respecting the user's overwrite +/// preferences and properly formatting WKT outputs. +fn create_writer( + directory: &Path, + filename: &str, + has_headers: bool, + quote_style: QuoteStyle, + overwrite: bool, +) -> Option>> { + let filepath = directory.join(filename); + if filepath.exists() && !overwrite { + return None; + } + let file = File::create(filepath).unwrap(); + let buffer = GzEncoder::new(file, Compression::default()); + let writer = csv::WriterBuilder::new() + .has_headers(has_headers) + .quote_style(quote_style) + .from_writer(buffer); + Some(writer) +} diff --git a/rust/bambam-omf/src/graph/segment_split.rs b/rust/bambam-omf/src/graph/segment_split.rs new file mode 100644 index 00000000..7cf04383 --- /dev/null +++ b/rust/bambam-omf/src/graph/segment_split.rs @@ -0,0 +1,65 @@ +use routee_compass_core::model::network::{Edge, EdgeId, EdgeListId, VertexId}; +use uom::si::f64::Length; + +use crate::{ + collection::{OvertureMapsCollectionError, TransportationSegmentRecord}, + graph::omf_graph::OmfGraphVectorized, +}; + +pub enum SegmentSplit { + ConnectorSplit { + connector_id_src: String, + at_src: f64, + connector_id_dst: String, + at_dst: f64, + }, +} + +impl SegmentSplit { + /// Modify in-place a vectorized graph according to a split logic + pub fn split( + &self, + vectorized_graph: &mut OmfGraphVectorized, + segment: &TransportationSegmentRecord, + ) -> Result<(), OvertureMapsCollectionError> { + match self { + SegmentSplit::ConnectorSplit { + connector_id_src, + at_src, + connector_id_dst, + at_dst, + } => { + // get src, dst VertexId via lookup into mapping->vertices + // Asumming `missing` is not valid in this case + let src_id = vectorized_graph.vertex_lookup.get(connector_id_src).ok_or( + OvertureMapsCollectionError::InvalidSegmentConnectors(format!( + "segment references unknown connector {}", + connector_id_src + )), + )?; + + let dst_id = vectorized_graph.vertex_lookup.get(connector_id_dst).ok_or( + OvertureMapsCollectionError::InvalidSegmentConnectors(format!( + "segment references unknown connector {}", + connector_id_dst + )), + )?; + + // create this edge, push onto edges + let distance = + segment.get_distance_at(*at_dst)? - segment.get_distance_at(*at_src)?; + let edge = Edge { + edge_list_id: EdgeListId(vectorized_graph.edge_list_id), + edge_id: EdgeId(vectorized_graph.edges.len()), + src_vertex_id: VertexId(*src_id), + dst_vertex_id: VertexId(*dst_id), + distance: Length::new::(distance), + }; + + vectorized_graph.edges.push(edge); + } + }; + + Ok(()) + } +} diff --git a/rust/bambam-omf/src/graph/serialize_ops.rs b/rust/bambam-omf/src/graph/serialize_ops.rs new file mode 100644 index 00000000..9da34358 --- /dev/null +++ b/rust/bambam-omf/src/graph/serialize_ops.rs @@ -0,0 +1,50 @@ +use geo::Geometry; +use itertools::Itertools; +use routee_compass_core::model::network::{Edge, EdgeId, EdgeListId, Vertex, VertexId}; +use std::collections::HashMap; + +use crate::{ + collection::{ + OvertureMapsCollectionError, OvertureRecord, TransportationConnectorRecord, + TransportationSegmentRecord, + }, + graph::segment_split::SegmentSplit, +}; + +pub fn get_connectors_mapping( + connectors: &Vec, +) -> Result<(Vec, HashMap), OvertureMapsCollectionError> { + let vertices = connectors + .iter() + .enumerate() + .map(|(idx, c)| c.try_to_vertex(idx)) + .collect::, OvertureMapsCollectionError>>()?; + + let mapping: HashMap = connectors + .iter() + .enumerate() + .map(|(idx, c)| (c.id.clone(), idx)) + .collect(); + + Ok((vertices, mapping)) +} + +pub fn get_connector_splits( + segment: &TransportationSegmentRecord, +) -> Result, OvertureMapsCollectionError> { + Ok(segment + .connectors + .as_ref() + .ok_or(OvertureMapsCollectionError::InvalidSegmentConnectors( + String::from("connectors is None"), + ))? + .iter() + .tuple_windows() + .map(|(src, dst)| SegmentSplit::ConnectorSplit { + connector_id_src: src.connector_id.clone(), + at_src: src.at, + connector_id_dst: dst.connector_id.clone(), + at_dst: dst.at, + }) + .collect::>()) +} diff --git a/rust/bambam-omf/src/graph/vertex_serializable.rs b/rust/bambam-omf/src/graph/vertex_serializable.rs new file mode 100644 index 00000000..52417118 --- /dev/null +++ b/rust/bambam-omf/src/graph/vertex_serializable.rs @@ -0,0 +1,19 @@ +use routee_compass_core::model::network::Vertex; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct VertexSerializable { + vertex_id: usize, + x: f32, + y: f32, +} + +impl From for VertexSerializable { + fn from(value: Vertex) -> Self { + Self { + vertex_id: value.vertex_id.0, + x: value.x(), + y: value.y(), + } + } +} diff --git a/rust/bambam-omf/src/lib.rs b/rust/bambam-omf/src/lib.rs index a3907ad8..a0651a2e 100644 --- a/rust/bambam-omf/src/lib.rs +++ b/rust/bambam-omf/src/lib.rs @@ -1 +1,3 @@ +pub mod app; pub mod collection; +pub mod graph; From fc3580922eed30526a0c174f1b91ad06a60da488 Mon Sep 17 00:00:00 2001 From: Yamil Essus Date: Tue, 16 Dec 2025 16:07:56 -0500 Subject: [PATCH 03/14] clippy --- rust/bambam-omf/src/app/omf_app.rs | 3 +-- .../src/collection/record/transportation_collection.rs | 10 ++++------ .../src/collection/record/transportation_connector.rs | 3 +-- .../src/collection/record/transportation_segment.rs | 7 ++----- rust/bambam-omf/src/graph/omf_graph.rs | 10 ++++------ rust/bambam-omf/src/graph/segment_split.rs | 6 ++---- rust/bambam-omf/src/graph/serialize_ops.rs | 5 ++--- 7 files changed, 16 insertions(+), 28 deletions(-) diff --git a/rust/bambam-omf/src/app/omf_app.rs b/rust/bambam-omf/src/app/omf_app.rs index b5a90576..312aea56 100644 --- a/rust/bambam-omf/src/app/omf_app.rs +++ b/rust/bambam-omf/src/app/omf_app.rs @@ -1,7 +1,6 @@ use std::path::Path; use clap::{Parser, Subcommand}; -use object_store::aws::AmazonS3; use serde::{Deserialize, Serialize}; use crate::{ @@ -30,7 +29,7 @@ pub enum OmfOperation { impl OmfOperation { pub fn run(self) -> Result<(), OvertureMapsCollectionError> { match self { - OmfOperation::Download {} => { + OmfOperation::Download => { let collector = OvertureMapsCollectorConfig::new(ObjectStoreSource::AmazonS3, 128).build()?; let release = ReleaseVersion::Latest; diff --git a/rust/bambam-omf/src/collection/record/transportation_collection.rs b/rust/bambam-omf/src/collection/record/transportation_collection.rs index e9ddece9..e4bf3150 100644 --- a/rust/bambam-omf/src/collection/record/transportation_collection.rs +++ b/rust/bambam-omf/src/collection/record/transportation_collection.rs @@ -28,8 +28,7 @@ impl TransportationCollection { Ok(transportation_connector_record) } _ => Err(OvertureMapsCollectionError::DeserializeTypeError(format!( - "expected connector type, got {:?}", - record + "expected connector type, got {record:?}" ))), }) .collect::, OvertureMapsCollectionError>>()?; @@ -46,15 +45,14 @@ impl TransportationCollection { Ok(transportation_connector_record) } _ => Err(OvertureMapsCollectionError::DeserializeTypeError(format!( - "expected segment type, got {:?}", - record + "expected segment type, got {record:?}" ))), }) .collect::, OvertureMapsCollectionError>>()?; Ok(Self { - connectors: connectors, - segments: segments, + connectors, + segments, }) } } diff --git a/rust/bambam-omf/src/collection/record/transportation_connector.rs b/rust/bambam-omf/src/collection/record/transportation_connector.rs index d024089d..969949d0 100644 --- a/rust/bambam-omf/src/collection/record/transportation_connector.rs +++ b/rust/bambam-omf/src/collection/record/transportation_connector.rs @@ -48,8 +48,7 @@ impl TransportationConnectorRecord { let (x, y) = match geometry { Geometry::Point(point) => Ok(point.x_y()), _ => Err(OvertureMapsCollectionError::SerializationError(format!( - "Incorrect geometry in ConnectorRecord: {:?}", - geometry + "Incorrect geometry in ConnectorRecord: {geometry:?}" ))), }?; diff --git a/rust/bambam-omf/src/collection/record/transportation_segment.rs b/rust/bambam-omf/src/collection/record/transportation_segment.rs index 81f1ed62..ef73d691 100644 --- a/rust/bambam-omf/src/collection/record/transportation_segment.rs +++ b/rust/bambam-omf/src/collection/record/transportation_segment.rs @@ -54,15 +54,12 @@ impl TransportationSegmentRecord { let geometry = self.geometry .as_ref() - .ok_or(OvertureMapsCollectionError::InvalidGeometry(format!( - "empty geometry" - )))?; + .ok_or(OvertureMapsCollectionError::InvalidGeometry("empty geometry".to_string()))?; match geometry { Geometry::LineString(line_string) => Ok(Haversine.length(line_string) * at), _ => Err(OvertureMapsCollectionError::InvalidGeometry(format!( - "geometry was not a linestring {:?}", - geometry + "geometry was not a linestring {geometry:?}" ))), } } diff --git a/rust/bambam-omf/src/graph/omf_graph.rs b/rust/bambam-omf/src/graph/omf_graph.rs index 6a706e32..5edb9068 100644 --- a/rust/bambam-omf/src/graph/omf_graph.rs +++ b/rust/bambam-omf/src/graph/omf_graph.rs @@ -51,10 +51,10 @@ impl OmfGraphVectorized { pub fn add_vertex(&mut self, x: f32, y: f32) -> usize { let current_vertex_id = self.vertices.len(); // Should we just use UUID? - let new_name = format!("new-{}", current_vertex_id); + let new_name = format!("new-{current_vertex_id}"); self.vertex_lookup.insert(new_name, current_vertex_id); self.vertices - .push(Vertex::new(current_vertex_id, x as f32, y as f32)); + .push(Vertex::new(current_vertex_id, x, y)); current_vertex_id } @@ -90,8 +90,7 @@ impl OmfGraphVectorized { let vertex_ser = VertexSerializable::from(*vertex); writer.serialize(vertex_ser).map_err(|e| { OvertureMapsCollectionError::CsvWriteError(format!( - "Failed to write to vertices-compass.csv.gz: {}", - e + "Failed to write to vertices-compass.csv.gz: {e}" )) })?; } @@ -113,8 +112,7 @@ impl OmfGraphVectorized { }; writer.serialize(edge).map_err(|e| { OvertureMapsCollectionError::CsvWriteError(format!( - "Failed to write to edges-compass.csv.gz: {}", - e + "Failed to write to edges-compass.csv.gz: {e}" )) })?; } diff --git a/rust/bambam-omf/src/graph/segment_split.rs b/rust/bambam-omf/src/graph/segment_split.rs index 7cf04383..937c07ca 100644 --- a/rust/bambam-omf/src/graph/segment_split.rs +++ b/rust/bambam-omf/src/graph/segment_split.rs @@ -33,15 +33,13 @@ impl SegmentSplit { // Asumming `missing` is not valid in this case let src_id = vectorized_graph.vertex_lookup.get(connector_id_src).ok_or( OvertureMapsCollectionError::InvalidSegmentConnectors(format!( - "segment references unknown connector {}", - connector_id_src + "segment references unknown connector {connector_id_src}" )), )?; let dst_id = vectorized_graph.vertex_lookup.get(connector_id_dst).ok_or( OvertureMapsCollectionError::InvalidSegmentConnectors(format!( - "segment references unknown connector {}", - connector_id_dst + "segment references unknown connector {connector_id_dst}" )), )?; diff --git a/rust/bambam-omf/src/graph/serialize_ops.rs b/rust/bambam-omf/src/graph/serialize_ops.rs index 9da34358..e36ff9fa 100644 --- a/rust/bambam-omf/src/graph/serialize_ops.rs +++ b/rust/bambam-omf/src/graph/serialize_ops.rs @@ -1,11 +1,10 @@ -use geo::Geometry; use itertools::Itertools; -use routee_compass_core::model::network::{Edge, EdgeId, EdgeListId, Vertex, VertexId}; +use routee_compass_core::model::network::Vertex; use std::collections::HashMap; use crate::{ collection::{ - OvertureMapsCollectionError, OvertureRecord, TransportationConnectorRecord, + OvertureMapsCollectionError, TransportationConnectorRecord, TransportationSegmentRecord, }, graph::segment_split::SegmentSplit, From c08c5cf9ebf6383a9e92aa53b32a0ddd58d3b8ce Mon Sep 17 00:00:00 2001 From: Yamil Essus Date: Tue, 16 Dec 2025 16:08:05 -0500 Subject: [PATCH 04/14] fmt --- .../src/collection/record/transportation_segment.rs | 4 +++- rust/bambam-omf/src/graph/omf_graph.rs | 3 +-- rust/bambam-omf/src/graph/serialize_ops.rs | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/bambam-omf/src/collection/record/transportation_segment.rs b/rust/bambam-omf/src/collection/record/transportation_segment.rs index ef73d691..9c013f02 100644 --- a/rust/bambam-omf/src/collection/record/transportation_segment.rs +++ b/rust/bambam-omf/src/collection/record/transportation_segment.rs @@ -54,7 +54,9 @@ impl TransportationSegmentRecord { let geometry = self.geometry .as_ref() - .ok_or(OvertureMapsCollectionError::InvalidGeometry("empty geometry".to_string()))?; + .ok_or(OvertureMapsCollectionError::InvalidGeometry( + "empty geometry".to_string(), + ))?; match geometry { Geometry::LineString(line_string) => Ok(Haversine.length(line_string) * at), diff --git a/rust/bambam-omf/src/graph/omf_graph.rs b/rust/bambam-omf/src/graph/omf_graph.rs index 5edb9068..987125ba 100644 --- a/rust/bambam-omf/src/graph/omf_graph.rs +++ b/rust/bambam-omf/src/graph/omf_graph.rs @@ -53,8 +53,7 @@ impl OmfGraphVectorized { // Should we just use UUID? let new_name = format!("new-{current_vertex_id}"); self.vertex_lookup.insert(new_name, current_vertex_id); - self.vertices - .push(Vertex::new(current_vertex_id, x, y)); + self.vertices.push(Vertex::new(current_vertex_id, x, y)); current_vertex_id } diff --git a/rust/bambam-omf/src/graph/serialize_ops.rs b/rust/bambam-omf/src/graph/serialize_ops.rs index e36ff9fa..d85f266c 100644 --- a/rust/bambam-omf/src/graph/serialize_ops.rs +++ b/rust/bambam-omf/src/graph/serialize_ops.rs @@ -4,8 +4,7 @@ use std::collections::HashMap; use crate::{ collection::{ - OvertureMapsCollectionError, TransportationConnectorRecord, - TransportationSegmentRecord, + OvertureMapsCollectionError, TransportationConnectorRecord, TransportationSegmentRecord, }, graph::segment_split::SegmentSplit, }; From 76225e41bd8a7be1dfe5197f30537c9ea76f3575 Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:14:36 -0700 Subject: [PATCH 05/14] Update rust/bambam-omf/src/collection/error.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- rust/bambam-omf/src/collection/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/bambam-omf/src/collection/error.rs b/rust/bambam-omf/src/collection/error.rs index a3d129d7..331a77db 100644 --- a/rust/bambam-omf/src/collection/error.rs +++ b/rust/bambam-omf/src/collection/error.rs @@ -30,7 +30,7 @@ pub enum OvertureMapsCollectionError { GroupMappingError(String), #[error("Processing records into opportunities failed: {0}")] ProcessingError(String), - #[error("Serializing record into compass format failed failed: {0}")] + #[error("Serializing record into compass format failed: {0}")] SerializationError(String), #[error("Segment connectors vector is invalid or not specified: {0}")] InvalidSegmentConnectors(String), From 0080a5ca17437dbf2467b2c2021fdb302b92fdd5 Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:14:53 -0700 Subject: [PATCH 06/14] Update rust/bambam-omf/src/graph/serialize_ops.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- rust/bambam-omf/src/graph/serialize_ops.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/bambam-omf/src/graph/serialize_ops.rs b/rust/bambam-omf/src/graph/serialize_ops.rs index d85f266c..bfe5c4ff 100644 --- a/rust/bambam-omf/src/graph/serialize_ops.rs +++ b/rust/bambam-omf/src/graph/serialize_ops.rs @@ -10,7 +10,7 @@ use crate::{ }; pub fn get_connectors_mapping( - connectors: &Vec, + connectors: &[TransportationConnectorRecord], ) -> Result<(Vec, HashMap), OvertureMapsCollectionError> { let vertices = connectors .iter() From ec1dcf2a978770bbab8df74f798d7d1a801c24f0 Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:15:01 -0700 Subject: [PATCH 07/14] Update rust/bambam-omf/src/graph/omf_graph.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- rust/bambam-omf/src/graph/omf_graph.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/bambam-omf/src/graph/omf_graph.rs b/rust/bambam-omf/src/graph/omf_graph.rs index 987125ba..909d4d48 100644 --- a/rust/bambam-omf/src/graph/omf_graph.rs +++ b/rust/bambam-omf/src/graph/omf_graph.rs @@ -101,10 +101,10 @@ impl OmfGraphVectorized { total = self.edges.len(), desc = "write edges dataset" ); - for (edge_id, row) in e_iter { + for (_edge_id, row) in e_iter { if let Some(ref mut writer) = edge_writer { let edge = EdgeConfig { - edge_id: EdgeId(edge_id), + edge_id: row.edge_id, src_vertex_id: row.src_vertex_id, dst_vertex_id: row.dst_vertex_id, distance: row.distance.get::(), From b65649ca93f2112c77bfac172df9834e2dea38e9 Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:16:52 -0700 Subject: [PATCH 08/14] Update rust/bambam-omf/src/graph/segment_split.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- rust/bambam-omf/src/graph/segment_split.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust/bambam-omf/src/graph/segment_split.rs b/rust/bambam-omf/src/graph/segment_split.rs index 937c07ca..68d516c4 100644 --- a/rust/bambam-omf/src/graph/segment_split.rs +++ b/rust/bambam-omf/src/graph/segment_split.rs @@ -44,6 +44,12 @@ impl SegmentSplit { )?; // create this edge, push onto edges + if at_dst < at_src { + return Err(OvertureMapsCollectionError::InvalidSegmentConnectors(format!( + "ConnectorSplit: at_dst ({}) < at_src ({}) for connectors {} -> {}", + at_dst, at_src, connector_id_src, connector_id_dst + ))); + } let distance = segment.get_distance_at(*at_dst)? - segment.get_distance_at(*at_src)?; let edge = Edge { From 6f776a5b764501268621a623e56bc60fc8fada94 Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:17:02 -0700 Subject: [PATCH 09/14] Update rust/bambam-omf/src/app/omf_app.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- rust/bambam-omf/src/app/omf_app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/bambam-omf/src/app/omf_app.rs b/rust/bambam-omf/src/app/omf_app.rs index 312aea56..26b2c66b 100644 --- a/rust/bambam-omf/src/app/omf_app.rs +++ b/rust/bambam-omf/src/app/omf_app.rs @@ -11,7 +11,7 @@ use crate::{ graph::OmfGraphVectorized, }; -/// command line tool for batch downloading and summarizing of GTFS archives +/// Command line tool for batch downloading and summarizing of OMF (Overture Maps Foundation) data #[derive(Parser)] #[command(author, version, about, long_about = None)] #[command(propagate_version = true)] From bee590a70bde6e9a60eddb89dab3872efe57f8bb Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:17:10 -0700 Subject: [PATCH 10/14] Update rust/bambam-omf/src/app/omf_app.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- rust/bambam-omf/src/app/omf_app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/bambam-omf/src/app/omf_app.rs b/rust/bambam-omf/src/app/omf_app.rs index 26b2c66b..bc4e9da2 100644 --- a/rust/bambam-omf/src/app/omf_app.rs +++ b/rust/bambam-omf/src/app/omf_app.rs @@ -22,7 +22,7 @@ pub struct OmfApp { #[derive(Debug, Clone, Serialize, Deserialize, Subcommand)] pub enum OmfOperation { - /// download all of the GTFS archives + /// download all of the OMF transportation data Download, } From 53b5790294c3be42fdeb25757e88775e06f3b31a Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:18:02 -0700 Subject: [PATCH 11/14] Update rust/bambam-omf/src/graph/omf_graph.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- rust/bambam-omf/src/graph/omf_graph.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/rust/bambam-omf/src/graph/omf_graph.rs b/rust/bambam-omf/src/graph/omf_graph.rs index 909d4d48..11b82b50 100644 --- a/rust/bambam-omf/src/graph/omf_graph.rs +++ b/rust/bambam-omf/src/graph/omf_graph.rs @@ -116,8 +116,22 @@ impl OmfGraphVectorized { })?; } } + // Explicitly flush the writers to ensure all data is written + if let Some(ref mut writer) = vertex_writer { + writer.flush().map_err(|e| { + OvertureMapsCollectionError::CsvWriteError(format!( + "Failed to flush vertices-compass.csv.gz: {e}" + )) + })?; + } + if let Some(ref mut writer) = edge_writer { + writer.flush().map_err(|e| { + OvertureMapsCollectionError::CsvWriteError(format!( + "Failed to flush edges-compass.csv.gz: {e}" + )) + })?; + } Ok(()) - } } /// helper function to build a filewriter for writing either .csv.gz or From a3080f8eabb7f36a8853ae42342d038e5b580cd6 Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:19:23 -0700 Subject: [PATCH 12/14] Update rust/bambam-omf/src/collection/record/transportation_collection.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../src/collection/record/transportation_collection.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/bambam-omf/src/collection/record/transportation_collection.rs b/rust/bambam-omf/src/collection/record/transportation_collection.rs index e4bf3150..53fdd6e5 100644 --- a/rust/bambam-omf/src/collection/record/transportation_collection.rs +++ b/rust/bambam-omf/src/collection/record/transportation_collection.rs @@ -41,8 +41,8 @@ impl TransportationCollection { )? .into_iter() .map(|record| match record { - OvertureRecord::Segment(transportation_connector_record) => { - Ok(transportation_connector_record) + OvertureRecord::Segment(transportation_segment_record) => { + Ok(transportation_segment_record) } _ => Err(OvertureMapsCollectionError::DeserializeTypeError(format!( "expected segment type, got {record:?}" From b77b6c16cfcbf03230d2513e2958b77e54c055d0 Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:20:28 -0700 Subject: [PATCH 13/14] copilot bug --- rust/bambam-omf/src/graph/omf_graph.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/bambam-omf/src/graph/omf_graph.rs b/rust/bambam-omf/src/graph/omf_graph.rs index 11b82b50..d2e59047 100644 --- a/rust/bambam-omf/src/graph/omf_graph.rs +++ b/rust/bambam-omf/src/graph/omf_graph.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, fs::File, path::Path}; use csv::QuoteStyle; use flate2::{write::GzEncoder, Compression}; use kdam::tqdm; -use routee_compass_core::model::network::{Edge, EdgeConfig, EdgeId, Vertex}; +use routee_compass_core::model::network::{Edge, EdgeConfig, Vertex}; use super::serialize_ops::get_connectors_mapping; use crate::{ @@ -132,6 +132,7 @@ impl OmfGraphVectorized { })?; } Ok(()) + } } /// helper function to build a filewriter for writing either .csv.gz or From 1a4d6e247c14f4f8a4f73a594caba1ae4a69ffea Mon Sep 17 00:00:00 2001 From: Rob Fitzgerald Date: Thu, 18 Dec 2025 13:20:33 -0700 Subject: [PATCH 14/14] fmt --- rust/bambam-omf/src/graph/segment_split.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rust/bambam-omf/src/graph/segment_split.rs b/rust/bambam-omf/src/graph/segment_split.rs index 68d516c4..832dde1c 100644 --- a/rust/bambam-omf/src/graph/segment_split.rs +++ b/rust/bambam-omf/src/graph/segment_split.rs @@ -45,10 +45,12 @@ impl SegmentSplit { // create this edge, push onto edges if at_dst < at_src { - return Err(OvertureMapsCollectionError::InvalidSegmentConnectors(format!( - "ConnectorSplit: at_dst ({}) < at_src ({}) for connectors {} -> {}", - at_dst, at_src, connector_id_src, connector_id_dst - ))); + return Err(OvertureMapsCollectionError::InvalidSegmentConnectors( + format!( + "ConnectorSplit: at_dst ({}) < at_src ({}) for connectors {} -> {}", + at_dst, at_src, connector_id_src, connector_id_dst + ), + )); } let distance = segment.get_distance_at(*at_dst)? - segment.get_distance_at(*at_src)?;