Merge pull request #72 from NREL/rjf/mode-edgelists

Rjf/mode edgelists

Author: robfitzgerald

.gitignore

@@ -14,4 +14,4 @@ configuration/test_gtfs_config_boulder_gtfs.toml
 denver_co/
 boulder_co/
 denver_rtd/
-
+out/

README.md

@@ -24,6 +24,7 @@ First, install using cargo (via [rustup](rustup.rs)): `cargo build --release --m
   - **bambam_util** - runs specific support utilities
   - **bambam_gtfs** - [GTFS](https://gtfs.org/documentation/schedule/reference/) analysis and import script
   - **bambam_osm** - [OpenStreetMap](https://www.openstreetmap.org/) import script
+  - **bambam_omf** - [OvertureMaps](https://docs.overturemaps.org/) import script
 
 We can list the command arguments for bambam (will document app as "RouteE Compass"):
 
@@ -95,8 +96,8 @@ $ RUST_LOG=info ./rust/target/release/bambam --config-file configuration/boulder
 - [ ] R API
 - [ ] OvertureMaps network import
 - [ ] methodological improvements for walk/bike/drive realism
-- [ ] transit-mode travel using GTFS Schedule data networks
-- [ ] multimodal route planning
+- [x] transit-mode travel using GTFS Schedule data networks
+- [x] multimodal route planning
 
 # License
 

configuration/bambam-omf/travel-mode-filter.json

@@ -0,0 +1,43 @@
+{
+  "edge_lists": [
+    {
+      "mode": "walk",
+      "filter": [
+        { "type": "subtype", "subtype": "road" },
+        { 
+          "type": "class", 
+          "classes": ["secondary", "tertiary", "residential", "living_street", "unclassified", "service", "pedestrian", "footway", "steps", "path", "track", "bridleway", "unknown"], 
+          "ignore_unset": true,
+          "behavior": "include"
+        },
+        { "type": "access_mode", "modes": ["foot"] }
+      ]
+    },
+    {
+      "mode": "bike",
+      "filter": [
+        { "type": "subtype", "subtype": "road" },
+        { 
+          "type": "class", 
+          "classes": ["primary", "secondary", "tertiary", "residential", "living_street", "unclassified", "service", "pedestrian", "path", "track", "cycleway", "bridleway", "unknown"], 
+          "ignore_unset": true,
+          "behavior": "include"
+        },
+        { "type": "access_mode", "modes": ["bicycle"] }
+      ]
+    },
+    {
+      "mode": "drive",
+      "filter": [
+        { "type": "subtype", "subtype": "road" },
+        { 
+          "type": "class", 
+          "classes": ["motorway", "trunk", "primary", "secondary", "tertiary", "residential", "living_street", "unclassified", "service", "unknown"], 
+          "ignore_unset": true,
+          "behavior": "include"
+        },
+        { "type": "access_mode", "modes": ["motor_vehicle", "car", "truck", "motorcycle"] }
+      ]
+    }
+  ]
+}

rust/Cargo.toml

@@ -42,6 +42,7 @@ kdam = "0.6.2"
 log = "0.4.19"
 num-traits = "0.2.19"
 object_store = { "version" = "0.12.0", features = ["aws"] }
+opening-hours-syntax = "1.1.5"
 ordered-float = { version = "5.1.0", features = ["serde"] }
 osmio = "0.14.0"
 osmpbf = "0.3.4"

rust/bambam-omf/Cargo.toml

@@ -29,6 +29,7 @@ routee-compass-powertrain = { workspace = true }
 arrow = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true }
+config = { workspace = true }
 csv = { workspace = true }
 env_logger = { workspace = true }
 flate2 = { workspace = true }
@@ -41,6 +42,7 @@ kdam = { workspace = true }
 log = { workspace = true }
 object_store = { workspace = true }
 parquet = { workspace = true }
+opening-hours-syntax = { workspace = true }
 ordered-float = { workspace = true }
 rayon = { workspace = true }
 reqwest = { workspace = true }
@@ -50,3 +52,4 @@ serde_arrow = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 uom = { workspace = true }
+wkb = { workspace = true }

rust/bambam-omf/src/app/cli_bbox.rs

@@ -0,0 +1,67 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct CliBoundingBox {
+    pub xmin: f32,
+    pub xmax: f32,
+    pub ymin: f32,
+    pub ymax: f32,
+}
+
+pub fn parse_bbox(s: &str) -> Result<CliBoundingBox, String> {
+    let parts: Vec<&str> = s.split(',').collect();
+    if parts.len() != 4 {
+        return Err(format!("expected format: xmin,xmax,ymin,ymax, got: {}", s));
+    }
+
+    let xmin = parse_lon(parts[0])?;
+    let xmax = parse_lon(parts[1])?;
+
+    let ymin = parse_lat(parts[2])?;
+    let ymax = parse_lat(parts[3])?;
+
+    if !(xmin < xmax) {
+        Err(format!(
+            "bbox: xmin must be less than xmax, but found [{xmin},{xmax}]"
+        ))
+    } else if !(ymin < ymax) {
+        Err(format!(
+            "bbox: ymin must be less than ymax, but found [{ymin},{ymax}]"
+        ))
+    } else {
+        Ok(CliBoundingBox {
+            xmin,
+            xmax,
+            ymin,
+            ymax,
+        })
+    }
+}
+
+impl std::fmt::Display for CliBoundingBox {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{},{},{},{}", self.xmin, self.xmax, self.ymin, self.ymax)
+    }
+}
+
+fn parse_lat(lat: &str) -> Result<f32, String> {
+    parse_num(lat, -90.0, 90.0).map_err(|e| format!("invalid latitude: {e}"))
+}
+
+fn parse_lon(lat: &str) -> Result<f32, String> {
+    parse_num(lat, -180.0, 180.0).map_err(|e| format!("invalid longitude: {e}"))
+}
+
+fn parse_num(s: &str, min: f32, max: f32) -> Result<f32, String> {
+    let v = s
+        .trim()
+        .parse::<f32>()
+        .map_err(|_| format!("not a number: {s}"))?;
+    if v < min || max < v {
+        Err(format!(
+            "number '{v}' is not valid, must be in range [{min},{max}]"
+        ))
+    } else {
+        Ok(v)
+    }
+}

rust/bambam-omf/src/app/mod.rs

@@ -1,3 +1,6 @@
+mod cli_bbox;
 mod omf_app;
 
 pub use omf_app::OmfApp;
+pub mod network;
+pub use cli_bbox::CliBoundingBox;

rust/bambam-omf/src/app/network.rs

@@ -0,0 +1,81 @@
+use std::path::Path;
+
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    app::CliBoundingBox,
+    collection::{
+        filter::TravelModeFilter, ObjectStoreSource, OvertureMapsCollectionError,
+        OvertureMapsCollectorConfig, ReleaseVersion, TransportationCollection,
+    },
+    graph::OmfGraphVectorized,
+    util,
+};
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct NetworkEdgeListConfiguration {
+    pub mode: String,
+    pub filter: Vec<TravelModeFilter>,
+}
+
+/// runs an OMF network import using the provided configuration.
+pub fn run(
+    bbox: Option<&CliBoundingBox>,
+    modes: &[NetworkEdgeListConfiguration],
+    output_directory: &Path,
+    local_source: Option<&Path>,
+    write_json: bool,
+) -> Result<(), OvertureMapsCollectionError> {
+    let collection: TransportationCollection = match local_source {
+        Some(src_path) => read_local(src_path),
+        None => run_collector(bbox),
+    }?;
+
+    if write_json {
+        util::fs::create_dirs(output_directory)?;
+        collection.to_json(output_directory)?;
+    }
+
+    let vectorized_graph = OmfGraphVectorized::new(&collection, modes)?;
+    vectorized_graph.write_compass(output_directory, true)?;
+
+    Ok(())
+}
+
+fn read_local(path: &Path) -> Result<TransportationCollection, OvertureMapsCollectionError> {
+    let contents = std::fs::read(path).map_err(|e| OvertureMapsCollectionError::ReadError {
+        path: path.to_owned(),
+        message: e.to_string(),
+    })?;
+    let collection =
+        serde_json::from_slice::<TransportationCollection>(&contents).map_err(|e| {
+            OvertureMapsCollectionError::ReadError {
+                path: path.to_owned(),
+                message: format!("failed to deserialize from JSON: {e}"),
+            }
+        })?;
+    Ok(collection)
+}
+
+/// retrieve a TransportationCollection from a URL.
+fn run_collector(
+    bbox_arg: Option<&CliBoundingBox>,
+) -> Result<TransportationCollection, OvertureMapsCollectionError> {
+    let object_store = ObjectStoreSource::AmazonS3;
+    let batch_size = 128;
+    let collector = OvertureMapsCollectorConfig::new(object_store, batch_size).build()?;
+    let release = ReleaseVersion::Latest;
+    let bbox = bbox_arg.ok_or_else(|| {
+        let msg = String::from("must provide bbox argument for download");
+        OvertureMapsCollectionError::InvalidUserInput(msg)
+    })?;
+    log::info!(
+        "running OMF import with
+        object store {object_store:?}
+        batch size {batch_size}
+        release {release}
+        (xmin,xmax,ymin,ymax): {bbox}"
+    );
+
+    TransportationCollection::try_from_collector(collector, release, Some(bbox.into()))
+}

rust/bambam-omf/src/app/omf_app.rs

@@ -1,15 +1,12 @@
 use std::path::Path;
 
 use clap::{Parser, Subcommand};
-use routee_compass_core::model::network::EdgeListId;
+use config::{Config, File};
 use serde::{Deserialize, Serialize};
 
 use crate::{
-    collection::{
-        ObjectStoreSource, OvertureMapsCollectionError, OvertureMapsCollectorConfig,
-        ReleaseVersion, RowFilterConfig, TransportationCollection,
-    },
-    graph::OmfGraphVectorized,
+    app::{cli_bbox::parse_bbox, network::NetworkEdgeListConfiguration, CliBoundingBox},
+    collection::OvertureMapsCollectionError,
 };
 
 /// Command line tool for batch downloading and summarizing of OMF (Overture Maps Foundation) data
@@ -24,41 +21,64 @@ pub struct OmfApp {
 #[derive(Debug, Clone, Serialize, Deserialize, Subcommand)]
 pub enum OmfOperation {
     /// download all of the OMF transportation data
-    Download {
+    Network {
+        /// configuration file defining how the network is imported and separated
+        /// into mode-specific edge lists.
+        #[arg(short, long)]
+        configuration_file: String,
+
         /// location on disk to write output files. if not provided,
         /// use the current working directory.
         #[arg(short, long)]
         output_directory: Option<String>,
+
+        /// use a stored raw data export from a previous run of OmfOperation::Network
+        /// which is a JSON file containing a TransportationCollection.
+        #[arg(short, long)]
+        local_source: Option<String>,
+
+        /// write the raw OMF dataset as a JSON blob to the output directory.
+        #[arg(short, long)]
+        store_raw: bool,
+
+        /// bounding box to filter data (format: xmin,xmax,ymin,ymax)
+        #[arg(short, long, value_parser = parse_bbox, allow_hyphen_values(true))]
+        bbox: Option<CliBoundingBox>,
     },
 }
 
 impl OmfOperation {
     pub fn run(&self) -> Result<(), OvertureMapsCollectionError> {
         match self {
-            OmfOperation::Download { output_directory } => {
-                let collector =
-                    OvertureMapsCollectorConfig::new(ObjectStoreSource::AmazonS3, 128).build()?;
-                let release = ReleaseVersion::Latest;
-                let row_filter_config = RowFilterConfig::Bbox {
-                    xmin: -105.254,
-                    xmax: -105.197,
-                    ymin: 39.733,
-                    ymax: 39.784,
-                };
-
-                let collection = TransportationCollection::try_from_collector(
-                    collector,
-                    release,
-                    Some(row_filter_config),
-                )?;
-                let vectorized_graph = OmfGraphVectorized::new(collection, EdgeListId(0))?;
-                let output_path = match output_directory {
-                    Some(o) => Path::new(o),
+            OmfOperation::Network {
+                configuration_file,
+                output_directory,
+                local_source,
+                store_raw,
+                bbox,
+            } => {
+                let filepath = Path::new(configuration_file);
+                let config = Config::builder()
+                    .add_source(File::from(filepath))
+                    .build()
+                    .map_err(|e| {
+                        let msg = format!("file '{configuration_file}' produced error: {e}");
+                        OvertureMapsCollectionError::InvalidUserInput(msg)
+                    })?;
+                let network_config = config
+                    .get::<Vec<NetworkEdgeListConfiguration>>("edge_lists")
+                    .map_err(|e| {
+                        let msg = format!(
+                            "error reading 'edge_lists' key in '{configuration_file}': {e}"
+                        );
+                        OvertureMapsCollectionError::InvalidUserInput(msg)
+                    })?;
+                let outdir = match output_directory {
+                    Some(out) => Path::new(out),
                     None => Path::new(""),
                 };
-                vectorized_graph.write_compass(output_path, true)?;
-
-                Ok(())
+                let local = local_source.as_ref().map(Path::new);
+                crate::app::network::run(bbox.as_ref(), &network_config, outdir, local, *store_raw)
             }
         }
     }