Merge pull request #122 from NatLabRockies/sa/integrate-gtfs-flex

Sa/integrate gtfs flex

Author: robfitzgerald

rust/Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "bambam-core",
     "bambam-gbfs",
     "bambam-gtfs",
+    "bambam-gtfs-flex",
     "bambam-omf",
     "bambam-osm",
     "bambam-py",

rust/bambam-core/src/util/geo_utils.rs

@@ -1,4 +1,4 @@
-use geo::Centroid;
+use geo::{Centroid, MapCoords};
 use geo::{Geometry, Point};
 use rstar::RTreeObject;
 use rstar::AABB;
@@ -30,3 +30,25 @@ pub fn get_centroid_as_envelope(geometry: &Geometry<f32>) -> Option<AABB<Point<f
         Geometry::Triangle(g) => Some(AABB::from_point(g.centroid())),
     }
 }
+
+/// attempt to convert a geometry from 64 bit to 32 bit floating point representation.
+/// this operation is destructive but warranted when working with lat/lon values and
+/// scaling RAM consumption for national runs.
+pub fn try_convert_f32(g: &Geometry<f64>) -> Result<Geometry<f32>, String> {
+    let (min, max) = (f32::MIN as f64, f32::MAX as f64);
+    g.try_map_coords(|geo::Coord { x, y }| {
+        if x < min || max < x {
+            Err(format!("could not express x value '{x}' as f32, exceeds range of possible values [{min}, {max}]"))
+        } else if y < min || max < y {
+            Err(format!("could not express y value '{y}' as f32, exceeds range of possible values [{min}, {max}]"))
+        } else {
+            let x32 = std::panic::catch_unwind(|| x as f32).map_err(|_| {
+                format!("could not express x value '{x}' as f32")
+            })?;
+            let y32 = std::panic::catch_unwind(|| y as f32).map_err(|_| {
+                format!("could not express y value '{x}' as f32")
+            })?;
+            Ok(geo::Coord { x: x32, y: y32 })
+        }
+    })
+}

rust/bambam-gtfs-flex/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "bambam-gtfs-flex"
+version = "0.2.4"
+edition = "2021"
+license = "BSD-3-Clause"
+description = "Simple GTFS-Flex scanner"
+repository = "https://github.com/NREL/bambam"
+readme = "README.md"
+
+[features]
+default = ["type4"]
+type4 = ["dep:gtfs-structures"]
+
+[dependencies]
+bambam-core = { version = "0.2.3", path = "../bambam-core" } 
+chrono = { workspace = true }
+clap = { workspace = true }
+csv = { workspace = true }
+geo = { workspace = true }
+geo-types = { workspace = true }
+geozero = { workspace = true }
+gtfs-structures = { workspace = true, optional = true }
+kdam = { workspace = true }
+ordered-float = { workspace = true }
+routee-compass-core = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+skiplist = { workspace = true }
+thiserror = { workspace = true }
+uom = { workspace = true }
+zip = { workspace = true }
+# No external dependencies needed for the simple version

rust/bambam-gtfs-flex/README.md

@@ -0,0 +1,69 @@
+# bambam-gtfs-flex
+A set of extensions that enable On-Demand Transit modeling in BAMBAM using [GTFS-Flex](https://gtfs.org/community/extensions/flex/) datasets.
+
+## GTFS Flex Service Types
+
+We have observed four types of services:
+
+1. **Within a Single Zone**  
+  Pickups and drop-offs at any two points within the same zone.
+
+2. **Across Multiple Zones**  
+  Trips are allowed between any points in different zones (no within-zone trips).
+
+3. **With Specified Stops**  
+  Similar to services (1) and (2), but pickups and drop-offs are allowed only at designated stops.
+
+4. **With Deviated Route**  
+  Vehicles follow a fixed route but can deviate to pick up or drop off between stops.
+
+## Model Design
+
+To track the state of a GTFS Flex trip, we store a few additional fields on the trip state. These fields and their implications in service types 1-3 are described below. Note: service type 4 is implemented via network dataset modifications and not in the search algorithm state.
+
+### Service Type 1: Within a Single Zone
+
+In this service type, trips are assigned a `src_zone_id` when they board. The trip may travel anywhere but may only treat locations within this zone as destinations. This requires a lookup table from `EdgeId -> ZoneId`.
+
+### Service Type 2: Across Multiple Zones
+
+In this service type, trips are assigned a `src_zone_id` and `departure_time` when they board. The trip may travel anywhere but may only treat particular locations as destinations. This requires the above `EdgeId -> ZoneId` lookup as well as a `(ZoneId, DepartureTime) -> [ZoneId]` lookup.
+
+### Service Type 3: With Specified Stops
+
+In this service type, trips are assigned a `src_zone_id` and `departure_time` when they board. Using the same lookups as (2), we additionally require an `EdgeId -> bool` mask function which further restricts where boarding and alighting may occur.
+
+### Service Type 4: With Deviated Route
+
+In this service type, we are actually running GTFS-style routing. However, we also need to modify some static weights based on the expected delays due to trip deviations. These weights should be modified during trip/model initialization but made fixed to ensure search correctness.
+
+## Processing GTFS Flex Feeds Using CLI
+
+To process GTFS Flex feeds, you can use the provided command-line interface (CLI) tool. Follow the steps below:
+
+1. **Install Dependencies**  
+  Ensure you have Rust installed on your system. If not, install it from [rustup.rs](https://rustup.rs/).
+
+2. **Build the Project**  
+  Navigate to the project directory and build the CLI tool:
+  ```bash
+  cd rust/bambam-gtfs-flex
+  cargo build --release
+  ```
+
+3. **Run the CLI Tool**  
+  Use the following command to process a GTFS Flex feed:
+  ```bash
+  ./target/release/bambam-gtfs-flex process-feeds ./src/test/assets 20240903 valid_zone.csv
+  ```
+  If you want to process GTFS-Flex feeds without completing `cargo build --release` in Step 2, you can simply use the following command:
+  ```bash
+  cd rust/bambam-gtfs-flex
+  cargo run -- process-feeds ./src/test/assets 20240903
+  ```
+  Replace `./src/test/assets` with the path to the folder where your GTFS-Flex feeds (.zip files) are located, `20240903` with the desired date in `YYYYMMDD` format for which you want to process the feeds, and `valid_zones.csv` (optional) with the name of the output CSV file, which will be written to the GTFS-Flex feeds directory.
+
+4. **Verify Output**  
+  After processing, the output directory will contain the processed valid zone CSV for the requested date, ready for use in BAMBAM.
+
+Refer to the project's documentation for more details on further usage and configuration.

rust/bambam-gtfs-flex/src/agency.rs

@@ -0,0 +1,91 @@
+use csv::ReaderBuilder;
+use serde::{self, Deserialize};
+use std::{fs::File, io, path::Path};
+use zip::ZipArchive;
+
+/// single row from agency.txt in a GTFS-Flex feed
+#[derive(Debug, Deserialize)]
+pub struct Agency {
+    /// unique agency identifier
+    pub agency_id: Option<String>,
+
+    /// agency URL
+    pub _agency_url: Option<String>,
+
+    /// primary language used by the agency
+    pub _agency_lang: Option<String>,
+
+    /// full agency name
+    pub _agency_name: Option<String>,
+
+    /// agency phone number
+    pub _agency_phone: Option<String>,
+
+    /// agency timezone
+    pub _agency_timezone: Option<String>,
+
+    /// URL to fare information
+    pub _agency_fare_url: Option<String>,
+
+    /// text-to-speech version of agency name
+    pub _tts_agency_name: Option<String>,
+}
+
+/// read `agency.txt` from a single GTFS-Flex zip file
+///
+/// streams data directly from the zip
+/// returns None if agency.txt is missing or duplicated
+/// returns typed Agency rows on success
+pub fn read_agency_from_flex(zip_path: &Path) -> io::Result<Option<Vec<Agency>>> {
+    // open the zip file
+    let file = File::open(zip_path)?;
+    let mut archive = ZipArchive::new(file)?;
+
+    // locate agency.txt inside the zip
+    let mut agency_name: Option<String> = None;
+
+    for i in 0..archive.len() {
+        let file_in_zip = archive.by_index(i)?;
+
+        if file_in_zip.name().ends_with("agency.txt") {
+            // do not allow multiple agency.txt files
+            if agency_name.is_some() {
+                eprintln!(
+                    "WARNING: Multiple agency.txt found in {:?}. Skipping ZIP.",
+                    zip_path
+                );
+                return Ok(None);
+            }
+
+            agency_name = Some(file_in_zip.name().to_string());
+        }
+    }
+
+    // handle missing agency.txt
+    let agency_name = match agency_name {
+        Some(name) => name,
+        None => {
+            println!("No agency.txt found in {:?}", zip_path);
+            return Ok(None);
+        }
+    };
+
+    // open agency.txt as streaming reader
+    let file_in_zip = archive.by_name(&agency_name)?;
+
+    // create CSV reader
+    let mut rdr = ReaderBuilder::new()
+        .has_headers(true)
+        .from_reader(file_in_zip);
+
+    // deserialize rows into Agency struct
+    let mut agencies = Vec::new();
+
+    for result in rdr.deserialize::<Agency>() {
+        let agency = result.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+
+        agencies.push(agency);
+    }
+
+    Ok(Some(agencies))
+}

rust/bambam-gtfs-flex/src/app/gtfs_flex_cli.rs

@@ -0,0 +1,31 @@
+use clap::{Parser, Subcommand};
+
+/// command line tool providing GTFS-Flex processing scripts
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+pub struct Cli {
+    #[command(subcommand)]
+    pub command: Commands,
+}
+
+#[derive(Subcommand, Debug)]
+pub enum Commands {
+    /// process GTFS-Flex feeds
+    #[command(name = "process-feeds")]
+    ProcessGtfsFlexFeeds(GTFSFLexCliArguments),
+}
+
+/// arguments for the process-gtfs-flex-feeds subcommand
+#[derive(Parser, Debug)]
+pub struct GTFSFLexCliArguments {
+    /// directory containing GTFS-Flex feeds to process
+    pub flex_dir: String,
+
+    /// date for which to process GTFS-Flex feeds (format: YYYYMMDD)
+    pub date_requested: String,
+
+    /// output CSV file name for valid zones
+    /// file will be created in the specified GTFS-Flex directory
+    #[arg(short, long, default_value = "valid-zones.csv")]
+    pub output_csv: String,
+}

rust/bambam-gtfs-flex/src/app/mod.rs

@@ -0,0 +1,3 @@
+mod gtfs_flex_cli;
+
+pub use gtfs_flex_cli::{Cli, Commands};

rust/bambam-gtfs-flex/src/calendar.rs

@@ -0,0 +1,97 @@
+use chrono::NaiveDate;
+use csv::ReaderBuilder;
+use serde::{self, Deserialize, Deserializer};
+use std::{fs::File, io, path::Path};
+use zip::ZipArchive;
+
+/// a single row from calendar.txt in a GTFS-Flex feed
+#[derive(Debug, Deserialize)]
+pub struct Calendar {
+    /// unique service identifier
+    pub service_id: String,
+
+    /// service availability by day (0 or 1)
+    pub monday: u8,
+    pub tuesday: u8,
+    pub wednesday: u8,
+    pub thursday: u8,
+    pub friday: u8,
+    pub saturday: u8,
+    pub sunday: u8,
+
+    /// service start date (YYYYMMDD)
+    #[serde(deserialize_with = "gtfs_flex_date")]
+    pub start_date: NaiveDate,
+
+    /// service end date (YYYYMMDD)
+    #[serde(deserialize_with = "gtfs_flex_date")]
+    pub end_date: NaiveDate,
+}
+
+/// deserialize GTFS-Flex dates in YYYYMMDD format
+fn gtfs_flex_date<'de, D>(deserializer: D) -> Result<NaiveDate, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s = String::deserialize(deserializer)?;
+    NaiveDate::parse_from_str(&s, "%Y%m%d").map_err(serde::de::Error::custom)
+}
+
+/// read calendar.txt from a single GTFS-Flex ZIP file
+///
+/// streams data directly from the ZIP
+/// returns None if calendar.txt is missing or duplicated
+/// returns typed Calendar rows on success
+pub fn read_calendar_from_flex(zip_path: &Path) -> io::Result<Option<Vec<Calendar>>> {
+    // open the zip file
+    let file = File::open(zip_path)?;
+    let mut archive = ZipArchive::new(file)?;
+
+    // locate calendar.txt
+    let mut calendar_name: Option<String> = None;
+
+    for i in 0..archive.len() {
+        let file_in_zip = archive.by_index(i)?;
+
+        if file_in_zip.name().ends_with("calendar.txt") {
+            // donot allow multiple calendar.txt files in a zip
+            if calendar_name.is_some() {
+                eprintln!(
+                    "WARNING: Multiple calendar.txt found in {:?}. Skipping ZIP.",
+                    zip_path
+                );
+                return Ok(None);
+            }
+
+            calendar_name = Some(file_in_zip.name().to_string());
+        }
+    }
+
+    // handle missing calendar.txt
+    let calendar_name = match calendar_name {
+        Some(name) => name,
+        None => {
+            println!("No calendar.txt found in {:?}", zip_path);
+            return Ok(None);
+        }
+    };
+
+    // open calendar.txt as a streaming reader
+    let file_in_zip = archive.by_name(&calendar_name)?;
+
+    // create a CSV reader
+    let mut rdr = ReaderBuilder::new()
+        .has_headers(true)
+        .from_reader(file_in_zip);
+
+    // deserialize each row into Calendar
+    let mut calendars = Vec::new();
+
+    for result in rdr.deserialize::<Calendar>() {
+        let calendar = result.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+
+        calendars.push(calendar);
+    }
+
+    Ok(Some(calendars))
+}