feat: Add bounding box and pmtiles information at feed level (#1355)

Author: cka-y

api/src/feeds/impl/models/gtfs_feed_impl.py

@@ -1,3 +1,4 @@
+from feeds.impl.models.bounding_box_impl import BoundingBoxImpl
 from feeds.impl.models.feed_impl import FeedImpl
 from shared.database_gen.sqlacodegen_models import Gtfsfeed as GtfsfeedOrm
 from feeds.impl.models.latest_dataset_impl import LatestDatasetImpl
@@ -26,4 +27,8 @@ def from_orm(cls, feed: GtfsfeedOrm | None) -> GtfsFeed | None:
             (dataset for dataset in feed.gtfsdatasets if dataset is not None and dataset.latest), None
         )
         gtfs_feed.latest_dataset = LatestDatasetImpl.from_orm(latest_dataset)
+        gtfs_feed.bounding_box = BoundingBoxImpl.from_orm(feed.bounding_box)
+        gtfs_feed.visualization_dataset_id = (
+            feed.visualization_dataset.stable_id if feed.visualization_dataset else None
+        )
         return gtfs_feed

api/src/shared/common/db_utils.py

@@ -86,6 +86,7 @@ def get_gtfs_feeds_query(
             contains_eager(Gtfsfeed.gtfsdatasets)
             .joinedload(Gtfsdataset.validation_reports)
             .joinedload(Validationreport.features),
+            joinedload(Gtfsfeed.visualization_dataset),
             *get_joinedload_options(),
         ).order_by(Gtfsfeed.provider, Gtfsfeed.stable_id)
 
@@ -182,6 +183,7 @@ def get_all_gtfs_feeds(
                     contains_eager(Gtfsfeed.gtfsdatasets)
                     .joinedload(Gtfsdataset.validation_reports)
                     .joinedload(Validationreport.features),
+                    joinedload(Gtfsfeed.visualization_dataset),
                     *get_joinedload_options(include_extracted_location_entities=True),
                 )
             )

api/tests/integration/cascade_delete/test_cascade_delete.py

@@ -119,8 +119,10 @@ def test_delete_feed_cascadeto_gtfsdataset(test_database):
 
     with test_database.start_db_session() as session:
         feed = Feed(id="f1")
+        session.add(feed)
+        session.flush()
         dataset = Gtfsdataset(id="d1", feed_id="f1")
-        session.add_all([feed, dataset])
+        session.add(dataset)
         session.commit()
 
         delete_and_assert(

docs/DatabaseCatalogAPI.yaml

@@ -472,6 +472,13 @@ components:
               $ref: "#/components/schemas/Locations"
             latest_dataset:
               $ref: "#/components/schemas/LatestDataset"
+            bounding_box:
+              $ref: "#/components/schemas/BoundingBox"
+            visualization_dataset_id:
+              description: >
+                The dataset ID of the dataset used to compute the visualization files.
+              type: string
+              example: mdb-1210-202402121801
 
     GbfsFeed:
       allOf:

functions-python/pmtiles_builder/src/main.py

@@ -24,7 +24,11 @@
 import subprocess
 import tempfile
 from enum import Enum
+
+import flask
+import functions_framework
 from google.cloud import storage
+from sqlalchemy.orm import Session
 
 from csv_cache import (
     CsvCache,
@@ -35,12 +39,11 @@
     AGENCY_FILE,
     SHAPES_FILE,
 )
-from shared.helpers.runtime_metrics import track_metrics
-from shared.helpers.logger import get_logger, init_logger
 from gtfs_stops_to_geojson import convert_stops_to_geojson
-
-import flask
-import functions_framework
+from shared.database_gen.sqlacodegen_models import Gtfsdataset, Gtfsfeed
+from shared.helpers.logger import get_logger, init_logger
+from shared.helpers.runtime_metrics import track_metrics
+from shared.database.database import with_db_session
 
 init_logger()
 
@@ -198,6 +201,7 @@ def build_pmtiles(self):
 
         files_to_upload = ["routes.pmtiles", "stops.pmtiles", "routes.json"]
         self._upload_files_to_gcs(files_to_upload)
+        self._update_database()
 
         return self.OperationStatus.SUCCESS, "success"
 
@@ -521,3 +525,30 @@ def _load_agencies(self):
             agencies[agency_id] = agency_name
 
         return agencies
+
+    @with_db_session
+    def _update_database(self, db_session: Session = None):
+        dataset = (
+            db_session.query(Gtfsdataset)
+            .filter(Gtfsdataset.stable_id == self.dataset_stable_id)
+            .one_or_none()
+        )
+        if not dataset:
+            self.logger.error(
+                "Dataset %s not found in database, cannot update pmtiles_generated.",
+                self.dataset_stable_id,
+            )
+            return
+
+        # fetch the subclass row that shares the same PK as Feed
+        gtfsfeed = db_session.get(Gtfsfeed, dataset.feed_id)
+        if not gtfsfeed:
+            self.logger.error(
+                "Gtfsfeed(id=%s) not found (but Feed exists) — cannot set visualization_dataset.",
+                dataset.feed_id,
+            )
+            return
+
+        # set the relationship on the subclass
+        gtfsfeed.visualization_dataset = dataset
+        db_session.commit()

functions-python/reverse_geolocation/src/reverse_geolocation_processor.py

@@ -213,7 +213,7 @@ def get_storage_client():
 @with_db_session
 @track_metrics(metrics=("time", "memory", "cpu"))
 def update_dataset_bounding_box(
-    dataset_id: str, stops_df: pd.DataFrame, logger: logging.Logger, db_session: Session
+    dataset_id: str, stops_df: pd.DataFrame, db_session: Session
 ) -> shapely.Polygon:
     """
     Update the bounding box of the dataset using the stops DataFrame.
@@ -240,7 +240,15 @@ def update_dataset_bounding_box(
     )
     if not gtfs_dataset:
         raise ValueError(f"Dataset {dataset_id} does not exist in the database.")
+    gtfs_feed = db_session.get(Gtfsfeed, gtfs_dataset.feed_id)
+    if not gtfs_feed:
+        raise ValueError(
+            f"GTFS feed for dataset {dataset_id} does not exist in the database."
+        )
+    gtfs_feed.bounding_box = bounding_box
+    gtfs_feed.bounding_box_dataset = gtfs_dataset
     gtfs_dataset.bounding_box = bounding_box
+
     return to_shape(bounding_box)
 
 
@@ -323,7 +331,7 @@ def reverse_geolocation_process(
 
     try:
         # Update the bounding box of the dataset
-        bounding_box = update_dataset_bounding_box(dataset_id, stops_df, logger)
+        bounding_box = update_dataset_bounding_box(dataset_id, stops_df)
 
         location_groups = reverse_geolocation(
             strategy=strategy,

functions-python/reverse_geolocation/tests/test_reverse_geolocation_processor.py

@@ -375,7 +375,7 @@ def test_update_dataset_bounding_box_success(self, db_session):
 
         # Call the function
         bounding_box = update_dataset_bounding_box(
-            dataset_id, stops_df, db_session=db_session, logger=logger
+            dataset_id, stops_df, db_session=db_session
         )
 
         # Expected bounding box: POLYGON((30 10, 40 10, 40 20, 30 20, 30 10))

liquibase/changelog.xml

@@ -66,5 +66,6 @@
     <include file="changes/feat_1265_cascade_delete.sql" relativeToChangelogFile="true"/>
     <include file="changes/feat_1259.sql" relativeToChangelogFile="true"/>
     <include file="changes/feat_1260.sql" relativeToChangelogFile="true"/>
+    <include file="changes/feat_1333.sql" relativeToChangelogFile="true"/>
     <include file="changes/feat_pt_152.sql" relativeToChangelogFile="true"/>
 </databaseChangeLog>

liquibase/changes/feat_1333.sql

@@ -0,0 +1,41 @@
+-- Drop old columns if they exist (cleanup for consistency / schema reset)
+ALTER TABLE Gtfsfeed DROP COLUMN IF EXISTS visualization_dataset_id;
+ALTER TABLE Gtfsfeed DROP COLUMN IF EXISTS bounding_box_dataset_id;
+ALTER TABLE Gtfsfeed DROP COLUMN IF EXISTS bounding_box;
+
+-- Add new columns with proper definitions
+-- visualization_dataset_id: references the dataset used for UI/visualization
+ALTER TABLE Gtfsfeed
+  ADD COLUMN visualization_dataset_id VARCHAR(255) REFERENCES Gtfsdataset(id);
+
+-- bounding_box_dataset_id: references the dataset that provided the bounding box
+ALTER TABLE Gtfsfeed
+  ADD COLUMN bounding_box_dataset_id VARCHAR(255) REFERENCES Gtfsdataset(id);
+
+-- bounding_box: stores the geometry for the feed’s bounding box (WGS84 / SRID 4326)
+ALTER TABLE Gtfsfeed
+  ADD COLUMN bounding_box geometry(Polygon, 4326);
+
+-- Populate the bounding box info
+-- For each feed, find the most recent dataset that has a bounding box
+-- (ROW_NUMBER() = 1 keeps only the latest per feed_id)
+WITH latest AS (
+  SELECT
+    id,
+    feed_id,
+    bounding_box,
+    ROW_NUMBER() OVER (
+      PARTITION BY feed_id
+      ORDER BY downloaded_at DESC NULLS LAST, id DESC
+    ) AS rn
+  FROM Gtfsdataset
+  WHERE bounding_box IS NOT NULL
+)
+UPDATE Gtfsfeed gf
+SET
+  bounding_box_dataset_id = l.id,       -- link feed to the chosen dataset
+  bounding_box            = l.bounding_box -- copy its bounding box geometry
+FROM latest l
+WHERE l.rn = 1                          -- keep only the latest dataset per feed
+  AND gf.id = l.feed_id;                -- match feed with its dataset
+

web-app/src/app/screens/Feed/components/FullMapView.tsx

@@ -15,9 +15,9 @@ import {
   StyledMapControlPanel,
 } from '../Map.styles';
 import {
-  selectBoundingBoxFromLatestDataset,
   selectGtfsDatasetRoutesJson,
   selectGtfsDatasetRouteTypes,
+  selectGtfsFeedBoundingBox,
   selectLatestDatasetsData,
 } from '../../../store/selectors';
 import { useSelector } from 'react-redux';
@@ -40,7 +40,7 @@ export default function FullMapView(): React.ReactElement {
     useState<boolean>(false);
   const { config } = useRemoteConfig();
   const latestDataset = useSelector(selectLatestDatasetsData);
-  const boundingBox = useSelector(selectBoundingBoxFromLatestDataset);
+  const boundingBox = useSelector(selectGtfsFeedBoundingBox);
   const routes = useSelector(selectGtfsDatasetRoutesJson);
   const routeTypes = useSelector(selectGtfsDatasetRouteTypes);
   const dispatch = useAppDispatch();