Moved files around. Adjusted requirements.txt

Author: jcpitre

.github/workflows/build-test.yml

@@ -8,6 +8,9 @@ on:
       - "functions/**"
       - ".github/workflows/web-*.yml"
   workflow_call:
+  push:
+    branches: [ 779-generate-new-sourcescsv-from-postgresql-database ]
+
 
 env:
   python_version: '3.11'

api/src/common/common.py

@@ -1,9 +1,15 @@
+from geoalchemy2 import WKTElement
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.query import Query
+from sqlalchemy.orm.strategy_options import _AbstractLoad
 
-from common.common import apply_bounding_filtering, get_joinedload_options
-from common.error_handling import raise_internal_http_validation_error
+# from common.common import apply_bounding_filtering, get_joinedload_options
+from common.error_handling import (
+    raise_internal_http_validation_error,
+    invalid_bounding_method,
+    invalid_bounding_coordinates,
+)
 from database.database import Database
 from database_gen.sqlacodegen_models import (
     Gtfsdataset,
@@ -12,6 +18,7 @@
     Validationreport,
     Gtfsrealtimefeed,
     Entitytype,
+    Feed,
 )
 
 from feeds.filters.gtfs_feed_filter import GtfsFeedFilter, LocationFilter
@@ -133,3 +140,66 @@ def get_gtfs_rt_feeds_query(
         .offset(offset)
     )
     return feed_query
+
+
+def get_joinedload_options() -> [_AbstractLoad]:
+    """Returns common joinedload options for feeds queries."""
+    return [joinedload(Feed.locations), joinedload(Feed.externalids), joinedload(Feed.redirectingids)]
+
+
+def apply_bounding_filtering(
+    query: Query,
+    bounding_latitudes: str,
+    bounding_longitudes: str,
+    bounding_filter_method: str,
+) -> Query:
+    """Create a new query based on the bounding parameters."""
+
+    if not bounding_latitudes or not bounding_longitudes or not bounding_filter_method:
+        return query
+
+    if (
+        len(bounding_latitudes_tokens := bounding_latitudes.split(",")) != 2
+        or len(bounding_longitudes_tokens := bounding_longitudes.split(",")) != 2
+    ):
+        raise_internal_http_validation_error(
+            invalid_bounding_coordinates.format(bounding_latitudes, bounding_longitudes)
+        )
+    min_latitude, max_latitude = bounding_latitudes_tokens
+    min_longitude, max_longitude = bounding_longitudes_tokens
+    try:
+        min_latitude = float(min_latitude)
+        max_latitude = float(max_latitude)
+        min_longitude = float(min_longitude)
+        max_longitude = float(max_longitude)
+    except ValueError:
+        raise_internal_http_validation_error(
+            invalid_bounding_coordinates.format(bounding_latitudes, bounding_longitudes)
+        )
+
+    points = [
+        (min_longitude, min_latitude),
+        (min_longitude, max_latitude),
+        (max_longitude, max_latitude),
+        (max_longitude, min_latitude),
+        (min_longitude, min_latitude),
+    ]
+    wkt_polygon = f"POLYGON(({', '.join(f'{lon} {lat}' for lon, lat in points)}))"
+    bounding_box = WKTElement(
+        wkt_polygon,
+        srid=Gtfsdataset.bounding_box.type.srid,
+    )
+
+    if bounding_filter_method == "partially_enclosed":
+        return query.filter(
+            or_(
+                Gtfsdataset.bounding_box.ST_Overlaps(bounding_box),
+                Gtfsdataset.bounding_box.ST_Contains(bounding_box),
+            )
+        )
+    elif bounding_filter_method == "completely_enclosed":
+        return query.filter(bounding_box.ST_Covers(Gtfsdataset.bounding_box))
+    elif bounding_filter_method == "disjoint":
+        return query.filter(Gtfsdataset.bounding_box.ST_Disjoint(bounding_box))
+    else:
+        raise raise_internal_http_validation_error(invalid_bounding_method.format(bounding_filter_method))

api/src/common/db_utils.py

@@ -243,7 +243,7 @@ def get_gtfs_feeds(
         except InternalHTTPException as e:
             # get_gtfs_feeds_query cannot throw HTTPException since it's part of fastapi and it's
             # not necessarily deployed (e.g. for python functions). Instead it throws an InternalHTTPException
-            # that needs to be converted to HTTPException before being thrown.
+            # that needs to be converted to HTTPException before being returned to the calling api.
             raise convert_exception(e)
 
         return self._get_response(feed_query, GtfsFeedImpl)

api/src/feeds/impl/feeds_api_impl.py

@@ -6,7 +6,7 @@
   "memory": "256Mi",
   "trigger_http": true,
   "include_folders": ["helpers", "dataset_service"],
-  "include_common_folders": ["utils", "database_gen", "common", "database", "feeds/filters"],
+  "include_api_folders": ["database_gen", "common", "database", "feeds/filters", "utils"],
   "secret_environment_variables": [
     {
       "key": "FEEDS_DATABASE_URL"

functions-python/create_csv/function_config.json

@@ -9,6 +9,8 @@ requests~=2.32.3
 attrs~=23.1.0
 pluggy~=1.3.0
 certifi~=2024.7.4
+python-dotenv==1.0.0
+fastapi-filter[sqlalchemy]==1.0.0
 
 # SQL Alchemy and Geo Alchemy
 SQLAlchemy==2.0.23
@@ -18,3 +20,6 @@ geoalchemy2==0.14.7
 google-cloud-pubsub
 google-cloud-datastore
 cloudevents~=1.10.1
+
+# Additional packages for this function
+pandas