Merge branch 'main' into 1024-load-tests-are-failing-with-gcp-token-message

qcdyx · qcdyx · commit cbb6cfff4472 · 2025-03-24T14:15:26.000-04:00
diff --git a/api/src/feeds/impl/feeds_api_impl.py b/api/src/feeds/impl/feeds_api_impl.py
@@ -14,7 +14,6 @@
     Gtfsfeed,
     Gtfsrealtimefeed,
     Location,
-    Validationreport,
     Entitytype,
 )
 from shared.feed_filters.feed_filter import FeedFilter
@@ -129,27 +128,11 @@ def get_gtfs_feed(self, id: str, db_session: Session) -> GtfsFeed:
             raise_http_error(404, gtfs_feed_not_found.format(id))
 
     @staticmethod
-    def _get_gtfs_feed(stable_id: str, db_session: Session) -> Optional[Gtfsfeed]:
-        results = (
-            FeedFilter(
-                stable_id=stable_id,
-                status=None,
-                provider__ilike=None,
-                producer_url__ilike=None,
-            )
-            .filter(db_session.query(Gtfsfeed))
-            .filter(
-                or_(
-                    Gtfsfeed.operational_status == "published",
-                    not is_user_email_restricted(),  # Allow all feeds to be returned if the user is not restricted
-                )
-            )
-            .options(
-                joinedload(Gtfsfeed.gtfsdatasets)
-                .joinedload(Gtfsdataset.validation_reports)
-                .joinedload(Validationreport.notices),
-                *get_joinedload_options(),
-            )
+    def _get_gtfs_feed(
+        stable_id: str, db_session: Session, include_options_for_joinedload: bool = True
+    ) -> Optional[Gtfsfeed]:
+        results = get_gtfs_feeds_query(
+            db_session=db_session, stable_id=stable_id, include_options_for_joinedload=include_options_for_joinedload
         ).all()
         if len(results) == 0:
             return None
@@ -173,22 +156,7 @@ def get_gtfs_feed_datasets(
             raise_http_validation_error(invalid_date_message.format("downloaded_after"))
 
         # First make sure the feed exists. If not it's an error 404
-        feed = (
-            FeedFilter(
-                stable_id=gtfs_feed_id,
-                status=None,
-                provider__ilike=None,
-                producer_url__ilike=None,
-            )
-            .filter(Database().get_query_model(db_session, Gtfsfeed))
-            .filter(
-                or_(
-                    Feed.operational_status == "published",
-                    not is_user_email_restricted(),  # Allow all feeds to be returned if the user is not restricted
-                )
-            )
-            .first()
-        )
+        feed = self._get_gtfs_feed(gtfs_feed_id, db_session, include_options_for_joinedload=False)
 
         if not feed:
             raise_http_error(404, f"Feed with id {gtfs_feed_id} not found")
diff --git a/api/src/shared/common/db_utils.py b/api/src/shared/common/db_utils.py
@@ -25,23 +25,25 @@
 
 
 def get_gtfs_feeds_query(
-    limit: int | None,
-    offset: int | None,
-    provider: str | None,
-    producer_url: str | None,
-    country_code: str | None,
-    subdivision_name: str | None,
-    municipality: str | None,
-    dataset_latitudes: str | None,
-    dataset_longitudes: str | None,
-    bounding_filter_method: str | None,
+    db_session: Session,
+    stable_id: str | None = None,
+    limit: int | None = None,
+    offset: int | None = None,
+    provider: str | None = None,
+    producer_url: str | None = None,
+    country_code: str | None = None,
+    subdivision_name: str | None = None,
+    municipality: str | None = None,
+    dataset_latitudes: str | None = None,
+    dataset_longitudes: str | None = None,
+    bounding_filter_method: str | None = None,
     is_official: bool = False,
     include_wip: bool = False,
-    db_session: Session = None,
+    include_options_for_joinedload: bool = True,
 ) -> Query[any]:
     """Get the DB query to use to retrieve the GTFS feeds.."""
     gtfs_feed_filter = GtfsFeedFilter(
-        stable_id=None,
+        stable_id=stable_id,
         provider__ilike=provider,
         producer_url__ilike=producer_url,
         location=LocationFilter(
@@ -65,12 +67,13 @@ def get_gtfs_feeds_query(
     if not include_wip:
         feed_query = feed_query.filter(Gtfsfeed.operational_status == "published")
 
-    feed_query = feed_query.options(
-        contains_eager(Gtfsfeed.gtfsdatasets)
-        .joinedload(Gtfsdataset.validation_reports)
-        .joinedload(Validationreport.notices),
-        *get_joinedload_options(),
-    ).order_by(Gtfsfeed.provider, Gtfsfeed.stable_id)
+    if include_options_for_joinedload:
+        feed_query = feed_query.options(
+            contains_eager(Gtfsfeed.gtfsdatasets)
+            .joinedload(Gtfsdataset.validation_reports)
+            .joinedload(Validationreport.notices),
+            *get_joinedload_options(),
+        ).order_by(Gtfsfeed.provider, Gtfsfeed.stable_id)
     if is_official:
         feed_query = feed_query.filter(Feed.official)
     feed_query = feed_query.limit(limit).offset(offset)
@@ -100,9 +103,11 @@ def get_all_gtfs_feeds(
         stable_ids = (f.stable_id for f in batch)
         yield from (
             db_session.query(Gtfsfeed)
+            .outerjoin(Gtfsfeed.gtfsdatasets)
             .filter(Gtfsfeed.stable_id.in_(stable_ids))
+            .filter((Gtfsdataset.latest) | (Gtfsdataset.id == None))  # noqa: E711
             .options(
-                joinedload(Gtfsfeed.gtfsdatasets)
+                contains_eager(Gtfsfeed.gtfsdatasets)
                 .joinedload(Gtfsdataset.validation_reports)
                 .joinedload(Validationreport.features),
                 *get_joinedload_options(),
diff --git a/functions-python/export_csv/src/main.py b/functions-python/export_csv/src/main.py
@@ -251,15 +251,16 @@ def get_feed_csv_data(feed: Feed):
     redirect_ids = []
     redirect_comments = []
     # Add concatenated redirect IDs
-    if feed.redirectingids:
-        for redirect in feed.redirectingids:
+    sorted_redirects = sorted(feed.redirectingids, key=lambda x: x.target.stable_id)
+    if sorted_redirects:
+        for redirect in sorted_redirects:
             if redirect and redirect.target and redirect.target.stable_id:
                 stripped_id = redirect.target.stable_id.strip()
                 if stripped_id:
                     redirect_ids.append(stripped_id)
                     redirect_comment = redirect.redirect_comment or ""
                     redirect_comments.append(redirect_comment)
-
+    redirect_ids = sorted(redirect_ids)
     redirect_ids_str = "|".join(redirect_ids)
     redirect_comments_str = "|".join(redirect_comments)
 
diff --git a/functions-python/export_csv/tests/conftest.py b/functions-python/export_csv/tests/conftest.py
@@ -41,10 +41,9 @@ def populate_database():
         - 2 active
         - 1 inactive
         - 2 deprecated
-    - 5 GTFS Realtime feeds
-    - 4 3TFS rt datasets, with 1 of them inactive
+    - 3 GTFS datasets
+    - 4 GTFS Realtime feeds, with 1 of them inactive
     """
-    clean_testing_db()
     session = get_testing_session()
     fake = Faker()
 
@@ -138,7 +137,7 @@ def populate_database():
         .all()
     )
 
-    # the first 2 datasets are for the first feed
+    # the first 2 datasets are for the first active feed, one dataset is for the second active feed
     for i in range(1, 4):
         feed_index = 0 if i in [1, 2] else 1
         wkt_polygon = "POLYGON((-18 -9, -18 9, 18 9, 18 -9, -18 -9))"
@@ -176,8 +175,6 @@ def populate_database():
     active_gtfs_feeds[0].locations = locations
     active_gtfs_feeds[1].locations = locations
 
-    # active_gtfs_feeds[0].gtfsdatasets.append() = gtfs_datasets
-
     vp_entitytype = session.query(Entitytype).filter_by(name="vp").first()
     if not vp_entitytype:
         vp_entitytype = Entitytype(name="vp")
