Merge branch 'main' into 293-Psycopg

Author: qcdyx

.github/workflows/api-deployer.yml

@@ -56,6 +56,10 @@ on:
         description: Validator endpoint
         required: true
         type: string
+      OPERATIONS_OAUTH2_CLIENT_ID_1PASSWORD:
+        description: Oauth client id part of the authorization for the operations API
+        required: true
+        type: string
 
 env:
   python_version: '3.11'
@@ -255,6 +259,11 @@ jobs:
           name: feeds_gen
           path: api/src/feeds_gen/
 
+      - uses: actions/download-artifact@v4
+        with:
+          name: feeds_operations_gen
+          path: functions-python/operations_api/src/feeds_operations_gen/
+
       - name: Build python functions
         run: |
           scripts/function-python-build.sh --all
@@ -290,11 +299,12 @@ jobs:
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
           TRANSITLAND_API_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/TansitLand API Key/credential"
+          OPERATIONS_OAUTH2_CLIENT_ID: ${{ inputs.OPERATIONS_OAUTH2_CLIENT_ID_1PASSWORD }}
 
       - name: Populate Variables
         run: |
           scripts/replace-variables.sh -in_file infra/backend.conf.rename_me -out_file infra/backend.conf -variables BUCKET_NAME,OBJECT_PREFIX
-          scripts/replace-variables.sh -in_file infra/vars.tfvars.rename_me -out_file infra/vars.tfvars -variables PROJECT_ID,REGION,ENVIRONMENT,DEPLOYER_SERVICE_ACCOUNT,FEED_API_IMAGE_VERSION,OAUTH2_CLIENT_ID,OAUTH2_CLIENT_SECRET,GLOBAL_RATE_LIMIT_REQ_PER_MINUTE,ARTIFACT_REPO_NAME,VALIDATOR_ENDPOINT,TRANSITLAND_API_KEY
+          scripts/replace-variables.sh -in_file infra/vars.tfvars.rename_me -out_file infra/vars.tfvars -variables PROJECT_ID,REGION,ENVIRONMENT,DEPLOYER_SERVICE_ACCOUNT,FEED_API_IMAGE_VERSION,OAUTH2_CLIENT_ID,OAUTH2_CLIENT_SECRET,GLOBAL_RATE_LIMIT_REQ_PER_MINUTE,ARTIFACT_REPO_NAME,VALIDATOR_ENDPOINT,TRANSITLAND_API_KEY,OPERATIONS_OAUTH2_CLIENT_ID
 
       - uses: hashicorp/setup-terraform@v3
         with:

.github/workflows/api-dev.yml

@@ -22,6 +22,7 @@ jobs:
       GLOBAL_RATE_LIMIT_REQ_PER_MINUTE: ${{ vars.GLOBAL_RATE_LIMIT_REQ_PER_MINUTE }}
       TF_APPLY: true
       VALIDATOR_ENDPOINT: https://stg-gtfs-validator-web-mbzoxaljzq-ue.a.run.app
+      OPERATIONS_OAUTH2_CLIENT_ID_1PASSWORD: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_RETOOL_OAUTH2_CREDS/username"
     secrets:
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.DEV_GCP_MOBILITY_FEEDS_SA_KEY }}
       OAUTH2_CLIENT_ID: ${{ secrets.DEV_MOBILITY_FEEDS_OAUTH2_CLIENT_ID}}

.github/workflows/api-prod.yml

@@ -18,6 +18,7 @@ jobs:
       GLOBAL_RATE_LIMIT_REQ_PER_MINUTE: ${{ vars.GLOBAL_RATE_LIMIT_REQ_PER_MINUTE }}
       TF_APPLY: true
       VALIDATOR_ENDPOINT: https://gtfs-validator-web-mbzoxaljzq-ue.a.run.app
+      OPERATIONS_OAUTH2_CLIENT_ID_1PASSWORD: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_RETOOL_OAUTH2_CREDS/username"
     secrets:
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
       OAUTH2_CLIENT_ID: ${{ secrets.PROD_MOBILITY_FEEDS_OAUTH2_CLIENT_ID}}

.github/workflows/api-qa.yml

@@ -18,6 +18,7 @@ jobs:
       TF_APPLY: true
       GLOBAL_RATE_LIMIT_REQ_PER_MINUTE: ${{ vars.GLOBAL_RATE_LIMIT_REQ_PER_MINUTE }}
       VALIDATOR_ENDPOINT: https://stg-gtfs-validator-web-mbzoxaljzq-ue.a.run.app
+      OPERATIONS_OAUTH2_CLIENT_ID_1PASSWORD: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_RETOOL_OAUTH2_CREDS/username"
     secrets:
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
       OAUTH2_CLIENT_ID: ${{ secrets.DEV_MOBILITY_FEEDS_OAUTH2_CLIENT_ID}}

.github/workflows/build-test.yml

@@ -80,6 +80,10 @@ jobs:
           scripts/setup-openapi-generator.sh
           scripts/api-gen.sh
 
+      - name: Generate Operations API code
+        run: |
+          scripts/api-operations-gen.sh
+
       - name: Unit tests - API
         shell: bash
         run: |
@@ -104,9 +108,16 @@ jobs:
           path: api/src/database_gen/
           overwrite: true
 
-      - name: API generated code
+      - name: Upload API generated code
         uses: actions/upload-artifact@v4
         with:
           name: feeds_gen
           path: api/src/feeds_gen/
+          overwrite: true
+
+      - name: Upload Operations API generated code
+        uses: actions/upload-artifact@v4
+        with:
+          name: feeds_operations_gen
+          path: functions-python/operations_api/src/feeds_operations_gen/
           overwrite: true

api/src/database/database.py

@@ -46,9 +46,20 @@ def configure_polymorphic_mappers():
 
 def with_db_session(func):
     """
-    Decorator to handle the session management
-    :param func: the function to decorate
-    :return: the decorated function
+    Decorator to handle the session management for the decorated function.
+
+    This decorator ensures that a database session is properly created, committed, rolled back in case of an exception,
+    and closed. It uses the @contextmanager decorator to manage the lifecycle of the session, providing a clean and
+    efficient way to handle database interactions.
+
+    How it works:
+        - The decorator checks if a 'db_session' keyword argument is provided to the decorated function.
+        - If 'db_session' is not provided, it creates a new Database instance and starts a new session using the
+          start_db_session context manager.
+        - The context manager ensures that the session is properly committed if no exceptions occur, rolled back if an
+          exception occurs, and closed in either case.
+        - The session is then passed to the decorated function as the 'db_session' keyword argument.
+        - If 'db_session' is already provided, it simply calls the decorated function with the existing session.
     """
 
     def wrapper(*args, **kwargs):
@@ -114,6 +125,13 @@ def is_connected(self):
 
     @contextmanager
     def start_db_session(self):
+        """
+        Context manager to start a database session with optional echo.
+
+        This method manages the lifecycle of a database session, ensuring that the session is properly created,
+        committed, rolled back in case of an exception, and closed. The @contextmanager decorator simplifies
+        resource management by handling the setup and cleanup logic within a single function.
+        """
         session = self.Session()
         try:
             yield session

api/src/feeds/impl/feeds_api_impl.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from typing import List, Union, TypeVar
 
+from sqlalchemy import or_
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload, Session
 from sqlalchemy.orm.query import Query
@@ -39,13 +40,13 @@
 from feeds_gen.models.gtfs_feed import GtfsFeed
 from feeds_gen.models.gtfs_rt_feed import GtfsRTFeed
 from middleware.request_context import is_user_email_restricted
-from sqlalchemy import or_
 from utils.date_utils import valid_iso_date
 from utils.location_translation import (
     create_location_translation_object,
     LocationTranslation,
     get_feeds_location_translations,
 )
+from utils.logger import Logger
 
 T = TypeVar("T", bound="BasicFeed")
 
@@ -59,9 +60,15 @@ class FeedsApiImpl(BaseFeedsApi):
 
     APIFeedType = Union[BasicFeed, GtfsFeed, GtfsRTFeed]
 
+    def __init__(self) -> None:
+        self.logger = Logger("FeedsApiImpl").get_logger()
+
     @with_db_session
     def get_feed(self, id: str, db_session: Session) -> BasicFeed:
         """Get the specified feed from the Mobility Database."""
+        is_email_restricted = is_user_email_restricted()
+        self.logger.info(f"User email is restricted: {is_email_restricted}")
+
         feed = (
             FeedFilter(stable_id=id, provider__ilike=None, producer_url__ilike=None, status=None)
             .filter(Database().get_query_model(db_session, Feed))
@@ -70,8 +77,7 @@ def get_feed(self, id: str, db_session: Session) -> BasicFeed:
                 or_(
                     Feed.operational_status == None,  # noqa: E711
                     Feed.operational_status != "wip",
-                    # Allow all feeds to be returned if the user is not restricted
-                    not is_user_email_restricted(),
+                    not is_email_restricted,  # Allow all feeds to be returned if the user is not restricted
                 )
             )
             .first()
@@ -83,19 +89,30 @@ def get_feed(self, id: str, db_session: Session) -> BasicFeed:
 
     @with_db_session
     def get_feeds(
-        self, limit: int, offset: int, status: str, provider: str, producer_url: str, db_session: Session
+        self,
+        limit: int,
+        offset: int,
+        status: str,
+        provider: str,
+        producer_url: str,
+        is_official: bool,
+        db_session: Session,
     ) -> List[BasicFeed]:
         """Get some (or all) feeds from the Mobility Database."""
+        is_email_restricted = is_user_email_restricted()
+        self.logger.info(f"User email is restricted: {is_email_restricted}")
         feed_filter = FeedFilter(
             status=status, provider__ilike=provider, producer_url__ilike=producer_url, stable_id=None
         )
         feed_query = feed_filter.filter(Database().get_query_model(db_session, Feed))
+        if is_official:
+            feed_query = feed_query.filter(Feed.official)
         feed_query = feed_query.filter(Feed.data_type != "gbfs")  # Filter out GBFS feeds
         feed_query = feed_query.filter(
             or_(
                 Feed.operational_status == None,  # noqa: E711
                 Feed.operational_status != "wip",
-                not is_user_email_restricted(),  # Allow all feeds to be returned if the user is not restricted
+                not is_email_restricted,  # Allow all feeds to be returned if the user is not restricted
             )
         )
         # Results are sorted by provider
@@ -217,6 +234,7 @@ def get_gtfs_feeds(
         dataset_latitudes: str,
         dataset_longitudes: str,
         bounding_filter_method: str,
+        is_official: bool,
         db_session: Session,
     ) -> List[GtfsFeed]:
         """Get some (or all) GTFS feeds from the Mobility Database."""
@@ -236,14 +254,16 @@ def get_gtfs_feeds(
             subquery, dataset_latitudes, dataset_longitudes, bounding_filter_method
         ).subquery()
 
+        is_email_restricted = is_user_email_restricted()
+        self.logger.info(f"User email is restricted: {is_email_restricted}")
         feed_query = (
             db_session.query(Gtfsfeed)
             .filter(Gtfsfeed.id.in_(subquery))
             .filter(
                 or_(
                     Gtfsfeed.operational_status == None,  # noqa: E711
                     Gtfsfeed.operational_status != "wip",
-                    not is_user_email_restricted(),  # Allow all feeds to be returned if the user is not restricted
+                    not is_email_restricted,  # Allow all feeds to be returned if the user is not restricted
                 )
             )
             .options(
@@ -253,9 +273,10 @@ def get_gtfs_feeds(
                 *BasicFeedImpl.get_joinedload_options(),
             )
             .order_by(Gtfsfeed.provider, Gtfsfeed.stable_id)
-            .limit(limit)
-            .offset(offset)
         )
+        if is_official:
+            feed_query = feed_query.filter(Feed.official)
+        feed_query = feed_query.limit(limit).offset(offset)
         return self._get_response(feed_query, GtfsFeedImpl, db_session)
 
     @with_db_session
@@ -303,6 +324,7 @@ def get_gtfs_rt_feeds(
         country_code: str,
         subdivision_name: str,
         municipality: str,
+        is_official: bool,
         db_session: Session,
     ) -> List[GtfsRTFeed]:
         """Get some (or all) GTFS Realtime feeds from the Mobility Database."""
@@ -350,9 +372,10 @@ def get_gtfs_rt_feeds(
                 *BasicFeedImpl.get_joinedload_options(),
             )
             .order_by(Gtfsrealtimefeed.provider, Gtfsrealtimefeed.stable_id)
-            .limit(limit)
-            .offset(offset)
         )
+        if is_official:
+            feed_query = feed_query.filter(Feed.official)
+        feed_query = feed_query.limit(limit).offset(offset)
         return self._get_response(feed_query, GtfsRTFeedImpl, db_session)
 
     @staticmethod

api/src/feeds/impl/models/basic_feed_impl.py

@@ -23,10 +23,14 @@ class Config:
     def from_orm(cls, feed: Feed | None, _=None) -> BasicFeed | None:
         if not feed:
             return None
+        latest_official_status = None
+        if len(feed.officialstatushistories) > 0:
+            latest_official_status = max(feed.officialstatushistories, key=lambda x: x.timestamp).is_official
         return cls(
             id=feed.stable_id,
             data_type=feed.data_type,
             status=feed.status,
+            official=latest_official_status,
             created_at=feed.created_at,
             external_ids=sorted(
                 [ExternalIdImpl.from_orm(item) for item in feed.externalids], key=lambda x: x.external_id
@@ -48,7 +52,12 @@ def from_orm(cls, feed: Feed | None, _=None) -> BasicFeed | None:
     @staticmethod
     def get_joinedload_options() -> [_AbstractLoad]:
         """Returns common joinedload options for feeds queries."""
-        return [joinedload(Feed.locations), joinedload(Feed.externalids), joinedload(Feed.redirectingids)]
+        return [
+            joinedload(Feed.locations),
+            joinedload(Feed.externalids),
+            joinedload(Feed.redirectingids),
+            joinedload(Feed.officialstatushistories),
+        ]
 
 
 class BasicFeedImpl(BaseFeedImpl, BasicFeed):

api/src/feeds/impl/models/search_feed_item_result_impl.py

@@ -25,6 +25,7 @@ def from_orm_gtfs(cls, feed_search_row):
             external_ids=feed_search_row.external_ids,
             provider=feed_search_row.provider,
             feed_name=feed_search_row.feed_name,
+            official=feed_search_row.official,
             note=feed_search_row.note,
             feed_contact_email=feed_search_row.feed_contact_email,
             source_info=SourceInfo(
@@ -58,6 +59,7 @@ def from_orm_gtfs_rt(cls, feed_search_row):
             external_ids=feed_search_row.external_ids,
             provider=feed_search_row.provider,
             feed_name=feed_search_row.feed_name,
+            official=feed_search_row.official,
             note=feed_search_row.note,
             feed_contact_email=feed_search_row.feed_contact_email,
             source_info=SourceInfo(