Merge branch 'main' into feat/997

Author: cka-y

.github/workflows/datasets-batch-deployer-prod.yml

@@ -14,8 +14,8 @@ jobs:
       REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
       ENVIRONMENT: 'prod'
       DEPLOYER_SERVICE_ACCOUNT: ${{ vars.PROD_MOBILITY_FEEDS_DEPLOYER_SERVICE_ACCOUNT }}
-      # every week on monday and thursday at 00:00
-      JOB_SCHEDULE: '0 0 * * 1,4'
+      # every day at 00:00 UTC
+      JOB_SCHEDULE: '0 0 * * *'
       DATASETS_BUCKET_NAME: 'mobilitydata-datasets-prod'
     secrets:
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}

.github/workflows/db-update.yml

@@ -192,16 +192,6 @@ jobs:
       id: getpath
       run: echo "PATH=$(realpath sources.csv)" >> $GITHUB_OUTPUT
 
-    - name: Download systems.csv
-      run: wget -O systems.csv https://raw.githubusercontent.com/MobilityData/gbfs/master/systems.csv
-
-    - name: Get full path of systems.csv
-      id: getsyspath
-      run: echo "PATH=$(realpath systems.csv)" >> $GITHUB_OUTPUT
-
-    - name: GBFS - Update Database Content
-      run: scripts/populate-db.sh ${{ steps.getsyspath.outputs.PATH }} gbfs >> populate-gbfs.log
-
     - name: GTFS - Update Database Content
       run: scripts/populate-db.sh ${{ steps.getpath.outputs.PATH }} > populate.log
 
@@ -212,13 +202,6 @@ jobs:
         name: populate-${{ inputs.ENVIRONMENT }}.log
         path: populate.log
 
-    - name: GBFS - Upload log file for verification
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v4
-      with:
-          name: populate-gbfs-${{ inputs.ENVIRONMENT }}.log
-          path: populate-gbfs.log
-
   update-gcp-secret:
     name: Update GCP Secrets
     if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }}

README.md

@@ -6,6 +6,9 @@
 
 The Mobility Feed API service a list of open mobility data sources from across the world. This repository is the effort the initial effort to convert the current [The Mobility Database Catalogs](https://github.com/MobilityData/mobility-database-catalogs) in an API service. [More info about the Mobility Database can be found on the website](https://mobilitydatabase.org/).
 
+The contents of the database is available in csv format [here](https://files.mobilitydatabase.org/feeds_v2.csv). This includes all feeds, including ones we automatically import from sources outside the catalogs repo.
+
+The schema for the csv file is available [here](docs/SpreadsheetSchemaV2.md).
 
 ## GBFS Feeds
 The repository also includes GBFS feeds extracted from [`systems.csv`](https://github.com/MobilityData/gbfs/blob/master/systems.csv) in the [GBFS repository](https://github.com/MobilityData/gbfs). However, these feeds are not being served yet. The supported versions of these feeds are specified in the file [api/src/scripts/gbfs_utils/gbfs_versions.py](https://github.com/MobilityData/mobility-feed-api/blob/main/api/src/scripts/gbfs_utils/gbfs_versions.py).

docs/OperationsAPI.yaml

@@ -3,7 +3,7 @@ info:
   version: 1.0.0
   title: Mobility Database Catalog Operations
   description: |
-    API for the Mobility Database Catalog Operations. See [https://mobilitydatabase.org/](https://mobilitydatabase.org/). 
+    API for the Mobility Database Catalog Operations. See [https://mobilitydatabase.org/](https://mobilitydatabase.org/).
     This API was designed for internal use and is not intended to be used by the general public.
     The Mobility Database Operation API uses Auth2.0 authentication.
   termsOfService: https://mobilitydatabase.org/terms-and-conditions
@@ -33,29 +33,29 @@ paths:
           required: false
           schema:
             type: string
-            enum: [wip, published]
+            enum: [ wip, published ]
         - name: data_type
           in: query
           description: Filter feeds by data type.
           required: false
           schema:
             type: string
-            enum: [gtfs, gtfs_rt]
+            enum: [ gtfs, gtfs_rt ]
         - name: offset
           in: query
           description: Number of items to skip for pagination.
           required: false
           schema:
-            type: integer
-            default: 0
+            type: string
+            default: "0"
         - name: limit
           in: query
           description: Maximum number of items to return.
           required: false
           schema:
-            type: integer
-            default: 50
-            maximum: 100
+            type: string
+            default: "50"
+            example: "100"
       responses:
         200:
           description: List of feeds retrieved successfully.
@@ -86,7 +86,7 @@ paths:
         - "operations"
       operationId: updateGtfsFeed
       security:
-        - ApiKeyAuth: []
+        - ApiKeyAuth: [ ]
       requestBody:
         description: Payload to update the specified GTFS feed.
         required: true
@@ -118,7 +118,7 @@ paths:
         - "operations"
       operationId: updateGtfsRtFeed
       security:
-        - ApiKeyAuth: []
+        - ApiKeyAuth: [ ]
       requestBody:
         description: Payload to update the specified GTFS-RT feed.
         required: true
@@ -179,7 +179,7 @@ components:
           $ref: "#/components/schemas/SourceInfo"
         operational_status:
           type: string
-          enum: [wip, published]
+          enum: [ wip, published ]
           description: Current operational status of the feed.
         created_at:
           type: string
@@ -245,10 +245,10 @@ components:
           type: integer
           description: Total number of feeds matching the criteria.
         offset:
-          type: integer
+          type: string
           description: Current offset for pagination.
         limit:
-          type: integer
+          type: string
           description: Maximum number of items per page.
         feeds:
           type: array
@@ -285,13 +285,13 @@ components:
           example: Los Angeles Department of Transportation (LADOT, DASH, Commuter Express)
         feed_name:
           description: >
-            An optional description of the data feed, e.g to specify if the data feed is an aggregate of 
+            An optional description of the data feed, e.g to specify if the data feed is an aggregate of
             multiple providers, or which network is represented by the feed.
           type: string
           example: Bus
         note:
-         description: A note to clarify complex use cases for consumers.
-         type: string
+          description: A note to clarify complex use cases for consumers.
+          type: string
         feed_contact_email:
           description: Use to contact the feed producer.
           type: string
@@ -344,13 +344,13 @@ components:
           example: Los Angeles Department of Transportation (LADOT, DASH, Commuter Express)
         feed_name:
           description: >
-            An optional description of the data feed, e.g to specify if the data feed is an aggregate of 
+            An optional description of the data feed, e.g to specify if the data feed is an aggregate of
             multiple providers, or which network is represented by the feed.
           type: string
           example: Bus
         note:
-         description: A note to clarify complex use cases for consumers.
-         type: string
+          description: A note to clarify complex use cases for consumers.
+          type: string
         feed_contact_email:
           description: Use to contact the feed producer.
           type: string
@@ -396,7 +396,7 @@ components:
       properties:
         producer_url:
           description: >
-            URL where the producer is providing the dataset. 
+            URL where the producer is providing the dataset.
             Refer to the authentication information to know how to access this URL.
           type: string
           format: url
@@ -405,7 +405,7 @@ components:
           $ref: "#/components/schemas/Authentication_type"
         authentication_info_url:
           description: >
-            Contains a URL to a human-readable page describing how the authentication should be performed and how credentials can be created. 
+            Contains a URL to a human-readable page describing how the authentication should be performed and how credentials can be created.
             This field is required for `authentication_type=1` and `authentication_type=2`.
           type: string
           format: url
@@ -500,4 +500,4 @@ components:
       in: header
 
 security:
-  - ApiKeyAuth: []
+  - ApiKeyAuth: [ ]