feat: support feed stable ids as request parameter in the GBFS validator batch function (#1306)

Author: davidgamez

functions-python/gbfs_validator/README.md

@@ -30,7 +30,7 @@ The message published by the batch function to the Pub/Sub topic follows this fo
 
 ### Functionality Details
 
-- **`gbfs-validator-batch`**: Triggered per execution ID, this function iterates over all GBFS feeds, preparing and publishing individual messages to the Pub/Sub topic.
+- **`gbfs-validator-batch`**: Triggered per execution ID, when the request is a POST request with a JSON body containing `feed_stable_ids`, it publishes events related to only those feeds. Otherwise, it publishes avents of all feeds to the Pub/Sub topic.
 - **`gbfs-validator-pubsub`**: Triggered per feed, this function performs the following steps:
   1. **Access the autodiscovery URL and update versions**: The function accesses the autodiscovery URL to update the **GBFSVersions** table.
   2. **Measure latency and validate the feed**: For each version, the function measures the response latency and validates the feed. The validation summary is stored in GCP, and the total error count is extracted and saved in the **GBFSValidationReport**.
@@ -46,6 +46,13 @@ The `gbfs-validator-batch` function requires the following environment variables
 - **`PROJECT_ID`**: The Google Cloud Project ID used to construct the full topic path.
 - **`FEEDS_DATABASE_URL`**: The database connection string for accessing the GBFS feeds.
 
+Optional request body parameters for the batch function:
+```json
+{
+    "feed_stable_ids": ["feed_id_1", "feed_id_2"]
+}
+```
+
 ### Pub/Sub Function Environment Variables
 
 The `gbfs-validator-pubsub` function requires the following environment variables:

functions-python/gbfs_validator/src/main.py

@@ -101,9 +101,11 @@ def gbfs_validator_pubsub(cloud_event: CloudEvent):
 
 @with_db_session
 @functions_framework.http
-def gbfs_validator_batch(_, db_session: Session):
+def gbfs_validator_batch(request, db_session: Session):
     """
-    HTTP Cloud Function to trigger the GBFS Validator function for multiple datasets.
+    HTTP Cloud  Function to trigger the GBFS Validator function for multiple datasets.
+    When the request is a POST request with a JSON body containing `feed_stable_ids`,
+    it processes only those feeds. Otherwise, it processes all feeds in the database.
     @param _: The request object.
     @return: The response of the function.
     """
@@ -113,9 +115,25 @@ def gbfs_validator_batch(_, db_session: Session):
         logging.error("PUBSUB_TOPIC_NAME environment variable not set.")
         return "PUBSUB_TOPIC_NAME environment variable not set.", 500
 
-    # Get all GBFS feeds from the database
     try:
-        gbfs_feeds = fetch_all_gbfs_feeds(db_session)
+        feed_stable_ids = None
+        if request and request.method == "POST" and request.is_json:
+            request_json = request.get_json()
+            feed_stable_ids = (
+                request_json.get("feed_stable_ids") if request_json else None
+            )
+        else:
+            logging.info("Request body not provided or not a valid JSON.")
+    except Exception as e:
+        logging.error("Error parsing request body: %s", e)
+        return "Invalid request body.", 400
+
+    try:
+        if feed_stable_ids:
+            gbfs_feeds = fetch_gbfs_feeds_by_stable_ids(db_session, feed_stable_ids)
+        else:
+            # Get all GBFS feeds from the database
+            gbfs_feeds = fetch_all_gbfs_feeds(db_session)
     except Exception:
         return "Error getting all GBFS feeds.", 500
 
@@ -150,3 +168,15 @@ def gbfs_validator_batch(_, db_session: Session):
         f"GBFS Validator batch function triggered successfully for {len(feeds_data)} feeds.",
         200,
     )
+
+
+def fetch_gbfs_feeds_by_stable_ids(db_session, feed_stable_ids):
+    """Fetch GBFS feeds by their IDs and not deprecated from the database"""
+    gbfs_feeds = (
+        db_session.query(Gbfsfeed)
+        .filter(
+            Gbfsfeed.stable_id.in_(feed_stable_ids), Gbfsfeed.status != "deprecated"
+        )
+        .all()
+    )
+    return gbfs_feeds

functions-python/gbfs_validator/tests/test_gbfs_validator.py

@@ -75,7 +75,6 @@ def test_gbfs_validator_batch(
     ):
         # Prepare mocks
         mock_session = MagicMock()
-        # mock_database.return_value.start_db_session.return_value = mock_session
 
         mock_publisher = MagicMock()
         mock_publisher_client.return_value = mock_publisher
@@ -179,3 +178,41 @@ def test_gbfs_validator_batch_publish_exception(
         # Call the function
         result = gbfs_validator_batch(None)
         self.assertEqual(result[1], 500)
+
+    @patch.dict(
+        os.environ,
+        {
+            "PUBSUB_TOPIC_NAME": "mock-topic",
+        },
+    )
+    @patch("main.pubsub_v1.PublisherClient")
+    @patch("main.fetch_gbfs_feeds_by_stable_ids")
+    def test_gbfs_validator_batch_by_feed_stable_ids(
+        self, fetch_gbfs_feeds_by_stable_ids, mock_publisher_client
+    ):
+        # Prepare mocks
+        mock_session = MagicMock()
+
+        mock_publisher = MagicMock()
+        mock_publisher_client.return_value = mock_publisher
+
+        mock_feed = MagicMock()
+        mock_feed.stable_id = "mock-stable-id"
+        mock_feed.id = str(uuid.uuid4())
+        mock_feed.auto_discovery_url = "http://mock-url.com"
+        mock_feed.gbfsversions = [MagicMock(version="1.0")]
+        mock_feed_2 = copy.deepcopy(mock_feed)
+        mock_feed_2.gbfsversions = []
+        fetch_gbfs_feeds_by_stable_ids.return_value = [mock_feed, mock_feed_2]
+        request = MagicMock()
+        request.method = "POST"
+        request.is_json = True
+        request.get_json.return_value = {
+            "feed_stable_ids": [mock_feed.id, mock_feed_2.id]
+        }
+        # Call the function
+        result = gbfs_validator_batch(request, db_session=mock_session)
+        self.assertEqual(result[1], 200)
+
+        fetch_gbfs_feeds_by_stable_ids.assert_called_once()
+        self.assertEqual(mock_publisher.publish.call_count, 2)