fix: missing validatorVersion from GTFS validation report (#1142)

Author: davidgamez

functions-python/process_validation_report/src/main.py

@@ -108,7 +108,9 @@ def parse_json_report(json_report):
     try:
         dt = json_report["summary"]["validatedAt"]
         validated_at = datetime.fromisoformat(dt.replace("Z", "+00:00"))
-        version = json_report["summary"]["validatorVersion"]
+        version = None
+        if "validatorVersion" in json_report["summary"]:
+            version = json_report["summary"]["validatorVersion"]
         logging.info(
             f"Validation report validated at {validated_at} with version {version}."
         )
@@ -141,9 +143,11 @@ def generate_report_entities(
     json_report_url = (
         f"{FILES_ENDPOINT}/{feed_stable_id}/{dataset_stable_id}/report_{version}.json"
     )
-    if get_validation_report(report_id, session):  # Check if report already exists
+    # Check if report already exists
+    # If exists, the function should graceful finish avoiding retry mechanism to trigger again
+    if get_validation_report(report_id, session):
         logging.warning(f"Validation report {report_id} already exists. Terminating.")
-        raise Exception(f"Validation report {report_id} already exists.")
+        return []
 
     validation_report_entity = Validationreport(
         id=report_id,
@@ -248,15 +252,16 @@ def create_validation_report_entities(
         return json_report, code
 
     try:
-        validated_at, version = parse_json_report(json_report)
+        validated_at, version_from_json = parse_json_report(json_report)
     except Exception as error:
         return str(error), 500
 
     try:
         # Generate the database entities required for the report
         # If an error is thrown we should let the retry mechanism to do its work
         entities = generate_report_entities(
-            version,
+            # default to the version parameter
+            version_from_json if version_from_json else version,
             validated_at,
             json_report,
             dataset_stable_id,

functions-python/process_validation_report/tests/test_validation_report.py

@@ -329,3 +329,115 @@ def test_compute_validation_report_counters(self, db_session):
         self.assertEqual(validation_report.unique_info_count, 1)
         self.assertEqual(validation_report.unique_warning_count, 1)
         self.assertEqual(validation_report.unique_error_count, 2)
+
+    @mock.patch("requests.get")
+    @with_db_session(db_url=default_db_url)
+    def test_create_validation_report_entities_missing_validator_version(
+        self, mock_get, db_session
+    ):
+        """Test create_validation_report_entities function
+        when the validator version is missing from the JSON report."""
+        mock_get.return_value = MagicMock(
+            status_code=200,
+            json=lambda: {
+                "summary": {
+                    "validatedAt": "2021-01-01T00:00:00Z",
+                    "gtfsFeatures": ["stops", "routes"],
+                },
+                "notices": [
+                    {"code": "notice_code", "severity": "ERROR", "totalNotices": 1}
+                ],
+            },
+        )
+        feed_stable_id = faker.word()
+        dataset_stable_id = faker.word()
+
+        # Create GTFS Feed
+        feed = Gtfsfeed(id=faker.word(), data_type="gtfs", stable_id=feed_stable_id)
+        # Create a new dataset
+        dataset = Gtfsdataset(
+            id=faker.word(), feed_id=feed.id, stable_id=dataset_stable_id, latest=True
+        )
+        try:
+            db_session.add(feed)
+            db_session.add(dataset)
+            db_session.commit()
+            create_validation_report_entities(feed_stable_id, dataset_stable_id, "1.0")
+
+            # Validate that the validation report was created
+            validation_report = (
+                db_session.query(Validationreport)
+                .filter(Validationreport.id == f"{dataset_stable_id}_1.0")
+                .one_or_none()
+            )
+            self.assertIsNotNone(validation_report)
+        except Exception as e:
+            raise e
+        finally:
+            db_session.rollback()
+            db_session.close()
+
+    @mock.patch("requests.get")
+    @with_db_session(db_url=default_db_url)
+    def test_create_validation_report_entities_validation_report_exists(
+        self, mock_get, db_session
+    ):
+        """Test create_validation_report_entities function
+        when the validation report already exists."""
+        version = "1.0"
+        mock_get.return_value = MagicMock(
+            status_code=200,
+            json=lambda: {
+                "summary": {
+                    "validatedAt": "2021-01-01T00:00:00Z",
+                    "gtfsFeatures": ["stops", "routes"],
+                    "validatorVersion": version,
+                },
+                "notices": [
+                    {"code": "notice_code", "severity": "ERROR", "totalNotices": 1}
+                ],
+            },
+        )
+        feed_stable_id = faker.word()
+        dataset_stable_id = faker.word()
+        report_id = f"{dataset_stable_id}_{version}"
+        # Create GTFS Feed
+        feed = Gtfsfeed(id=faker.word(), data_type="gtfs", stable_id=feed_stable_id)
+        # Create a new dataset
+        dataset = Gtfsdataset(
+            id=faker.word(),
+            feed_id=feed.id,
+            stable_id=dataset_stable_id,
+            latest=True,
+            validation_reports=[
+                Validationreport(
+                    id=report_id, validator_version="1.0", notices=[], features=[]
+                )
+            ],
+        )
+
+        try:
+            db_session.add(feed)
+            db_session.add(dataset)
+            db_session.commit()
+            # Validate that the validation report is already in the DB
+            validation_report = (
+                db_session.query(Validationreport)
+                .filter(Validationreport.id == report_id)
+                .one_or_none()
+            )
+            self.assertIsNotNone(validation_report)
+            create_validation_report_entities(feed_stable_id, dataset_stable_id, "1.0")
+
+            # Validate that the validation report remained in the DB
+            validation_report = (
+                db_session.query(Validationreport)
+                .filter(Validationreport.id == report_id)
+                .one_or_none()
+            )
+            self.assertIsNotNone(validation_report)
+        except Exception as e:
+            raise e
+        finally:
+            db_session.rollback()
+            db_session.close()