fix: process validation report when dataset is not committed to the DB yet (#1127)

Author: davidgamez

functions-python/batch_process_dataset/src/main.py

@@ -238,9 +238,13 @@ def create_dataset(self, dataset_file: DatasetFile, db_session: Session):
                 latest_dataset.latest = False
                 db_session.add(latest_dataset)
             db_session.add(new_dataset)
+            db_session.commit()
+            logging.info(f"[{self.feed_stable_id}] Dataset created successfully.")
 
             refresh_materialized_view(db_session, t_feedsearch.name)
-            logging.info(f"[{self.feed_stable_id}] Dataset created successfully.")
+            logging.info(
+                f"[{self.feed_stable_id}] Materialized view refresh event triggered successfully."
+            )
         except Exception as e:
             raise Exception(f"Error creating dataset: {e}")
 

functions-python/process_validation_report/src/main.py

@@ -155,6 +155,8 @@ def generate_report_entities(
     entities.append(validation_report_entity)
 
     dataset = get_dataset(dataset_stable_id, session)
+    if not dataset:
+        raise Exception(f"Dataset {dataset_stable_id} not found.")
     dataset.validation_reports.append(validation_report_entity)
 
     extracted_timezone = extract_timezone_from_json_validation_report(json_report)
@@ -251,32 +253,36 @@ def create_validation_report_entities(
         return str(error), 500
 
     try:
-        logging.info("Database session started.")
         # Generate the database entities required for the report
-        try:
-            entities = generate_report_entities(
-                version,
-                validated_at,
-                json_report,
-                dataset_stable_id,
-                db_session,
-                feed_stable_id,
-            )
-        except Exception as error:
-            return str(error), 200  # Report already exists
+        # If an error is thrown we should let the retry mechanism to do its work
+        entities = generate_report_entities(
+            version,
+            validated_at,
+            json_report,
+            dataset_stable_id,
+            db_session,
+            feed_stable_id,
+        )
 
-        # Commit the entities to the database
         for entity in entities:
             db_session.add(entity)
-        logging.info(f"Committing {len(entities)} entities to the database.")
-        db_session.commit()
-        logging.info("Entities committed successfully.")
+        # In this case the report entities are already in the DB or cannot be saved for other reasons
+        # In any case, this will fail in any retried event
+        try:
+            logging.info("Committing %s entities to the database.", len(entities))
+            db_session.commit()
+            logging.info("Entities committed successfully.")
+        except Exception as error:
+            logging.warning(
+                "Could not commit %s entities to the database: %s", entities, error
+            )
+            return str(error), 200
 
         update_feed_statuses_query(db_session, [feed_stable_id])
 
         return f"Created {len(entities)} entities.", 200
     except Exception as error:
-        logging.error(f"Error creating validation report entities: {error}")
+        logging.error("Error creating validation report entities: : %s", error)
         return f"Error creating validation report entities: {error}", 500
     finally:
         pass

functions-python/process_validation_report/tests/test_validation_report.py

@@ -176,6 +176,36 @@ def test_create_validation_report_entities_json_error2(self, mock_get):
         )
         self.assertEqual(status, 500)
 
+    @mock.patch("requests.get")
+    def test_create_validation_report_entities_missing_dataset(self, mock_get):
+        """
+        Test the create_validation_report_entities function when the dataset is not found in the DB
+        """
+        mock_get.return_value = MagicMock(
+            status_code=200,
+            json=lambda: {
+                "summary": {
+                    "validatedAt": "2021-01-01T00:00:00Z",
+                    "validatorVersion": "1.0",
+                    "gtfsFeatures": ["stops", "routes"],
+                },
+                "notices": [
+                    {"code": "notice_code", "severity": "ERROR", "totalNotices": 1}
+                ],
+            },
+        )
+        feed_stable_id = faker.word()
+        dataset_stable_id = "MISSING_ID"
+
+        message, status = create_validation_report_entities(
+            feed_stable_id, dataset_stable_id, "1.0"
+        )
+        self.assertEqual(500, status)
+        self.assertEqual(
+            "Error creating validation report entities: Dataset MISSING_ID not found.",
+            message,
+        )
+
     @patch("main.Logger")
     @patch("main.create_validation_report_entities")
     def test_process_validation_report(self, create_validation_report_entities_mock, _):