feat: 1055 cache validation report notices summary (#1079)

Author: qcdyx

api/src/feeds/impl/models/latest_dataset_impl.py

@@ -2,7 +2,6 @@
 
 from shared.database_gen.sqlacodegen_models import Gtfsdataset
 from feeds.impl.models.bounding_box_impl import BoundingBoxImpl
-from feeds.impl.models.validation_report_impl import ValidationReportImpl
 from feeds_gen.models.latest_dataset import LatestDataset
 from feeds_gen.models.latest_dataset_validation_report import LatestDatasetValidationReport
 from utils.model_utils import compare_java_versions
@@ -30,21 +29,13 @@ def from_orm(cls, dataset: Gtfsdataset | None) -> LatestDataset | None:
                 lambda a, b: a if compare_java_versions(a.validator_version, b.validator_version) == 1 else b,
                 dataset.validation_reports,
             )
-            (
-                total_error,
-                total_info,
-                total_warning,
-                unique_error_count,
-                unique_info_count,
-                unique_warning_count,
-            ) = ValidationReportImpl.compute_totals(latest_report)
             validation_report = LatestDatasetValidationReport(
-                total_error=total_error,
-                total_warning=total_warning,
-                total_info=total_info,
-                unique_error_count=unique_error_count,
-                unique_warning_count=unique_warning_count,
-                unique_info_count=unique_info_count,
+                total_error=latest_report.total_error,
+                total_warning=latest_report.total_warning,
+                total_info=latest_report.total_info,
+                unique_error_count=latest_report.unique_error_count,
+                unique_warning_count=latest_report.unique_warning_count,
+                unique_info_count=latest_report.unique_info_count,
             )
         return cls(
             id=dataset.stable_id,

api/src/feeds/impl/models/validation_report_impl.py

@@ -16,27 +16,6 @@ class Config:
 
         from_attributes = True
 
-    @staticmethod
-    def compute_totals(validation_report) -> tuple[int, int, int, int, int, int]:
-        """Compute the total number of errors, info, and warnings from a validation report,
-        and count the number of distinct codes for each."""
-        total_info, total_warning, total_error = 0, 0, 0
-        info_codes, warning_codes, error_codes = set(), set(), set()
-        for notice in validation_report.notices:
-            match notice.severity:
-                case "INFO":
-                    total_info += notice.total_notices
-                    info_codes.add(notice.notice_code)
-                case "WARNING":
-                    total_warning += notice.total_notices
-                    warning_codes.add(notice.notice_code)
-                case "ERROR":
-                    total_error += notice.total_notices
-                    error_codes.add(notice.notice_code)
-                case _:
-                    ValidationReportImpl._get_logger().warning(f"Unknown severity: {notice.severity}")
-        return total_error, total_info, total_warning, len(error_codes), len(info_codes), len(warning_codes)
-
     @classmethod
     def _get_logger(cls):
         return Logger(ValidationReportImpl.__class__.__module__).get_logger()
@@ -46,24 +25,17 @@ def from_orm(cls, validation_report: Validationreport | None) -> ValidationRepor
         """Create a model instance from a SQLAlchemy a Validation Report row object."""
         if not validation_report:
             return None
-        (
-            total_error,
-            total_info,
-            total_warning,
-            unique_error_count,
-            unique_info_count,
-            unique_warning_count,
-        ) = cls.compute_totals(validation_report)
+
         return cls(
             validated_at=validation_report.validated_at,
             features=[feature.name for feature in validation_report.features],
             validator_version=validation_report.validator_version,
-            total_error=total_error,
-            total_warning=total_warning,
-            total_info=total_info,
-            unique_error_count=unique_error_count,
-            unique_warning_count=unique_warning_count,
-            unique_info_count=unique_info_count,
+            total_error=validation_report.total_error,
+            total_warning=validation_report.total_warning,
+            total_info=validation_report.total_info,
+            unique_error_count=validation_report.unique_error_count,
+            unique_warning_count=validation_report.unique_warning_count,
+            unique_info_count=validation_report.unique_info_count,
             url_json=validation_report.json_report,
             url_html=validation_report.html_report,
             # TODO this field is not in the database

api/src/shared/__init__.py

@@ -78,9 +78,7 @@ def get_gtfs_feeds_query(
 
     if include_options_for_joinedload:
         feed_query = feed_query.options(
-            contains_eager(Gtfsfeed.gtfsdatasets)
-            .joinedload(Gtfsdataset.validation_reports)
-            .joinedload(Validationreport.notices),
+            contains_eager(Gtfsfeed.gtfsdatasets).joinedload(Gtfsdataset.validation_reports),
             *get_joinedload_options(),
         ).order_by(Gtfsfeed.provider, Gtfsfeed.stable_id)
 

api/src/shared/common/db_utils.py

@@ -131,9 +131,6 @@ def test_validation_report(test_database):
     assert validation_report is not None
     assert validation_report.validator_version == NEW_VALIDATION_VERSION
     assert validation_report.validated_at == NEW_VALIDATION_TIME
-    assert validation_report.total_info == VALIDATION_INFO_COUNT_PER_NOTICE * VALIDATION_INFO_NOTICES
-    assert validation_report.total_warning == VALIDATION_WARNING_COUNT_PER_NOTICE * VALIDATION_WARNING_NOTICES
-    assert validation_report.total_error == VALIDATION_ERROR_COUNT_PER_NOTICE * VALIDATION_ERROR_NOTICES
 
 
 def test_generate_unique_id():

api/tests/integration/test_database.py

@@ -185,6 +185,13 @@ class TestGtfsFeedImpl(unittest.TestCase):
 
     def test_from_orm_all_fields(self):
         """Test the `from_orm` method with all fields."""
+        # Update the validation report in gtfs_feed_orm to include precomputed counters
+        gtfs_feed_orm.gtfsdatasets[0].validation_reports[0].total_error = 27
+        gtfs_feed_orm.gtfsdatasets[0].validation_reports[0].total_warning = 64
+        gtfs_feed_orm.gtfsdatasets[0].validation_reports[0].total_info = 4
+        gtfs_feed_orm.gtfsdatasets[0].validation_reports[0].unique_error_count = 3
+        gtfs_feed_orm.gtfsdatasets[0].validation_reports[0].unique_warning_count = 4
+        gtfs_feed_orm.gtfsdatasets[0].validation_reports[0].unique_info_count = 2
         result = GtfsFeedImpl.from_orm(gtfs_feed_orm)
         assert result == expected_gtfs_feed_result
 

api/tests/unittest/models/test_gtfs_feed_impl.py

@@ -4,7 +4,7 @@
 
 from geoalchemy2 import WKTElement
 
-from shared.database_gen.sqlacodegen_models import Gtfsdataset, Feed, Validationreport, Notice
+from shared.database_gen.sqlacodegen_models import Gtfsdataset, Feed, Validationreport
 from feeds.impl.models.bounding_box_impl import BoundingBoxImpl
 from feeds.impl.models.latest_dataset_impl import LatestDatasetImpl
 
@@ -28,17 +28,33 @@ def test_from_orm(self):
                 service_date_range_end=datetime(2025, 1, 1, 0, 0, 0, tzinfo=ZoneInfo("Canada/Atlantic")),
                 agency_timezone="Canada/Atlantic",
                 validation_reports=[
-                    Validationreport(validator_version="1.0.0"),
+                    Validationreport(
+                        validator_version="1.0.0",
+                        total_error=0,
+                        total_warning=0,
+                        total_info=0,
+                        unique_error_count=0,
+                        unique_warning_count=0,
+                        unique_info_count=0,
+                    ),
                     Validationreport(
                         validator_version="1.2.0",
-                        notices=[
-                            Notice(severity="INFO", total_notices=1),
-                            Notice(severity="ERROR", total_notices=2, notice_code="foreign_key_violation"),
-                            Notice(severity="ERROR", total_notices=1, notice_code="empty_column_name"),
-                            Notice(severity="WARNING", total_notices=3),
-                        ],
+                        total_error=3,
+                        total_warning=3,
+                        total_info=1,
+                        unique_error_count=2,
+                        unique_warning_count=1,
+                        unique_info_count=1,
+                    ),
+                    Validationreport(
+                        validator_version="1.1.1",
+                        total_error=1,
+                        total_warning=2,
+                        total_info=0,
+                        unique_error_count=1,
+                        unique_warning_count=1,
+                        unique_info_count=0,
                     ),
-                    Validationreport(validator_version="1.1.1"),
                 ],
             )
         ) == LatestDatasetImpl(

api/tests/unittest/models/test_latest_dataset_impl.py

@@ -12,6 +12,12 @@ def test_from_orm(self):
             validator_version="1.0.0",
             json_report="http://json_report",
             html_report="http://html_report",
+            total_error=33,
+            total_warning=22,
+            total_info=11,
+            unique_error_count=2,
+            unique_warning_count=1,
+            unique_info_count=1,
             notices=[
                 Notice(severity="INFO", total_notices=10),
                 Notice(severity="INFO", total_notices=1),

api/tests/unittest/models/test_validation_report_impl.py

@@ -18,12 +18,12 @@
 import logging
 from datetime import datetime
 import requests
-from shared.helpers.database import Database
 from shared.helpers.timezone import (
     extract_timezone_from_json_validation_report,
     get_service_date_range_with_timezone_utc,
 )
 import functions_framework
+from shared.helpers.database import Database
 from shared.database_gen.sqlacodegen_models import (
     Validationreport,
     Feature,
@@ -164,6 +164,10 @@ def generate_report_entities(
         feature = get_feature(feature_name, session)
         feature.validations.append(validation_report_entity)
         entities.append(feature)
+
+    # Process notices and compute counters
+    counters = process_validation_report_notices(json_report["notices"])
+
     for notice in json_report["notices"]:
         notice_entity = Notice(
             dataset_id=dataset.id,
@@ -174,6 +178,14 @@ def generate_report_entities(
         )
         dataset.notices.append(notice_entity)
         entities.append(notice_entity)
+
+    # Update the validation report entity with computed counters
+    validation_report_entity.total_info = counters["total_info"]
+    validation_report_entity.total_warning = counters["total_warning"]
+    validation_report_entity.total_error = counters["total_error"]
+    validation_report_entity.unique_info_count = counters["unique_info_count"]
+    validation_report_entity.unique_warning_count = counters["unique_warning_count"]
+    validation_report_entity.unique_error_count = counters["unique_error_count"]
     return entities
 
 
@@ -227,7 +239,6 @@ def create_validation_report_entities(feed_stable_id, dataset_stable_id, version
     try:
         with db.start_db_session() as session:
             logging.info("Database session started.")
-
             # Generate the database entities required for the report
             try:
                 entities = generate_report_entities(
@@ -301,3 +312,88 @@ def process_validation_report(request):
         f"Processing validation report version {validator_version} for dataset {dataset_id} in feed {feed_id}."
     )
     return create_validation_report_entities(feed_id, dataset_id, validator_version)
+
+
+@functions_framework.http
+def compute_validation_report_counters(request):
+    """
+    Compute the total number of errors, warnings, and info notices,
+    as well as the number of distinct codes for each severity level
+    across all validation reports in the database, and write the results to the database.
+    """
+    batch_size = 100  # Number of reports to process in each batch
+    offset = 0
+    db = Database()
+    with db.start_db_session(echo=False) as session:
+        while True:
+            validation_reports = (
+                session.query(Validationreport).limit(batch_size).offset(offset).all()
+            )
+            print(
+                f"Processing {len(validation_reports)} validation reports from offset {offset}."
+            )
+            # Break the loop if no more reports are found
+            if len(validation_reports) == 0:
+                break
+
+            for report in validation_reports:
+                counters = process_validation_report_notices(report.notices)
+
+                # Update the report with computed counters
+                report.total_info = counters["total_info"]
+                report.total_warning = counters["total_warning"]
+                report.total_error = counters["total_error"]
+                report.unique_info_count = counters["unique_info_count"]
+                report.unique_warning_count = counters["unique_warning_count"]
+                report.unique_error_count = counters["unique_error_count"]
+
+                logging.info(
+                    f"Updated ValidationReport {report.id} with counters: "
+                    f"INFO={report.total_info}, WARNING={report.total_warning}, ERROR={report.total_error}, "
+                    f"Unique INFO Code={report.unique_info_count}, Unique WARNING Code={report.unique_warning_count}, "
+                    f"Unique ERROR Code={report.unique_error_count}"
+                )
+
+            # Commit the changes for the current batch
+            session.commit()
+
+            # Move to the next batch
+            offset += batch_size
+
+    return {"message": "Validation report counters computed successfully."}, 200
+
+
+def process_validation_report_notices(notices):
+    """
+    Processes the notices of a validation report and computes counters for different severities.
+
+    :param report: A Validationreport object containing associated notices.
+    :return: A dictionary with computed counters for total and unique counts of INFO, WARNING, and ERROR severities.
+    """
+    # Initialize counters for the current report
+    total_info, total_warning, total_error = 0, 0, 0
+    info_codes, warning_codes, error_codes = set(), set(), set()
+
+    # Process associated notices
+    for notice in notices:
+        match notice.severity:
+            case "INFO":
+                total_info += notice.total_notices
+                info_codes.add(notice.notice_code)
+            case "WARNING":
+                total_warning += notice.total_notices
+                warning_codes.add(notice.notice_code)
+            case "ERROR":
+                total_error += notice.total_notices
+                error_codes.add(notice.notice_code)
+            case _:
+                logging.warning(f"Unknown severity: {notice.severity}")
+
+    return {
+        "total_info": total_info,
+        "total_warning": total_warning,
+        "total_error": total_error,
+        "unique_info_count": len(info_codes),
+        "unique_warning_count": len(warning_codes),
+        "unique_error_count": len(error_codes),
+    }