fix: acceptance tests keep dropping errors for missing_required_column (#2007)

Author: davidgamez

output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/Arguments.java

@@ -23,6 +23,9 @@
 /** Command-line arguments for output-comparator CLI. */
 public class Arguments {
 
+  public static final String REFERENCE_SYSTEM_ERRORS_NAME = "reference_errors.json";
+  public static final String LATEST_SYSTEM_ERRORS_NAME = "latest_errors.json";
+
   @Parameter(
       names = {"-d", "--report_directory"},
       description = "Directory where reports are stored.",
@@ -42,11 +45,21 @@ public class Arguments {
       description = "Name of the reference validation report.")
   private String referenceValidationReportName;
 
+  @Parameter(
+      names = {"-rse", "--reference_system_errors_name"},
+      description = "Name of the reference system errors file.")
+  private String referenceSystemErrorsName;
+
   @Parameter(
       names = {"-l", "--latest_report_name"},
       description = "Name of the latest validation report.")
   private String latestValidationReportName;
 
+  @Parameter(
+      names = {"-lse", "--latest_system_errors_name"},
+      description = "Name of the latest system errors file.")
+  private String latestSystemErrorsName;
+
   @Parameter(
       names = {"-p", "--percent_invalid_datasets_threshold"},
       description =
@@ -164,4 +177,22 @@ public Optional<String> getCommitSha() {
   public void setCommitSha(String commitSha) {
     this.commitSha = commitSha;
   }
+
+  public String getReferenceSystemErrorsName() {
+    return referenceSystemErrorsName == null
+        ? REFERENCE_SYSTEM_ERRORS_NAME
+        : referenceSystemErrorsName;
+  }
+
+  public void setReferenceSystemErrorsName(String referenceSystemErrorsName) {
+    this.referenceSystemErrorsName = referenceSystemErrorsName;
+  }
+
+  public String getLatestSystemErrorsName() {
+    return latestSystemErrorsName == null ? LATEST_SYSTEM_ERRORS_NAME : latestSystemErrorsName;
+  }
+
+  public void setLatestSystemErrorsName(String latestSystemErrorsName) {
+    this.latestSystemErrorsName = latestSystemErrorsName;
+  }
 }

output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparator.java

@@ -72,7 +72,9 @@ public Result compareValidationRuns(
       String sourceUrl = sourceUrlContainer.getUrlForSourceId(sourceId);
 
       Path referenceReportPath = file.toPath().resolve(args.getReferenceValidationReportName());
+      Path referenceErrorsPath = file.toPath().resolve(args.getReferenceSystemErrorsName());
       Path latestReportPath = file.toPath().resolve(args.getLatestValidationReportName());
+      Path latestErrorsPath = file.toPath().resolve(args.getLatestSystemErrorsName());
       // in case a validation report does not exist for a sourceId we add the sourceId to
       // the list of corrupted sources
       if (!(referenceReportPath.toFile().exists() && latestReportPath.toFile().exists())) {
@@ -86,22 +88,31 @@ public Result compareValidationRuns(
       }
       ValidationReport referenceReport;
       ValidationReport latestReport;
-      try {
-        referenceReport = ValidationReport.fromPath(referenceReportPath);
-      } catch (IOException ioException) {
-        logger.atSevere().withCause(ioException).log("Error reading reference validation report");
+      referenceReport = getValidationReport(referenceReportPath);
+      if (referenceReport == null) {
         // in case a file is corrupted, add the sourceId to the list of corrupted sources
         corruptedSources.addCorruptedSource(sourceId, true, false, true, null);
         continue;
       }
-      try {
-        latestReport = ValidationReport.fromPath(latestReportPath);
-      } catch (IOException ioException) {
-        logger.atSevere().withCause(ioException).log("Error reading latest validation report");
+      latestReport = getValidationReport(latestReportPath);
+      if (latestReport == null) {
         // in case a file is corrupted, add the sourceId to the list of corrupted sources
-        corruptedSources.addCorruptedSource(sourceId, true, true, true, false);
+        corruptedSources.addCorruptedSource(sourceId, true, false, true, null);
+        continue;
+      }
+
+      // As an edge case, when the execution raise a system exception the report will contain
+      // no notices but system notices are found in the system_errors.json file.
+      // In this case, the system notices are not considered as corrupted sources.
+      if (hasSystemErrors(referenceReport, referenceErrorsPath)) {
+        corruptedSources.addCorruptedSource(sourceId, true, true, true, true);
+        continue;
+      }
+      if (hasSystemErrors(latestReport, latestErrorsPath)) {
+        corruptedSources.addCorruptedSource(sourceId, true, true, true, true);
         continue;
       }
+
       newErrors.compareValidationReports(sourceId, sourceUrl, referenceReport, latestReport);
       droppedErrors.compareValidationReports(sourceId, sourceUrl, latestReport, referenceReport);
       newWarnings.compareValidationReports(sourceId, sourceUrl, referenceReport, latestReport);
@@ -140,6 +151,25 @@ public Result compareValidationRuns(
     return Result.create(report, reportSummaryString, failure);
   }
 
+  private boolean hasSystemErrors(ValidationReport validationReport, Path path) {
+    var systemErrors = getValidationReport(path);
+    return validationReport.getNotices().isEmpty()
+        && systemErrors != null
+        && !systemErrors.getNotices().isEmpty();
+  }
+
+  private static ValidationReport getValidationReport(Path referenceReportPath) {
+    ValidationReport referenceReport;
+    try {
+      referenceReport = ValidationReport.fromPath(referenceReportPath);
+    } catch (IOException ioException) {
+      logger.atSevere().withCause(ioException).log(
+          "Error reading %s validation report", referenceReportPath);
+      return null;
+    }
+    return referenceReport;
+  }
+
   @AutoValue
   abstract static class Result {