feat: added support + automated documentation extraction for summary metadata

core/build.gradle

@@ -25,9 +25,14 @@ plugins {
 //    publishing and signing are done in gtfs-validator build.gradle to minimize repetition
 //}
 
+tasks.withType(JavaCompile) {
+    options.compilerArgs += ["-AsummaryMetadataOutputDir=${project.rootDir}/web/client/static"]
+}
+
 dependencies {
     implementation project(':model')
     annotationProcessor project(':processor:notices')
+    annotationProcessor project(':processor:summary')
     annotationProcessor libs.auto.value
     compileOnly libs.auto.value.annotations
     implementation libs.commons.compress

core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryUsage.java

@@ -7,10 +7,43 @@
 public class MemoryUsage {
   private static final DecimalFormat TWO_DECIMAL_FORMAT = new DecimalFormat("0.00");
 
+  /**
+   * The key value associated with the method is the registered memory usage. Example,
+   * GtfsFeedLoader.loadTables
+   */
   private String key;
+
+  /**
+   * The total amount of memory currently available for current and future objects, measured in
+   * bytes. <a
+   * href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Runtime.html#totalMemory()"
+   * target="_blank">Reference</a>
+   */
   private long totalMemory;
+
+  /**
+   * The total amount of memory currently available for future allocated objects, measured in bytes.
+   * <a
+   * href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Runtime.html#freeMemory()"
+   * target="_blank">Reference</a>
+   */
   private long freeMemory;
+
+  /**
+   * The maximum amount of memory that the virtual machine will attempt to use, measured in bytes.
+   * <a
+   * href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Runtime.html#maxMemory()"
+   * target="_blank">Reference</a>
+   */
   private long maxMemory;
+
+  /**
+   * This is only applicable when the memory snapshot has the reference of its “previous” register,
+   * for example, at the start and end of a method. The value reflects the free memory minus the
+   * memory of the previous register(freeMemory - previousFreeMemory), measured in bytes. This
+   * supports negative and positive values. Negative values mean the memory decreased at the end of
+   * the method. Positive values mean the memory increased.
+   */
   private Long diffMemory;
 
   public MemoryUsage() {}

main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/AgencyMetadata.java → core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/AgencyMetadata.java

@@ -1,6 +1,4 @@
-package org.mobilitydata.gtfsvalidator.report.model;
-
-import org.mobilitydata.gtfsvalidator.table.GtfsAgency;
+package org.mobilitydata.gtfsvalidator.reportsummary;
 
 public class AgencyMetadata {
   public final String name;
@@ -16,13 +14,4 @@ public AgencyMetadata(String name, String url, String phone, String email, Strin
     this.email = email.isEmpty() ? "N/A" : email;
     this.timezone = timezone.isEmpty() ? "N/A" : timezone;
   }
-
-  public static AgencyMetadata from(GtfsAgency agency) {
-    return new AgencyMetadata(
-        agency.agencyName(),
-        agency.agencyUrl(),
-        agency.agencyPhone(),
-        agency.agencyEmail(),
-        agency.agencyTimezone().getId());
-  }
 }

core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportAgencyMetadata.java

@@ -0,0 +1,29 @@
+package org.mobilitydata.gtfsvalidator.reportsummary;
+
+public class JsonReportAgencyMetadata {
+  /** `agency_name` from `agency.txt`. Full name of the transit agency. */
+  private final String name;
+
+  /** `agency_url` from `agency.txt`. URL of the transit agency. */
+  private final String url;
+
+  /** `agency_phone` from `agency.txt`. A voice telephone number for the transit agency. */
+  private final String phone;
+
+  /**
+   * `agency_email` from `agency.txt`. Email address actively monitored by the agency’s customer
+   * service department.
+   */
+  private final String email;
+
+  /** `agency_timezone` from `agency.txt`. Timezone where the transit agency is located. */
+  private final String timezone;
+
+  public JsonReportAgencyMetadata(AgencyMetadata agencyMetadata) {
+    name = agencyMetadata.name;
+    url = agencyMetadata.url;
+    phone = agencyMetadata.phone;
+    email = agencyMetadata.email;
+    timezone = agencyMetadata.timezone;
+  }
+}

core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportCounts.java

@@ -0,0 +1,52 @@
+package org.mobilitydata.gtfsvalidator.reportsummary;
+
+import com.google.gson.annotations.SerializedName;
+import java.util.Map;
+
+public class JsonReportCounts {
+
+  /*
+   * Use these strings as keys in the counts map. Also used to specify the info that will appear in
+   * the json report. Adding elements to feedInfo will not automatically be included in the json
+   * report and should be explicitly handled in the json report code.
+   */
+  public static final String COUNTS_SHAPES = "Shapes";
+  public static final String COUNTS_STOPS = "Stops";
+  public static final String COUNTS_ROUTES = "Routes";
+  public static final String COUNTS_TRIPS = "Trips";
+  public static final String COUNTS_AGENCIES = "Agencies";
+  public static final String COUNTS_BLOCKS = "Blocks";
+
+  public JsonReportCounts(Map<String, Integer> counts) {
+    shapes = counts.get(COUNTS_SHAPES);
+    stops = counts.get(COUNTS_STOPS);
+    routes = counts.get(COUNTS_ROUTES);
+    trips = counts.get(COUNTS_TRIPS);
+    agencies = counts.get(COUNTS_AGENCIES);
+    blocks = counts.get(COUNTS_BLOCKS);
+  }
+
+  /** Number of shapes in `shapes.txt`. */
+  @SerializedName("Shapes")
+  Integer shapes;
+
+  /** Number of stops in `stops.txt`. */
+  @SerializedName("Stops")
+  Integer stops;
+
+  /** Number of routes in `routes.txt`. */
+  @SerializedName("Routes")
+  Integer routes;
+
+  /** Number of trips in `trips.txt`. */
+  @SerializedName("Trips")
+  Integer trips;
+
+  /** Number of agencies in `agency.txt`. */
+  @SerializedName("Agencies")
+  Integer agencies;
+
+  /** Number of blocks in `blocks.txt`. */
+  @SerializedName("Blocks")
+  Integer blocks;
+}

core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportFeedInfo.java

@@ -0,0 +1,80 @@
+package org.mobilitydata.gtfsvalidator.reportsummary;
+
+import java.util.Map;
+
+/**
+ * This class is used as a container of data to be converted to json using the builtin object
+ * serialization of GSON. We use this class instead of the Map in FeedMetadata directly because the
+ * schema of the resulting json file has to be stable and controlled as it becomes part of the
+ * exported API.
+ */
+public class JsonReportFeedInfo {
+  /*
+   * Use these strings as keys in the FeedInfo map. Also used to specify the info that will appear
+   * in the json report. Adding elements to feedInfo will not automatically be included in the json
+   * report and should be explicitly handled in the json report code.
+   */
+  public static final String FEED_INFO_PUBLISHER_NAME = "Publisher Name";
+  public static final String FEED_INFO_PUBLISHER_URL = "Publisher URL";
+  public static final String FEED_INFO_FEED_CONTACT_EMAIL = "Feed Email";
+  public static final String FEED_INFO_FEED_LANGUAGE = "Feed Language";
+  public static final String FEED_INFO_FEED_START_DATE = "Feed Start Date";
+  public static final String FEED_INFO_FEED_END_DATE = "Feed End Date";
+  public static final String FEED_INFO_SERVICE_WINDOW = "Service Window";
+  public static final String FEED_INFO_SERVICE_WINDOW_START = "Service Window Start";
+  public static final String FEED_INFO_SERVICE_WINDOW_END = "Service Window End";
+
+  public JsonReportFeedInfo(Map<String, String> feedInfo) {
+    publisherName = feedInfo.get(FEED_INFO_PUBLISHER_NAME);
+    publisherUrl = feedInfo.get(FEED_INFO_PUBLISHER_URL);
+    feedEmail = feedInfo.get(FEED_INFO_FEED_CONTACT_EMAIL);
+    feedLanguage = feedInfo.get(FEED_INFO_FEED_LANGUAGE);
+    feedStartDate = feedInfo.get(FEED_INFO_FEED_START_DATE);
+    feedEndDate = feedInfo.get(FEED_INFO_FEED_END_DATE);
+    feedServiceWindowStart = feedInfo.get(FEED_INFO_SERVICE_WINDOW_START);
+    feedServiceWindowEnd = feedInfo.get(FEED_INFO_SERVICE_WINDOW_END);
+  }
+
+  /**
+   * `feed_publisher_name` from `feed_info.txt. Full name of the organization that publishes the
+   * dataset.
+   */
+  String publisherName;
+
+  /**
+   * `feed_publisher_url` from `feed_info.txt`. URL of the dataset publishing organization's
+   * website.
+   */
+  String publisherUrl;
+
+  /** `feed_lang` from `feed_info.txt`. Default language used for the text in this dataset. */
+  String feedLanguage;
+
+  /**
+   * `feed_start_date` from `feed_info.txt`. The dataset provides complete and reliable schedule
+   * information for service in the period from the beginning of the `feed_start_date` day to the
+   * end of the `feed_end_date` day.
+   */
+  String feedStartDate;
+
+  /** `feed_end_date` from `feed_info.txt`. See above. */
+  String feedEndDate;
+
+  /**
+   * `feed_contact_email` from `feed_info.txt`. Email address for communication regarding the GTFS
+   * dataset and data publishing practices.
+   */
+  String feedEmail;
+
+  /**
+   * The start date of the service, based on the earliest date referenced in `calendar.txt` or
+   * `calendar_dates.txt` that is used by a `trip_id` in `trips.txt`.
+   */
+  String feedServiceWindowStart;
+
+  /**
+   * The end date of the service, based on the latest date referenced in `calendar.txt` or
+   * `calendar_dates.txt` that is used by a `trip_id` in `trips.txt`.
+   */
+  String feedServiceWindowEnd;
+}

core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportSummary.java

@@ -0,0 +1,123 @@
+package org.mobilitydata.gtfsvalidator.reportsummary;
+
+import com.google.gson.annotations.SerializedName;
+import java.util.List;
+import java.util.Set;
+import org.mobilitydata.gtfsvalidator.annotation.GtfsReportSummary;
+import org.mobilitydata.gtfsvalidator.performance.MemoryUsage;
+
+/**
+ * Summary of the validation report. Each field in this class needs to be documented with Javadoc.
+ */
+@GtfsReportSummary
+public class JsonReportSummary {
+  /**
+   * The version of the validator used to produce the report, e.g <a
+   * href="https://github.com/MobilityData/gtfs-validator/releases/tag/v6.0.0">6.0</a>.
+   */
+  public final String validatorVersion;
+
+  /** Datetime of the validation. */
+  public final String validatedAt;
+
+  /** File or URL used for the validation. */
+  public final String gtfsInput;
+
+  /** Number of threads used for the validation. */
+  public final int threads;
+
+  /** Output directory used to save the validation report. */
+  public final String outputDirectory;
+
+  /** Filename of the report containing system errors generated during validation. */
+  public final String systemErrorsReportName;
+
+  /** Filename of the JSON validation report. */
+  public final String validationReportName;
+
+  /** Filename of the HTML validation report */
+  public final String htmlReportName;
+
+  /**
+   * The <a href="https://www.iso.org/iso-3166-country-codes.html">ISO 3166-1 Alpha 2</a> country
+   * code for the region input by a validator user before the report is generated. Specifying the
+   * region is optional and used to check the `invalid_phone_number` rule.
+   */
+  public final String countryCode;
+
+  /** Date of the validation. */
+  public final String dateForValidation;
+
+  /** Information about the dataset. */
+  public final JsonReportFeedInfo feedInfo;
+
+  /** List of agencies in the feed based on `agency.txt`. */
+  public final List<JsonReportAgencyMetadata> agencies;
+
+  /** List of GTFS files in the feed. */
+  public final Set<String> files;
+
+  /**
+   * The time it took to run the validation in seconds. This value is used internally for
+   * performance metrics, and it can change in future versions. <a
+   * href="https://github.com/MobilityData/gtfs-validator/pull/1963#issuecomment-2635037575"
+   * target="_blank">Example of internal use</a>.
+   */
+  public final Double validationTimeSeconds;
+
+  /**
+   * List of details for the memory usage of the validation. These values are used internally for
+   * performance metrics, and it can change in future versions. <a
+   * href="https://github.com/MobilityData/gtfs-validator/pull/1963#issuecomment-2635037575"
+   * target="_blank">Example of internal use</a>.
+   */
+  public final List<MemoryUsage> memoryUsageRecords;
+
+  /** Number of entities in the feed. */
+  @SerializedName("counts")
+  public final JsonReportCounts jsonReportCounts;
+
+  /**
+   * List of features in the dataset based on https://gtfs.org/getting-started/features/overview/.
+   * You can review how the validator detects features <a
+   * href="https://github.com/MobilityData/gtfs-validator/blob/577c2ceba1defea965620dc34af4fc9b26a32450/docs/FEATURES.md">here</a>.
+   */
+  public final List<String> gtfsFeatures;
+
+  public JsonReportSummary(
+      String validatorVersion,
+      String validatedAt,
+      String gtfsInput,
+      int threads,
+      String outputDirectory,
+      String systemErrorsReportName,
+      String validationReportName,
+      String htmlReportName,
+      String countryCode,
+      String dateForValidation,
+      JsonReportFeedInfo feedInfo,
+      List<JsonReportAgencyMetadata> agencies,
+      Set<String> files,
+      Double validationTimeSeconds,
+      List<MemoryUsage> memoryUsageRecords,
+      JsonReportCounts jsonReportCounts,
+      List<String> gtfsFeatures) {
+    this.validatorVersion = validatorVersion;
+    this.validatedAt = validatedAt;
+    this.gtfsInput = gtfsInput;
+    this.threads = threads;
+    this.outputDirectory = outputDirectory;
+    this.systemErrorsReportName = systemErrorsReportName;
+    this.validationReportName = validationReportName;
+    this.htmlReportName = htmlReportName;
+    this.countryCode = countryCode;
+    this.dateForValidation = dateForValidation;
+    this.feedInfo = feedInfo;
+    this.agencies = agencies;
+    this.files = files;
+    this.validationTimeSeconds = validationTimeSeconds;
+    this.memoryUsageRecords = memoryUsageRecords;
+    this.jsonReportCounts = jsonReportCounts;
+    this.gtfsFeatures = gtfsFeatures;
+  }
+}

gradle/libs.versions.toml

@@ -32,6 +32,7 @@ jetbrainsAnnotations = "20.1.0"
 aspectjrt = "1.9.20"
 aspectjrtweaver = "1.9.20"
 findbugs = "3.0.2"
+jacksonDatabind = "2.14.1"
 
 [plugins]
 spotless = { id = "com.diffplug.spotless", version.ref = "spotless" }
@@ -68,4 +69,5 @@ api-client = { module = "com.google.api-client:google-api-client", version.ref =
 jetbrains-annotations = { module = "org.jetbrains:annotations", version.ref = "jetbrainsAnnotations" }
 aspectjrt = { module = "org.aspectj:aspectjrt", version.ref = "aspectjrt" }
 aspectjrt-weaver = { module = "org.aspectj:aspectjweaver", version.ref = "aspectjrtweaver" }
-findbugs = { module = "com.google.code.findbugs:jsr305", version.ref = "findbugs" }
+findbugs = { module = "com.google.code.findbugs:jsr305", version.ref = "findbugs" }
+jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jacksonDatabind" }