Skip to content

Commit 7146461

Browse files
cka-ycka-y
authored
feat: added support + automated documentation extraction for summary metadata (#1974)
1 parent 409e5b4 commit 7146461

File tree

38 files changed

+1356
-335
lines changed

38 files changed

+1356
-335
lines changed

‎core/build.gradle‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,14 @@ plugins {
2525
// publishing and signing are done in gtfs-validator build.gradle to minimize repetition
2626
//}
2727

28+
tasks.withType(JavaCompile) {
29+
options.compilerArgs += ["-AsummaryMetadataOutputDir=${project.rootDir}/web/client/static"]
30+
}
31+
2832
dependencies {
2933
implementation project(':model')
3034
annotationProcessor project(':processor:notices')
35+
annotationProcessor project(':processor:summary')
3136
annotationProcessor libs.auto.value
3237
compileOnly libs.auto.value.annotations
3338
implementation libs.commons.compress

‎core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryUsage.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,43 @@
77
public class MemoryUsage {
88
private static final DecimalFormat TWO_DECIMAL_FORMAT = new DecimalFormat("0.00");
99

10+
/**
11+
* The key value associated with the method is the registered memory usage. Example,
12+
* GtfsFeedLoader.loadTables
13+
*/
1014
private String key;
15+
16+
/**
17+
* The total amount of memory currently available for current and future objects, measured in
18+
* bytes. <a
19+
* href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Runtime.html#totalMemory()"
20+
* target="_blank">Reference</a>
21+
*/
1122
private long totalMemory;
23+
24+
/**
25+
* The total amount of memory currently available for future allocated objects, measured in bytes.
26+
* <a
27+
* href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Runtime.html#freeMemory()"
28+
* target="_blank">Reference</a>
29+
*/
1230
private long freeMemory;
31+
32+
/**
33+
* The maximum amount of memory that the virtual machine will attempt to use, measured in bytes.
34+
* <a
35+
* href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Runtime.html#maxMemory()"
36+
* target="_blank">Reference</a>
37+
*/
1338
private long maxMemory;
39+
40+
/**
41+
* This is only applicable when the memory snapshot has the reference of its “previous” register,
42+
* for example, at the start and end of a method. The value reflects the free memory minus the
43+
* memory of the previous register(freeMemory - previousFreeMemory), measured in bytes. This
44+
* supports negative and positive values. Negative values mean the memory decreased at the end of
45+
* the method. Positive values mean the memory increased.
46+
*/
1447
private Long diffMemory;
1548

1649
public MemoryUsage() {}

main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/AgencyMetadata.java renamed to core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/AgencyMetadata.java

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
package org.mobilitydata.gtfsvalidator.report.model;
2-
3-
import org.mobilitydata.gtfsvalidator.table.GtfsAgency;
1+
package org.mobilitydata.gtfsvalidator.reportsummary;
42

53
public class AgencyMetadata {
64
public final String name;
@@ -16,13 +14,4 @@ public AgencyMetadata(String name, String url, String phone, String email, Strin
1614
this.email = email.isEmpty() ? "N/A" : email;
1715
this.timezone = timezone.isEmpty() ? "N/A" : timezone;
1816
}
19-
20-
public static AgencyMetadata from(GtfsAgency agency) {
21-
return new AgencyMetadata(
22-
agency.agencyName(),
23-
agency.agencyUrl(),
24-
agency.agencyPhone(),
25-
agency.agencyEmail(),
26-
agency.agencyTimezone().getId());
27-
}
2817
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportAgencyMetadata.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package org.mobilitydata.gtfsvalidator.reportsummary;
2+
3+
public class JsonReportAgencyMetadata {
4+
/** `agency_name` from `agency.txt`. Full name of the transit agency. */
5+
private final String name;
6+
7+
/** `agency_url` from `agency.txt`. URL of the transit agency. */
8+
private final String url;
9+
10+
/** `agency_phone` from `agency.txt`. A voice telephone number for the transit agency. */
11+
private final String phone;
12+
13+
/**
14+
* `agency_email` from `agency.txt`. Email address actively monitored by the agency’s customer
15+
* service department.
16+
*/
17+
private final String email;
18+
19+
/** `agency_timezone` from `agency.txt`. Timezone where the transit agency is located. */
20+
private final String timezone;
21+
22+
public JsonReportAgencyMetadata(AgencyMetadata agencyMetadata) {
23+
name = agencyMetadata.name;
24+
url = agencyMetadata.url;
25+
phone = agencyMetadata.phone;
26+
email = agencyMetadata.email;
27+
timezone = agencyMetadata.timezone;
28+
}
29+
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportCounts.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package org.mobilitydata.gtfsvalidator.reportsummary;
2+
3+
import com.google.gson.annotations.SerializedName;
4+
import java.util.Map;
5+
6+
public class JsonReportCounts {
7+
8+
/*
9+
* Use these strings as keys in the counts map. Also used to specify the info that will appear in
10+
* the json report. Adding elements to feedInfo will not automatically be included in the json
11+
* report and should be explicitly handled in the json report code.
12+
*/
13+
public static final String COUNTS_SHAPES = "Shapes";
14+
public static final String COUNTS_STOPS = "Stops";
15+
public static final String COUNTS_ROUTES = "Routes";
16+
public static final String COUNTS_TRIPS = "Trips";
17+
public static final String COUNTS_AGENCIES = "Agencies";
18+
public static final String COUNTS_BLOCKS = "Blocks";
19+
20+
public JsonReportCounts(Map<String, Integer> counts) {
21+
shapes = counts.get(COUNTS_SHAPES);
22+
stops = counts.get(COUNTS_STOPS);
23+
routes = counts.get(COUNTS_ROUTES);
24+
trips = counts.get(COUNTS_TRIPS);
25+
agencies = counts.get(COUNTS_AGENCIES);
26+
blocks = counts.get(COUNTS_BLOCKS);
27+
}
28+
29+
/** Number of shapes in `shapes.txt`. */
30+
@SerializedName("Shapes")
31+
Integer shapes;
32+
33+
/** Number of stops in `stops.txt`. */
34+
@SerializedName("Stops")
35+
Integer stops;
36+
37+
/** Number of routes in `routes.txt`. */
38+
@SerializedName("Routes")
39+
Integer routes;
40+
41+
/** Number of trips in `trips.txt`. */
42+
@SerializedName("Trips")
43+
Integer trips;
44+
45+
/** Number of agencies in `agency.txt`. */
46+
@SerializedName("Agencies")
47+
Integer agencies;
48+
49+
/** Number of blocks in `blocks.txt`. */
50+
@SerializedName("Blocks")
51+
Integer blocks;
52+
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportFeedInfo.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package org.mobilitydata.gtfsvalidator.reportsummary;
2+
3+
import java.util.Map;
4+
5+
/**
6+
* This class is used as a container of data to be converted to json using the builtin object
7+
* serialization of GSON. We use this class instead of the Map in FeedMetadata directly because the
8+
* schema of the resulting json file has to be stable and controlled as it becomes part of the
9+
* exported API.
10+
*/
11+
public class JsonReportFeedInfo {
12+
/*
13+
* Use these strings as keys in the FeedInfo map. Also used to specify the info that will appear
14+
* in the json report. Adding elements to feedInfo will not automatically be included in the json
15+
* report and should be explicitly handled in the json report code.
16+
*/
17+
public static final String FEED_INFO_PUBLISHER_NAME = "Publisher Name";
18+
public static final String FEED_INFO_PUBLISHER_URL = "Publisher URL";
19+
public static final String FEED_INFO_FEED_CONTACT_EMAIL = "Feed Email";
20+
public static final String FEED_INFO_FEED_LANGUAGE = "Feed Language";
21+
public static final String FEED_INFO_FEED_START_DATE = "Feed Start Date";
22+
public static final String FEED_INFO_FEED_END_DATE = "Feed End Date";
23+
public static final String FEED_INFO_SERVICE_WINDOW = "Service Window";
24+
public static final String FEED_INFO_SERVICE_WINDOW_START = "Service Window Start";
25+
public static final String FEED_INFO_SERVICE_WINDOW_END = "Service Window End";
26+
27+
public JsonReportFeedInfo(Map<String, String> feedInfo) {
28+
publisherName = feedInfo.get(FEED_INFO_PUBLISHER_NAME);
29+
publisherUrl = feedInfo.get(FEED_INFO_PUBLISHER_URL);
30+
feedEmail = feedInfo.get(FEED_INFO_FEED_CONTACT_EMAIL);
31+
feedLanguage = feedInfo.get(FEED_INFO_FEED_LANGUAGE);
32+
feedStartDate = feedInfo.get(FEED_INFO_FEED_START_DATE);
33+
feedEndDate = feedInfo.get(FEED_INFO_FEED_END_DATE);
34+
feedServiceWindowStart = feedInfo.get(FEED_INFO_SERVICE_WINDOW_START);
35+
feedServiceWindowEnd = feedInfo.get(FEED_INFO_SERVICE_WINDOW_END);
36+
}
37+
38+
/**
39+
* `feed_publisher_name` from `feed_info.txt. Full name of the organization that publishes the
40+
* dataset.
41+
*/
42+
String publisherName;
43+
44+
/**
45+
* `feed_publisher_url` from `feed_info.txt`. URL of the dataset publishing organization's
46+
* website.
47+
*/
48+
String publisherUrl;
49+
50+
/** `feed_lang` from `feed_info.txt`. Default language used for the text in this dataset. */
51+
String feedLanguage;
52+
53+
/**
54+
* `feed_start_date` from `feed_info.txt`. The dataset provides complete and reliable schedule
55+
* information for service in the period from the beginning of the `feed_start_date` day to the
56+
* end of the `feed_end_date` day.
57+
*/
58+
String feedStartDate;
59+
60+
/** `feed_end_date` from `feed_info.txt`. See above. */
61+
String feedEndDate;
62+
63+
/**
64+
* `feed_contact_email` from `feed_info.txt`. Email address for communication regarding the GTFS
65+
* dataset and data publishing practices.
66+
*/
67+
String feedEmail;
68+
69+
/**
70+
* The start date of the service, based on the earliest date referenced in `calendar.txt` or
71+
* `calendar_dates.txt` that is used by a `trip_id` in `trips.txt`.
72+
*/
73+
String feedServiceWindowStart;
74+
75+
/**
76+
* The end date of the service, based on the latest date referenced in `calendar.txt` or
77+
* `calendar_dates.txt` that is used by a `trip_id` in `trips.txt`.
78+
*/
79+
String feedServiceWindowEnd;
80+
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportSummary.java‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
package org.mobilitydata.gtfsvalidator.reportsummary;
2+
3+
import com.google.gson.annotations.SerializedName;
4+
import java.util.List;
5+
import java.util.Set;
6+
import org.mobilitydata.gtfsvalidator.annotation.GtfsReportSummary;
7+
import org.mobilitydata.gtfsvalidator.performance.MemoryUsage;
8+
9+
/**
10+
* Summary of the validation report. Each field in this class needs to be documented with Javadoc.
11+
*/
12+
@GtfsReportSummary
13+
public class JsonReportSummary {
14+
/**
15+
* The version of the validator used to produce the report, e.g <a
16+
* href="https://github.com/MobilityData/gtfs-validator/releases/tag/v6.0.0">6.0</a>.
17+
*/
18+
public final String validatorVersion;
19+
20+
/** Datetime of the validation. */
21+
public final String validatedAt;
22+
23+
/** File or URL used for the validation. */
24+
public final String gtfsInput;
25+
26+
/** Number of threads used for the validation. */
27+
public final int threads;
28+
29+
/** Output directory used to save the validation report. */
30+
public final String outputDirectory;
31+
32+
/** Filename of the report containing system errors generated during validation. */
33+
public final String systemErrorsReportName;
34+
35+
/** Filename of the JSON validation report. */
36+
public final String validationReportName;
37+
38+
/** Filename of the HTML validation report */
39+
public final String htmlReportName;
40+
41+
/**
42+
* The <a href="https://www.iso.org/iso-3166-country-codes.html">ISO 3166-1 Alpha 2</a> country
43+
* code for the region input by a validator user before the report is generated. Specifying the
44+
* region is optional and used to check the `invalid_phone_number` rule.
45+
*/
46+
public final String countryCode;
47+
48+
/** Date of the validation. */
49+
public final String dateForValidation;
50+
51+
/** Information about the dataset. */
52+
public final JsonReportFeedInfo feedInfo;
53+
54+
/** List of agencies in the feed based on `agency.txt`. */
55+
public final List<JsonReportAgencyMetadata> agencies;
56+
57+
/** List of GTFS files in the feed. */
58+
public final Set<String> files;
59+
60+
/**
61+
* The time it took to run the validation in seconds. This value is used internally for
62+
* performance metrics, and it can change in future versions. <a
63+
* href="https://github.com/MobilityData/gtfs-validator/pull/1963#issuecomment-2635037575"
64+
* target="_blank">Example of internal use</a>.
65+
*/
66+
public final Double validationTimeSeconds;
67+
68+
/**
69+
* List of details for the memory usage of the validation. These values are used internally for
70+
* performance metrics, and it can change in future versions. <a
71+
* href="https://github.com/MobilityData/gtfs-validator/pull/1963#issuecomment-2635037575"
72+
* target="_blank">Example of internal use</a>.
73+
*/
74+
public final List<MemoryUsage> memoryUsageRecords;
75+
76+
/** Number of entities in the feed. */
77+
@SerializedName("counts")
78+
public final JsonReportCounts jsonReportCounts;
79+
80+
/**
81+
* List of features in the dataset based on https://gtfs.org/getting-started/features/overview/.
82+
* You can review how the validator detects features <a
83+
* href="https://github.com/MobilityData/gtfs-validator/blob/577c2ceba1defea965620dc34af4fc9b26a32450/docs/FEATURES.md">here</a>.
84+
*/
85+
public final List<String> gtfsFeatures;
86+
87+
public JsonReportSummary(
88+
String validatorVersion,
89+
String validatedAt,
90+
String gtfsInput,
91+
int threads,
92+
String outputDirectory,
93+
String systemErrorsReportName,
94+
String validationReportName,
95+
String htmlReportName,
96+
String countryCode,
97+
String dateForValidation,
98+
JsonReportFeedInfo feedInfo,
99+
List<JsonReportAgencyMetadata> agencies,
100+
Set<String> files,
101+
Double validationTimeSeconds,
102+
List<MemoryUsage> memoryUsageRecords,
103+
JsonReportCounts jsonReportCounts,
104+
List<String> gtfsFeatures) {
105+
this.validatorVersion = validatorVersion;
106+
this.validatedAt = validatedAt;
107+
this.gtfsInput = gtfsInput;
108+
this.threads = threads;
109+
this.outputDirectory = outputDirectory;
110+
this.systemErrorsReportName = systemErrorsReportName;
111+
this.validationReportName = validationReportName;
112+
this.htmlReportName = htmlReportName;
113+
this.countryCode = countryCode;
114+
this.dateForValidation = dateForValidation;
115+
this.feedInfo = feedInfo;
116+
this.agencies = agencies;
117+
this.files = files;
118+
this.validationTimeSeconds = validationTimeSeconds;
119+
this.memoryUsageRecords = memoryUsageRecords;
120+
this.jsonReportCounts = jsonReportCounts;
121+
this.gtfsFeatures = gtfsFeatures;
122+
}
123+
}

‎gradle/libs.versions.toml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ jetbrainsAnnotations = "20.1.0"
3232
aspectjrt = "1.9.20"
3333
aspectjrtweaver = "1.9.20"
3434
findbugs = "3.0.2"
35+
jacksonDatabind = "2.14.1"
3536

3637
[plugins]
3738
spotless = { id = "com.diffplug.spotless", version.ref = "spotless" }
@@ -68,4 +69,5 @@ api-client = { module = "com.google.api-client:google-api-client", version.ref =
6869
jetbrains-annotations = { module = "org.jetbrains:annotations", version.ref = "jetbrainsAnnotations" }
6970
aspectjrt = { module = "org.aspectj:aspectjrt", version.ref = "aspectjrt" }
7071
aspectjrt-weaver = { module = "org.aspectj:aspectjweaver", version.ref = "aspectjrtweaver" }
71-
findbugs = { module = "com.google.code.findbugs:jsr305", version.ref = "findbugs" }
72+
findbugs = { module = "com.google.code.findbugs:jsr305", version.ref = "findbugs" }
73+
jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jacksonDatabind" }

0 commit comments

Comments
 (0)