Skip to content

Commit 0fd4eb2

Browse files
qcdyxqcdyx
authored
Merge branch 'main' into 1434-add-external-id-description-to-api-docs
2 parents 45128de + 8a89d38 commit 0fd4eb2

File tree

2 files changed

+253
-0
lines changed

2 files changed

+253
-0
lines changed

‎liquibase/changelog.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,4 +89,6 @@
8989
<!-- Enhance license queries on id and name columns. -->
9090
<!-- Individual transactions are needed to create these indexes, this is why XML file is used -->
9191
<include file="changes/feat_1432_indexes.xml" relativeToChangelogFile="true"/>
92+
<!-- Materialized view recreated - changed hosted_url column type to text to accomodate longer URLs. Materialized view is the same as feat_1396.sql -->
93+
<include file="changes/feat_1507.sql" relativeToChangelogFile="true"/>
9294
</databaseChangeLog>

‎liquibase/changes/feat_1507.sql‎

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
-- Change hosted_url column type to text to accommodate longer URLs
2+
DROP MATERIALIZED VIEW IF EXISTS feedsearch;
3+
ALTER TABLE gtfsdataset ALTER COLUMN hosted_url TYPE text;
4+
-- Recreate the FeedSearch materialized view.
5+
-- This CREATE MATERIALIZED VIEW is the same as the one in feat_1396.sql, with no change.
6+
CREATE MATERIALIZED VIEW FeedSearch AS
7+
SELECT
8+
-- feed
9+
Feed.stable_id AS feed_stable_id,
10+
Feed.id AS feed_id,
11+
Feed.data_type,
12+
Feed.status,
13+
Feed.feed_name,
14+
Feed.note,
15+
Feed.feed_contact_email,
16+
-- source
17+
Feed.producer_url,
18+
Feed.authentication_info_url,
19+
Feed.authentication_type,
20+
Feed.api_key_parameter_name,
21+
Feed.license_url,
22+
Feed.provider,
23+
Feed.operational_status,
24+
-- official status
25+
Feed.official AS official,
26+
-- created_at
27+
Feed.created_at AS created_at,
28+
-- latest_dataset
29+
Latest_dataset.stable_id AS latest_dataset_id,
30+
Latest_dataset.hosted_url AS latest_dataset_hosted_url,
31+
Latest_dataset.downloaded_at AS latest_dataset_downloaded_at,
32+
Latest_dataset.bounding_box AS latest_dataset_bounding_box,
33+
Latest_dataset.hash AS latest_dataset_hash,
34+
Latest_dataset.agency_timezone AS latest_dataset_agency_timezone,
35+
Latest_dataset.service_date_range_start AS latest_dataset_service_date_range_start,
36+
Latest_dataset.service_date_range_end AS latest_dataset_service_date_range_end,
37+
-- Latest dataset features
38+
LatestDatasetFeatures AS latest_dataset_features,
39+
-- Latest dataset validation totals
40+
COALESCE(LatestDatasetValidationReportJoin.total_error, 0) as latest_total_error,
41+
COALESCE(LatestDatasetValidationReportJoin.total_warning, 0) as latest_total_warning,
42+
COALESCE(LatestDatasetValidationReportJoin.total_info, 0) as latest_total_info,
43+
COALESCE(LatestDatasetValidationReportJoin.unique_error_count, 0) as latest_unique_error_count,
44+
COALESCE(LatestDatasetValidationReportJoin.unique_warning_count, 0) as latest_unique_warning_count,
45+
COALESCE(LatestDatasetValidationReportJoin.unique_info_count, 0) as latest_unique_info_count,
46+
-- external_ids
47+
ExternalIdJoin.external_ids,
48+
-- redirect_ids
49+
RedirectingIdJoin.redirect_ids,
50+
-- feed gtfs_rt references
51+
FeedReferenceJoin.feed_reference_ids,
52+
-- feed gtfs_rt entities
53+
EntityTypeFeedJoin.entities,
54+
-- locations
55+
FeedLocationJoin.locations,
56+
-- osm locations grouped
57+
OsmLocationJoin.osm_locations,
58+
-- gbfs versions
59+
COALESCE(GbfsVersionsJoin.versions, '[]'::jsonb) AS versions,
60+
61+
-- full-text searchable document
62+
setweight(to_tsvector('english', coalesce(unaccent(Feed.feed_name), '')), 'C') ||
63+
setweight(to_tsvector('english', coalesce(unaccent(Feed.provider), '')), 'C') ||
64+
setweight(to_tsvector('english', coalesce(unaccent((
65+
SELECT string_agg(
66+
coalesce(location->>'country_code', '') || ' ' ||
67+
coalesce(location->>'country', '') || ' ' ||
68+
coalesce(location->>'subdivision_name', '') || ' ' ||
69+
coalesce(location->>'municipality', ''),
70+
' '
71+
)
72+
FROM json_array_elements(FeedLocationJoin.locations) AS location
73+
)), '')), 'A') ||
74+
setweight(to_tsvector('english', coalesce(unaccent(OsmLocationNamesJoin.osm_location_names), '')), 'A')
75+
AS document
76+
FROM Feed
77+
78+
-- Latest dataset
79+
LEFT JOIN gtfsfeed gtf ON gtf.id = Feed.id AND Feed.data_type = 'gtfs'
80+
LEFT JOIN gtfsdataset Latest_dataset ON Latest_dataset.id = gtf.latest_dataset_id
81+
82+
-- Latest dataset features
83+
LEFT JOIN (
84+
SELECT
85+
GtfsDataset.id AS FeatureGtfsDatasetId,
86+
array_agg(DISTINCT FeatureValidationReport.feature) AS LatestDatasetFeatures
87+
FROM GtfsDataset
88+
JOIN ValidationReportGtfsDataset
89+
ON ValidationReportGtfsDataset.dataset_id = GtfsDataset.id
90+
JOIN (
91+
-- Pick latest ValidationReport per dataset based on validated_at
92+
SELECT DISTINCT ON (ValidationReportGtfsDataset.dataset_id)
93+
ValidationReportGtfsDataset.dataset_id,
94+
ValidationReport.id AS latest_validation_report_id
95+
FROM ValidationReportGtfsDataset
96+
JOIN ValidationReport
97+
ON ValidationReport.id = ValidationReportGtfsDataset.validation_report_id
98+
ORDER BY
99+
ValidationReportGtfsDataset.dataset_id,
100+
ValidationReport.validated_at DESC
101+
) AS LatestReports
102+
ON LatestReports.latest_validation_report_id = ValidationReportGtfsDataset.validation_report_id
103+
JOIN FeatureValidationReport
104+
ON FeatureValidationReport.validation_id = ValidationReportGtfsDataset.validation_report_id
105+
GROUP BY FeatureGtfsDatasetId
106+
) AS LatestDatasetFeaturesJoin ON Latest_dataset.id = FeatureGtfsDatasetId
107+
108+
-- Latest dataset validation report
109+
LEFT JOIN (
110+
SELECT
111+
GtfsDataset.id AS ValidationReportGtfsDatasetId,
112+
ValidationReport.total_error,
113+
ValidationReport.total_warning,
114+
ValidationReport.total_info,
115+
ValidationReport.unique_error_count,
116+
ValidationReport.unique_warning_count,
117+
ValidationReport.unique_info_count
118+
FROM GtfsDataset
119+
JOIN ValidationReportGtfsDataset
120+
ON ValidationReportGtfsDataset.dataset_id = GtfsDataset.id
121+
JOIN (
122+
-- Pick latest ValidationReport per dataset based on validated_at
123+
SELECT DISTINCT ON (ValidationReportGtfsDataset.dataset_id)
124+
ValidationReportGtfsDataset.dataset_id,
125+
ValidationReport.id AS latest_validation_report_id
126+
FROM ValidationReportGtfsDataset
127+
JOIN ValidationReport
128+
ON ValidationReport.id = ValidationReportGtfsDataset.validation_report_id
129+
ORDER BY
130+
ValidationReportGtfsDataset.dataset_id,
131+
ValidationReport.validated_at DESC
132+
) AS LatestReports
133+
ON LatestReports.latest_validation_report_id = ValidationReportGtfsDataset.validation_report_id
134+
JOIN ValidationReport
135+
ON ValidationReport.id = ValidationReportGtfsDataset.validation_report_id
136+
) AS LatestDatasetValidationReportJoin ON Latest_dataset.id = ValidationReportGtfsDatasetId
137+
138+
-- External ids
139+
LEFT JOIN (
140+
SELECT
141+
feed_id,
142+
json_agg(json_build_object('external_id', associated_id, 'source', source)) AS external_ids
143+
FROM externalid
144+
GROUP BY feed_id
145+
) AS ExternalIdJoin ON ExternalIdJoin.feed_id = Feed.id
146+
147+
-- feed reference ids
148+
LEFT JOIN (
149+
SELECT
150+
gtfs_rt_feed_id,
151+
array_agg(FeedReferenceJoinInnerQuery.stable_id) AS feed_reference_ids
152+
FROM FeedReference
153+
LEFT JOIN Feed AS FeedReferenceJoinInnerQuery ON FeedReferenceJoinInnerQuery.id = FeedReference.gtfs_feed_id
154+
GROUP BY gtfs_rt_feed_id
155+
) AS FeedReferenceJoin ON FeedReferenceJoin.gtfs_rt_feed_id = Feed.id AND Feed.data_type = 'gtfs_rt'
156+
157+
-- Redirect ids
158+
-- Redirect ids
159+
LEFT JOIN (
160+
SELECT
161+
r.target_id,
162+
json_agg(json_build_object('target_id', f.stable_id, 'comment', r.redirect_comment)) AS redirect_ids
163+
FROM RedirectingId r
164+
JOIN Feed f ON r.target_id = f.id
165+
GROUP BY r.target_id
166+
) AS RedirectingIdJoin ON RedirectingIdJoin.target_id = Feed.id
167+
-- Feed locations
168+
LEFT JOIN (
169+
SELECT
170+
LocationFeed.feed_id,
171+
json_agg(json_build_object('country', country, 'country_code', country_code, 'subdivision_name',
172+
subdivision_name, 'municipality', municipality)) AS locations
173+
FROM Location
174+
LEFT JOIN LocationFeed ON LocationFeed.location_id = Location.id
175+
GROUP BY LocationFeed.feed_id
176+
) AS FeedLocationJoin ON FeedLocationJoin.feed_id = Feed.id
177+
178+
-- Entity types
179+
LEFT JOIN (
180+
SELECT
181+
feed_id,
182+
array_agg(entity_name) AS entities
183+
FROM EntityTypeFeed
184+
GROUP BY feed_id
185+
) AS EntityTypeFeedJoin ON EntityTypeFeedJoin.feed_id = Feed.id AND Feed.data_type = 'gtfs_rt'
186+
187+
-- OSM locations
188+
LEFT JOIN (
189+
WITH locations_per_group AS (
190+
SELECT
191+
fog.feed_id,
192+
olg.group_name,
193+
jsonb_agg(
194+
DISTINCT jsonb_build_object(
195+
'admin_level', gp.admin_level,
196+
'name', gp.name
197+
)
198+
) AS locations
199+
FROM FeedOsmLocationGroup fog
200+
JOIN OsmLocationGroup olg ON olg.group_id = fog.group_id
201+
JOIN OsmLocationGroupGeopolygon olgg ON olgg.group_id = olg.group_id
202+
JOIN Geopolygon gp ON gp.osm_id = olgg.osm_id
203+
GROUP BY fog.feed_id, olg.group_name
204+
)
205+
SELECT
206+
feed_id,
207+
jsonb_agg(
208+
jsonb_build_object(
209+
'group_name', group_name,
210+
'locations', locations
211+
)
212+
)::json AS osm_locations
213+
FROM locations_per_group
214+
GROUP BY feed_id
215+
) AS OsmLocationJoin ON OsmLocationJoin.feed_id = Feed.id
216+
217+
-- OSM location names
218+
LEFT JOIN (
219+
SELECT
220+
fog.feed_id,
221+
string_agg(DISTINCT gp.name, ' ') AS osm_location_names
222+
FROM FeedOsmLocationGroup fog
223+
JOIN OsmLocationGroup olg ON olg.group_id = fog.group_id
224+
JOIN OsmLocationGroupGeopolygon olgg ON olgg.group_id = olg.group_id
225+
JOIN Geopolygon gp ON gp.osm_id = olgg.osm_id
226+
WHERE gp.name IS NOT NULL
227+
GROUP BY fog.feed_id
228+
) AS OsmLocationNamesJoin ON OsmLocationNamesJoin.feed_id = Feed.id
229+
230+
-- GBFS versions
231+
LEFT JOIN (
232+
SELECT
233+
Feed.id AS feed_id,
234+
to_jsonb(array_agg(DISTINCT GbfsVersion.version ORDER BY GbfsVersion.version)) AS versions
235+
FROM Feed
236+
JOIN GbfsFeed ON GbfsFeed.id = Feed.id
237+
JOIN GbfsVersion ON GbfsVersion.feed_id = GbfsFeed.id
238+
WHERE Feed.data_type = 'gbfs'
239+
GROUP BY Feed.id
240+
) AS GbfsVersionsJoin ON GbfsVersionsJoin.feed_id = Feed.id;
241+
242+
243+
-- This index allows concurrent refresh on the materialized view avoiding table locks
244+
CREATE UNIQUE INDEX idx_unique_feed_id ON FeedSearch(feed_id);
245+
246+
-- Indices for feedsearch view optimization
247+
CREATE INDEX feedsearch_document_idx ON FeedSearch USING GIN(document);
248+
CREATE INDEX feedsearch_feed_stable_id ON FeedSearch(feed_stable_id);
249+
CREATE INDEX feedsearch_data_type ON FeedSearch(data_type);
250+
CREATE INDEX feedsearch_status ON FeedSearch(status);
251+

0 commit comments

Comments
 (0)