Skip to content

Commit 99fbdfb

Browse files
qcdyxqcdyx
committed
added created_at to FeedSearch materialized view
1 parent 31000a6 commit 99fbdfb

File tree

2 files changed

+252
-0
lines changed

2 files changed

+252
-0
lines changed

‎liquibase/changelog.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,4 +57,6 @@
5757
<!-- Materialized view updated. Added versions of GBFS feeds-->
5858
<include file="changes/feat_1118.sql" relativeToChangelogFile="true"/>
5959
<include file="changes/feat_1125.sql" relativeToChangelogFile="true"/>
60+
<!-- Materialized view updated. Used Feed.created_at field as timestamp. -->
61+
<include file="changes/feat_1082.sql" relativeToChangelogFile="true"/>
6062
</databaseChangeLog>

‎liquibase/changes/feat_1082.sql‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
-- Updating the FeedSearch materialized view to include Feed.created_at field
2+
3+
DROP MATERIALIZED VIEW IF EXISTS FeedSearch;
4+
CREATE MATERIALIZED VIEW FeedSearch AS
5+
SELECT
6+
-- feed
7+
Feed.stable_id AS feed_stable_id,
8+
Feed.id AS feed_id,
9+
Feed.data_type,
10+
Feed.status,
11+
Feed.feed_name,
12+
Feed.note,
13+
Feed.feed_contact_email,
14+
-- source
15+
Feed.producer_url,
16+
Feed.authentication_info_url,
17+
Feed.authentication_type,
18+
Feed.api_key_parameter_name,
19+
Feed.license_url,
20+
Feed.provider,
21+
Feed.operational_status,
22+
-- official status
23+
Feed.official AS official,
24+
-- created_at
25+
Feed.created_at AS created_at,
26+
-- latest_dataset
27+
Latest_dataset.id AS latest_dataset_id,
28+
Latest_dataset.hosted_url AS latest_dataset_hosted_url,
29+
Latest_dataset.downloaded_at AS latest_dataset_downloaded_at,
30+
Latest_dataset.bounding_box AS latest_dataset_bounding_box,
31+
Latest_dataset.hash AS latest_dataset_hash,
32+
Latest_dataset.agency_timezone AS latest_dataset_agency_timezone,
33+
Latest_dataset.service_date_range_start AS latest_dataset_service_date_range_start,
34+
Latest_dataset.service_date_range_end AS latest_dataset_service_date_range_end,
35+
-- Latest dataset features
36+
LatestDatasetFeatures AS latest_dataset_features,
37+
-- Latest dataset validation totals
38+
COALESCE(LatestDatasetValidationReportJoin.total_error, 0) as latest_total_error,
39+
COALESCE(LatestDatasetValidationReportJoin.total_warning, 0) as latest_total_warning,
40+
COALESCE(LatestDatasetValidationReportJoin.total_info, 0) as latest_total_info,
41+
COALESCE(LatestDatasetValidationReportJoin.unique_error_count, 0) as latest_unique_error_count,
42+
COALESCE(LatestDatasetValidationReportJoin.unique_warning_count, 0) as latest_unique_warning_count,
43+
COALESCE(LatestDatasetValidationReportJoin.unique_info_count, 0) as latest_unique_info_count,
44+
-- external_ids
45+
ExternalIdJoin.external_ids,
46+
-- redirect_ids
47+
RedirectingIdJoin.redirect_ids,
48+
-- feed gtfs_rt references
49+
FeedReferenceJoin.feed_reference_ids,
50+
-- feed gtfs_rt entities
51+
EntityTypeFeedJoin.entities,
52+
-- locations
53+
FeedLocationJoin.locations,
54+
-- osm locations grouped
55+
OsmLocationJoin.osm_locations,
56+
-- gbfs versions
57+
COALESCE(GbfsVersionsJoin.versions, '[]'::jsonb) AS versions,
58+
59+
-- full-text searchable document
60+
setweight(to_tsvector('english', coalesce(unaccent(Feed.feed_name), '')), 'C') ||
61+
setweight(to_tsvector('english', coalesce(unaccent(Feed.provider), '')), 'C') ||
62+
setweight(to_tsvector('english', coalesce(unaccent((
63+
SELECT string_agg(
64+
coalesce(location->>'country_code', '') || ' ' ||
65+
coalesce(location->>'country', '') || ' ' ||
66+
coalesce(location->>'subdivision_name', '') || ' ' ||
67+
coalesce(location->>'municipality', ''),
68+
' '
69+
)
70+
FROM json_array_elements(FeedLocationJoin.locations) AS location
71+
)), '')), 'A') ||
72+
setweight(to_tsvector('english', coalesce(unaccent(OsmLocationNamesJoin.osm_location_names), '')), 'A')
73+
AS document
74+
FROM Feed
75+
76+
-- Latest dataset
77+
LEFT JOIN (
78+
SELECT *
79+
FROM gtfsdataset
80+
WHERE latest = true
81+
) AS Latest_dataset ON Latest_dataset.feed_id = Feed.id AND Feed.data_type = 'gtfs'
82+
83+
-- Latest dataset features
84+
LEFT JOIN (
85+
SELECT
86+
GtfsDataset.id AS FeatureGtfsDatasetId,
87+
array_agg(DISTINCT FeatureValidationReport.feature) AS LatestDatasetFeatures
88+
FROM GtfsDataset
89+
JOIN ValidationReportGtfsDataset
90+
ON ValidationReportGtfsDataset.dataset_id = GtfsDataset.id
91+
JOIN (
92+
-- Pick latest ValidationReport per dataset based on validated_at
93+
SELECT DISTINCT ON (ValidationReportGtfsDataset.dataset_id)
94+
ValidationReportGtfsDataset.dataset_id,
95+
ValidationReport.id AS latest_validation_report_id
96+
FROM ValidationReportGtfsDataset
97+
JOIN ValidationReport
98+
ON ValidationReport.id = ValidationReportGtfsDataset.validation_report_id
99+
ORDER BY
100+
ValidationReportGtfsDataset.dataset_id,
101+
ValidationReport.validated_at DESC
102+
) AS LatestReports
103+
ON LatestReports.latest_validation_report_id = ValidationReportGtfsDataset.validation_report_id
104+
JOIN FeatureValidationReport
105+
ON FeatureValidationReport.validation_id = ValidationReportGtfsDataset.validation_report_id
106+
GROUP BY FeatureGtfsDatasetId
107+
) AS LatestDatasetFeaturesJoin ON Latest_dataset.id = FeatureGtfsDatasetId
108+
109+
-- Latest dataset validation report
110+
LEFT JOIN (
111+
SELECT
112+
GtfsDataset.id AS ValidationReportGtfsDatasetId,
113+
ValidationReport.total_error,
114+
ValidationReport.total_warning,
115+
ValidationReport.total_info,
116+
ValidationReport.unique_error_count,
117+
ValidationReport.unique_warning_count,
118+
ValidationReport.unique_info_count
119+
FROM GtfsDataset
120+
JOIN ValidationReportGtfsDataset
121+
ON ValidationReportGtfsDataset.dataset_id = GtfsDataset.id
122+
JOIN (
123+
-- Pick latest ValidationReport per dataset based on validated_at
124+
SELECT DISTINCT ON (ValidationReportGtfsDataset.dataset_id)
125+
ValidationReportGtfsDataset.dataset_id,
126+
ValidationReport.id AS latest_validation_report_id
127+
FROM ValidationReportGtfsDataset
128+
JOIN ValidationReport
129+
ON ValidationReport.id = ValidationReportGtfsDataset.validation_report_id
130+
ORDER BY
131+
ValidationReportGtfsDataset.dataset_id,
132+
ValidationReport.validated_at DESC
133+
) AS LatestReports
134+
ON LatestReports.latest_validation_report_id = ValidationReportGtfsDataset.validation_report_id
135+
JOIN ValidationReport
136+
ON ValidationReport.id = ValidationReportGtfsDataset.validation_report_id
137+
) AS LatestDatasetValidationReportJoin ON Latest_dataset.id = ValidationReportGtfsDatasetId
138+
139+
-- External ids
140+
LEFT JOIN (
141+
SELECT
142+
feed_id,
143+
json_agg(json_build_object('external_id', associated_id, 'source', source)) AS external_ids
144+
FROM externalid
145+
GROUP BY feed_id
146+
) AS ExternalIdJoin ON ExternalIdJoin.feed_id = Feed.id
147+
148+
-- feed reference ids
149+
LEFT JOIN (
150+
SELECT
151+
gtfs_rt_feed_id,
152+
array_agg(FeedReferenceJoinInnerQuery.stable_id) AS feed_reference_ids
153+
FROM FeedReference
154+
LEFT JOIN Feed AS FeedReferenceJoinInnerQuery ON FeedReferenceJoinInnerQuery.id = FeedReference.gtfs_feed_id
155+
GROUP BY gtfs_rt_feed_id
156+
) AS FeedReferenceJoin ON FeedReferenceJoin.gtfs_rt_feed_id = Feed.id AND Feed.data_type = 'gtfs_rt'
157+
158+
-- Redirect ids
159+
LEFT JOIN (
160+
SELECT
161+
target_id,
162+
json_agg(json_build_object('target_id', target_id, 'comment', redirect_comment)) AS redirect_ids
163+
FROM RedirectingId
164+
GROUP BY target_id
165+
) AS RedirectingIdJoin ON RedirectingIdJoin.target_id = Feed.id
166+
167+
-- Feed locations
168+
LEFT JOIN (
169+
SELECT
170+
LocationFeed.feed_id,
171+
json_agg(json_build_object('country', country, 'country_code', country_code, 'subdivision_name',
172+
subdivision_name, 'municipality', municipality)) AS locations
173+
FROM Location
174+
LEFT JOIN LocationFeed ON LocationFeed.location_id = Location.id
175+
GROUP BY LocationFeed.feed_id
176+
) AS FeedLocationJoin ON FeedLocationJoin.feed_id = Feed.id
177+
178+
-- Entity types
179+
LEFT JOIN (
180+
SELECT
181+
feed_id,
182+
array_agg(entity_name) AS entities
183+
FROM EntityTypeFeed
184+
GROUP BY feed_id
185+
) AS EntityTypeFeedJoin ON EntityTypeFeedJoin.feed_id = Feed.id AND Feed.data_type = 'gtfs_rt'
186+
187+
-- OSM locations
188+
LEFT JOIN (
189+
WITH locations_per_group AS (
190+
SELECT
191+
fog.feed_id,
192+
olg.group_name,
193+
jsonb_agg(
194+
DISTINCT jsonb_build_object(
195+
'admin_level', gp.admin_level,
196+
'name', gp.name
197+
)
198+
) AS locations
199+
FROM FeedOsmLocationGroup fog
200+
JOIN OsmLocationGroup olg ON olg.group_id = fog.group_id
201+
JOIN OsmLocationGroupGeopolygon olgg ON olgg.group_id = olg.group_id
202+
JOIN Geopolygon gp ON gp.osm_id = olgg.osm_id
203+
GROUP BY fog.feed_id, olg.group_name
204+
)
205+
SELECT
206+
feed_id,
207+
jsonb_agg(
208+
jsonb_build_object(
209+
'group_name', group_name,
210+
'locations', locations
211+
)
212+
)::json AS osm_locations
213+
FROM locations_per_group
214+
GROUP BY feed_id
215+
) AS OsmLocationJoin ON OsmLocationJoin.feed_id = Feed.id
216+
217+
-- OSM location names
218+
LEFT JOIN (
219+
SELECT
220+
fog.feed_id,
221+
string_agg(DISTINCT gp.name, ' ') AS osm_location_names
222+
FROM FeedOsmLocationGroup fog
223+
JOIN OsmLocationGroup olg ON olg.group_id = fog.group_id
224+
JOIN OsmLocationGroupGeopolygon olgg ON olgg.group_id = olg.group_id
225+
JOIN Geopolygon gp ON gp.osm_id = olgg.osm_id
226+
WHERE gp.name IS NOT NULL
227+
GROUP BY fog.feed_id
228+
) AS OsmLocationNamesJoin ON OsmLocationNamesJoin.feed_id = Feed.id
229+
230+
-- GBFS versions
231+
LEFT JOIN (
232+
SELECT
233+
Feed.id AS feed_id,
234+
to_jsonb(array_agg(DISTINCT GbfsVersion.version ORDER BY GbfsVersion.version)) AS versions
235+
FROM Feed
236+
JOIN GbfsFeed ON GbfsFeed.id = Feed.id
237+
JOIN GbfsVersion ON GbfsVersion.feed_id = GbfsFeed.id
238+
WHERE Feed.data_type = 'gbfs'
239+
GROUP BY Feed.id
240+
) AS GbfsVersionsJoin ON GbfsVersionsJoin.feed_id = Feed.id;
241+
242+
243+
-- This index allows concurrent refresh on the materialized view avoiding table locks
244+
CREATE UNIQUE INDEX idx_unique_feed_id ON FeedSearch(feed_id);
245+
246+
-- Indices for feedsearch view optimization
247+
CREATE INDEX feedsearch_document_idx ON FeedSearch USING GIN(document);
248+
CREATE INDEX feedsearch_feed_stable_id ON FeedSearch(feed_stable_id);
249+
CREATE INDEX feedsearch_data_type ON FeedSearch(data_type);
250+
CREATE INDEX feedsearch_status ON FeedSearch(status);

0 commit comments

Comments
 (0)