Skip to content

Commit c7fb58c

Browse files
bitnerbitner
committed
create migrations for gt/gte fix
1 parent 375273b commit c7fb58c

File tree

3 files changed

+375
-0
lines changed

3 files changed

+375
-0
lines changed

‎pypgstac/pypgstac/migrations/pgstac.0.2.4--0.2.5.sql‎ renamed to ‎pypgstac/pypgstac/migrations/pgstac.0.2.4-0.2.5.sql‎

File renamed without changes.

‎pypgstac/pypgstac/migrations/pgstac.0.2.4-0.2.6.sql‎

Lines changed: 313 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,313 @@
1+
SET SEARCH_PATH TO pgstac, public;
2+
BEGIN;
3+
4+
CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$
5+
DECLARE
6+
ret text := '';
7+
op text;
8+
jp text;
9+
att_parts RECORD;
10+
val_str text;
11+
prop_path text;
12+
BEGIN
13+
val_str := lower(jsonb_build_object('a',val)->>'a');
14+
RAISE NOTICE 'val_str %', val_str;
15+
16+
att_parts := split_stac_path(att);
17+
prop_path := replace(att_parts.dotpath, 'properties.', '');
18+
19+
op := CASE _op
20+
WHEN 'eq' THEN '='
21+
WHEN 'gte' THEN '>='
22+
WHEN 'gt' THEN '>'
23+
WHEN 'lte' THEN '<='
24+
WHEN 'lt' THEN '<'
25+
WHEN 'ne' THEN '!='
26+
WHEN 'neq' THEN '!='
27+
WHEN 'startsWith' THEN 'LIKE'
28+
WHEN 'endsWith' THEN 'LIKE'
29+
WHEN 'contains' THEN 'LIKE'
30+
ELSE _op
31+
END;
32+
33+
val_str := CASE _op
34+
WHEN 'startsWith' THEN concat(val_str, '%')
35+
WHEN 'endsWith' THEN concat('%', val_str)
36+
WHEN 'contains' THEN concat('%',val_str,'%')
37+
ELSE val_str
38+
END;
39+
40+
41+
RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.');
42+
IF
43+
op = '='
44+
AND att_parts.col = 'properties'
45+
--AND count_by_delim(att_parts.dotpath,'\.') = 2
46+
THEN
47+
-- use jsonpath query to leverage index for eqaulity tests on single level deep properties
48+
jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb));
49+
raise notice 'jp: %', jp;
50+
ret := format($q$ properties @? %L $q$, jp);
51+
ELSIF jsonb_typeof(val) = 'number' THEN
52+
ret := format('properties ? %L AND (%s)::numeric %s %s', prop_path, att_parts.jspathtext, op, val);
53+
ELSE
54+
ret := format('properties ? %L AND %s %s %L', prop_path ,att_parts.jspathtext, op, val_str);
55+
END IF;
56+
RAISE NOTICE 'Op Query: %', ret;
57+
58+
return ret;
59+
END;
60+
$$ LANGUAGE PLPGSQL;
61+
62+
CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$
63+
SELECT CASE jsonb_array_length(_bbox)
64+
WHEN 4 THEN
65+
ST_SetSRID(ST_MakeEnvelope(
66+
(_bbox->>0)::float,
67+
(_bbox->>1)::float,
68+
(_bbox->>2)::float,
69+
(_bbox->>3)::float
70+
),4326)
71+
WHEN 6 THEN
72+
ST_SetSRID(ST_3DMakeBox(
73+
ST_MakePoint(
74+
(_bbox->>0)::float,
75+
(_bbox->>1)::float,
76+
(_bbox->>2)::float
77+
),
78+
ST_MakePoint(
79+
(_bbox->>3)::float,
80+
(_bbox->>4)::float,
81+
(_bbox->>5)::float
82+
)
83+
),4326)
84+
ELSE null END;
85+
;
86+
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
87+
88+
89+
CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$
90+
DECLARE
91+
qstart timestamptz := clock_timestamp();
92+
_sort text := '';
93+
_rsort text := '';
94+
_limit int := 10;
95+
_geom geometry;
96+
qa text[];
97+
pq text[];
98+
query text;
99+
pq_prop record;
100+
pq_op record;
101+
prev_id text := NULL;
102+
next_id text := NULL;
103+
whereq text := 'TRUE';
104+
links jsonb := '[]'::jsonb;
105+
token text;
106+
tok_val text;
107+
tok_q text := 'TRUE';
108+
tok_sort text;
109+
first_id text;
110+
first_dt timestamptz;
111+
last_id text;
112+
sort text;
113+
rsort text;
114+
dt text[];
115+
dqa text[];
116+
dq text;
117+
mq_where text;
118+
startdt timestamptz;
119+
enddt timestamptz;
120+
item items%ROWTYPE;
121+
counter int := 0;
122+
batchcount int;
123+
month timestamptz;
124+
m record;
125+
_dtrange tstzrange := tstzrange('-infinity','infinity');
126+
_dtsort text;
127+
_token_dtrange tstzrange := tstzrange('-infinity','infinity');
128+
_token_record items%ROWTYPE;
129+
is_prev boolean := false;
130+
includes text[];
131+
excludes text[];
132+
BEGIN
133+
-- Create table from sort query of items to sort
134+
CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby');
135+
136+
-- Get the datetime sort direction, necessary for efficient cycling through partitions
137+
SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime';
138+
RAISE NOTICE '_dtsort: %',_dtsort;
139+
140+
SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s;
141+
SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s;
142+
tok_sort := _sort;
143+
144+
145+
-- Get datetime from query as a tstzrange
146+
IF _search ? 'datetime' THEN
147+
_dtrange := search_dtrange(_search->'datetime');
148+
_token_dtrange := _dtrange;
149+
END IF;
150+
151+
-- Get the paging token
152+
IF _search ? 'token' THEN
153+
token := _search->>'token';
154+
tok_val := substr(token,6);
155+
IF starts_with(token, 'prev:') THEN
156+
is_prev := true;
157+
END IF;
158+
SELECT INTO _token_record * FROM items WHERE id=tok_val;
159+
IF
160+
(is_prev AND _dtsort = 'DESC')
161+
OR
162+
(not is_prev AND _dtsort = 'ASC')
163+
THEN
164+
_token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity');
165+
ELSIF
166+
_dtsort IS NOT NULL
167+
THEN
168+
_token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime);
169+
END IF;
170+
IF is_prev THEN
171+
tok_q := filter_by_order(tok_val, _search->'sortby', 'first');
172+
_sort := _rsort;
173+
ELSIF starts_with(token, 'next:') THEN
174+
tok_q := filter_by_order(tok_val, _search->'sortby', 'last');
175+
END IF;
176+
END IF;
177+
RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart);
178+
RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange;
179+
180+
IF _search ? 'ids' THEN
181+
RAISE NOTICE 'searching solely based on ids... %',_search;
182+
qa := array_append(qa, in_array_q('id', _search->'ids'));
183+
ELSE
184+
IF _search ? 'intersects' THEN
185+
_geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326);
186+
ELSIF _search ? 'bbox' THEN
187+
_geom := bbox_geom(_search->'bbox');
188+
END IF;
189+
190+
IF _geom IS NOT NULL THEN
191+
qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom));
192+
END IF;
193+
194+
IF _search ? 'collections' THEN
195+
qa := array_append(qa, in_array_q('collection_id', _search->'collections'));
196+
END IF;
197+
198+
IF _search ? 'query' THEN
199+
qa := array_cat(qa,
200+
stac_query(_search->'query')
201+
);
202+
END IF;
203+
END IF;
204+
205+
IF _search ? 'limit' THEN
206+
_limit := (_search->>'limit')::int;
207+
END IF;
208+
209+
IF _search ? 'fields' THEN
210+
IF _search->'fields' ? 'exclude' THEN
211+
excludes=textarr(_search->'fields'->'exclude');
212+
END IF;
213+
IF _search->'fields' ? 'include' THEN
214+
includes=textarr(_search->'fields'->'include');
215+
IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN
216+
includes = includes || '{id}';
217+
END IF;
218+
END IF;
219+
RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes;
220+
END IF;
221+
222+
whereq := COALESCE(array_to_string(qa,' AND '),' TRUE ');
223+
dq := COALESCE(array_to_string(dqa,' AND '),' TRUE ');
224+
RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart);
225+
226+
CREATE TEMP TABLE results_page ON COMMIT DROP AS
227+
SELECT * FROM items_by_partition(
228+
concat(whereq, ' AND ', tok_q),
229+
_token_dtrange,
230+
_sort,
231+
_limit + 1
232+
);
233+
RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart);
234+
235+
RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart);
236+
237+
IF is_prev THEN
238+
SELECT INTO last_id, first_id, counter
239+
first_value(id) OVER (),
240+
last_value(id) OVER (),
241+
count(*) OVER ()
242+
FROM results_page;
243+
ELSE
244+
SELECT INTO first_id, last_id, counter
245+
first_value(id) OVER (),
246+
last_value(id) OVER (),
247+
count(*) OVER ()
248+
FROM results_page;
249+
END IF;
250+
RAISE NOTICE 'firstid: %, lastid %', first_id, last_id;
251+
RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart);
252+
253+
254+
255+
256+
IF counter > _limit THEN
257+
next_id := last_id;
258+
RAISE NOTICE 'next_id: %', next_id;
259+
ELSE
260+
RAISE NOTICE 'No more next';
261+
END IF;
262+
263+
IF tok_q = 'TRUE' THEN
264+
RAISE NOTICE 'Not a paging query, no previous item';
265+
ELSE
266+
RAISE NOTICE 'Getting previous item id';
267+
RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart);
268+
SELECT INTO _token_record * FROM items WHERE id=first_id;
269+
IF
270+
_dtsort = 'DESC'
271+
THEN
272+
_token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity');
273+
ELSE
274+
_token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime);
275+
END IF;
276+
RAISE NOTICE '% %', _token_dtrange, _dtrange;
277+
SELECT id INTO prev_id FROM items_by_partition(
278+
concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')),
279+
_token_dtrange,
280+
_rsort,
281+
1
282+
);
283+
RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart);
284+
285+
RAISE NOTICE 'prev_id: %', prev_id;
286+
END IF;
287+
288+
289+
RETURN QUERY
290+
WITH features AS (
291+
SELECT filter_jsonb(content, includes, excludes) as content
292+
FROM results_page LIMIT _limit
293+
),
294+
j AS (SELECT jsonb_agg(content) as feature_arr FROM features)
295+
SELECT jsonb_build_object(
296+
'type', 'FeatureCollection',
297+
'features', coalesce (
298+
CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END
299+
,'[]'::jsonb),
300+
'links', links,
301+
'timeStamp', now(),
302+
'next', next_id,
303+
'prev', prev_id
304+
)
305+
FROM j
306+
;
307+
308+
309+
END;
310+
$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public;
311+
INSERT INTO migrations (version) VALUES ('0.2.6');
312+
313+
COMMIT;

‎pypgstac/pypgstac/migrations/pgstac.0.2.5-0.2.6.sql‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
SET SEARCH_PATH TO pgstac, public;
2+
BEGIN;
3+
CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$
4+
DECLARE
5+
ret text := '';
6+
op text;
7+
jp text;
8+
att_parts RECORD;
9+
val_str text;
10+
prop_path text;
11+
BEGIN
12+
val_str := lower(jsonb_build_object('a',val)->>'a');
13+
RAISE NOTICE 'val_str %', val_str;
14+
15+
att_parts := split_stac_path(att);
16+
prop_path := replace(att_parts.dotpath, 'properties.', '');
17+
18+
op := CASE _op
19+
WHEN 'eq' THEN '='
20+
WHEN 'gte' THEN '>='
21+
WHEN 'gt' THEN '>'
22+
WHEN 'lte' THEN '<='
23+
WHEN 'lt' THEN '<'
24+
WHEN 'ne' THEN '!='
25+
WHEN 'neq' THEN '!='
26+
WHEN 'startsWith' THEN 'LIKE'
27+
WHEN 'endsWith' THEN 'LIKE'
28+
WHEN 'contains' THEN 'LIKE'
29+
ELSE _op
30+
END;
31+
32+
val_str := CASE _op
33+
WHEN 'startsWith' THEN concat(val_str, '%')
34+
WHEN 'endsWith' THEN concat('%', val_str)
35+
WHEN 'contains' THEN concat('%',val_str,'%')
36+
ELSE val_str
37+
END;
38+
39+
40+
RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.');
41+
IF
42+
op = '='
43+
AND att_parts.col = 'properties'
44+
--AND count_by_delim(att_parts.dotpath,'\.') = 2
45+
THEN
46+
-- use jsonpath query to leverage index for eqaulity tests on single level deep properties
47+
jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb));
48+
raise notice 'jp: %', jp;
49+
ret := format($q$ properties @? %L $q$, jp);
50+
ELSIF jsonb_typeof(val) = 'number' THEN
51+
ret := format('properties ? %L AND (%s)::numeric %s %s', prop_path, att_parts.jspathtext, op, val);
52+
ELSE
53+
ret := format('properties ? %L AND %s %s %L', prop_path ,att_parts.jspathtext, op, val_str);
54+
END IF;
55+
RAISE NOTICE 'Op Query: %', ret;
56+
57+
return ret;
58+
END;
59+
$$ LANGUAGE PLPGSQL;
60+
INSERT INTO migrations (version) VALUES ('0.2.6');
61+
62+
COMMIT;

0 commit comments

Comments
 (0)