Merge pull request #23 from stac-utils/pgtaptests

PgTAP Tests

Author: lossyrob

docker-compose.yml

@@ -24,6 +24,9 @@ services:
       - POSTGRES_USER=username
       - POSTGRES_PASSWORD=password
       - POSTGRES_DB=postgis
+      - PGUSER=username
+      - PGPASSWORD=password
+      - PGDATABASE=postgis
     ports:
       - "5432:5432"
     volumes:

scripts/test

@@ -16,14 +16,10 @@ Runs tests for the project.
 
 if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
     echo "Running database tests..."
-    # TODO: Fix these tests
-    # scripts/server --detach;
-    # docker-compose  \
-    #     run --rm \
-    #     -e PGUSER=username \
-    #     -e PGPASSWORD=password \
-    #     -e PGHOST=database \
-    #     database /opt/src/test/test.sh;
+    scripts/server --detach;
+    docker-compose  \
+        exec -T \
+        database /opt/src/test/test.sh;
 
     echo "Running pypgstac tests..."
     docker-compose  \

sql/002_collections.sql

@@ -47,24 +47,3 @@ CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$
 SELECT jsonb_agg(content) FROM collections;
 ;
 $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public;
-
-
-
-
-
-
-/* CREATE OR REPLACE FUNCTION collections_trigger_func()
-RETURNS TRIGGER AS $$
-BEGIN
-    IF pg_trigger_depth() = 1 THEN
-        PERFORM create_collection(NEW.content);
-        RETURN NULL;
-    END IF;
-    RETURN NEW;
-END;
-$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public;
-
-CREATE TRIGGER collections_trigger
-BEFORE INSERT ON collections
-FOR EACH ROW EXECUTE PROCEDURE collections_trigger_func();
- */

sql/003_items.sql

@@ -19,23 +19,6 @@ CREATE TABLE items_template (
 
 ALTER TABLE items_template ADD PRIMARY KEY (id);
 
-/*
-CREATE TABLE IF NOT EXISTS items_search (
-    id text NOT NULL,
-    geometry geometry NOT NULL,
-    properties jsonb,
-    collection_id text NOT NULL,
-    datetime timestamptz NOT NULL
-)
-PARTITION BY RANGE (datetime)
-;
-
-CREATE TABLE IF NOT EXISTS items_search_template (
-    LIKE items_search
-)
-;
-ALTER TABLE items_search_template ADD PRIMARY KEY (id);
-*/
 
 DELETE from partman.part_config WHERE parent_table = 'pgstac.items';
 SELECT partman.create_parent(
@@ -79,55 +62,6 @@ CREATE TYPE item AS (
 );
 
 
-/*
-Converts single feature into an items row
-*/
-
-/*
-CREATE OR REPLACE FUNCTION feature_to_item(value jsonb) RETURNS item AS $$
-    SELECT
-        value->>'id' as id,
-        CASE
-            WHEN value->>'geometry' IS NOT NULL THEN
-                ST_GeomFromGeoJSON(value->>'geometry')
-            WHEN value->>'bbox' IS NOT NULL THEN
-                ST_MakeEnvelope(
-                    (value->'bbox'->>0)::float,
-                    (value->'bbox'->>1)::float,
-                    (value->'bbox'->>2)::float,
-                    (value->'bbox'->>3)::float,
-                    4326
-                )
-            ELSE NULL
-        END as geometry,
-        properties_idx(value ->'properties') as properties,
-        value->>'collection' as collection_id,
-        (value->'properties'->>'datetime')::timestamptz as datetime
-;
-$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public;
-*/
-/*
-Takes a single feature, an array of features, or a feature collection
-and returns a set up individual items rows
-*/
-/*
-CREATE OR REPLACE FUNCTION features_to_items(value jsonb) RETURNS SETOF item AS $$
-    WITH features AS (
-        SELECT
-        jsonb_array_elements(
-            CASE
-                WHEN jsonb_typeof(value) = 'array' THEN value
-                WHEN value->>'type' = 'Feature' THEN '[]'::jsonb || value
-                WHEN value->>'type' = 'FeatureCollection' THEN value->'features'
-                ELSE NULL
-            END
-        ) as value
-    )
-    SELECT feature_to_item(value) FROM features
-    ;
-$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public;
-*/
-
 CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$
     SELECT content FROM items WHERE id=_id;
 $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public;
@@ -186,62 +120,6 @@ END LOOP;
 END;
 $$ LANGUAGE PLPGSQL;
 
-/* Trigger Function to cascade inserts/updates/deletes
-from items table to items_search table */
-/*
-ALTER TABLE items_search ADD CONSTRAINT items_search_fk
-FOREIGN KEY (id) REFERENCES items(id)
-ON DELETE CASCADE DEFERRABLE;
-
-CREATE OR REPLACE FUNCTION items_trigger_func()
-RETURNS TRIGGER AS $$
-DECLARE
-BEGIN
-    IF TG_OP = 'UPDATE' THEN
-    RAISE NOTICE 'DELETING % BEFORE UPDATE', OLD;
-        DELETE FROM items_search WHERE id = OLD.id AND datetime = (OLD.content->'properties'->>'datetime')::timestamptz;
-    END IF;
-
-    INSERT INTO items_search SELECT * FROM feature_to_item(NEW.content);
-    RETURN NEW;
-END;
-$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public;
-
-DROP TRIGGER IF EXISTS items_insert_trigger ON items;
-CREATE TRIGGER items_insert_trigger
-AFTER INSERT ON items
-FOR EACH ROW EXECUTE PROCEDURE items_trigger_func();
-
-DROP TRIGGER IF EXISTS items_update_trigger ON items;
-CREATE TRIGGER items_update_trigger
-AFTER UPDATE ON items
-FOR EACH ROW
-WHEN (NEW.content IS DISTINCT FROM OLD.content)
-EXECUTE PROCEDURE items_trigger_func();
-*/
-
-/* Trigger Function to cascade inserts/updates/deletes
-from items table to items_search table */
-/*
-CREATE OR REPLACE FUNCTION items_search_trigger_delete_func()
-RETURNS TRIGGER AS $$
-DECLARE
-BEGIN
-    RAISE NOTICE 'Deleting from items_search: % Depth: %', OLD, pg_trigger_depth();
-    IF pg_trigger_depth()<3 THEN
-        RAISE NOTICE 'DELETING WITH datetime';
-        DELETE FROM items_search WHERE id=OLD.id AND datetime=OLD.datetime;
-        RETURN NULL;
-    END IF;
-    RETURN OLD;
-END;
-$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public;
-
-DROP TRIGGER IF EXISTS items_search_delete_trigger ON items_search;
-CREATE TRIGGER items_search_delete_trigger
-BEFORE DELETE ON items_search
-FOR EACH ROW EXECUTE PROCEDURE items_search_trigger_delete_func();
-*/
 CREATE OR REPLACE FUNCTION backfill_partitions()
 RETURNS VOID AS $$
 DECLARE

sql/004_search.sql

@@ -1,30 +1,5 @@
 SET SEARCH_PATH TO pgstac, public;
 
-/*
-View to get a table of available items partitions
-with date ranges
-*/
-DROP VIEW IF EXISTS items_partitions;
-CREATE VIEW items_partitions AS
-WITH base AS
-(SELECT
-    c.oid::pg_catalog.regclass::text as partition,
-    pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint,
-    regexp_matches(
-        pg_catalog.pg_get_expr(c.relpartbound, c.oid),
-        E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)'
-    ) as t,
-    reltuples::bigint as est_cnt
-FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i
-WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass)
-SELECT partition, tstzrange(
-    t[1]::timestamptz,
-    t[2]::timestamptz
-), est_cnt
-FROM base
-WHERE est_cnt >0
-ORDER BY 2 desc;
-
 
 CREATE OR REPLACE FUNCTION items_by_partition(
     IN _where text DEFAULT 'TRUE',

test/pgtap.sql

@@ -6,6 +6,7 @@
 \pset format unaligned
 \pset tuples_only true
 \pset pager off
+\timing off
 
 -- Revert all changes on failure.
 \set ON_ERROR_ROLLBACK 1
@@ -17,15 +18,23 @@ CREATE EXTENSION IF NOT EXISTS pgtap;
 SET SEARCH_PATH TO pgstac, pgtap, public;
 
 -- Plan the tests.
-SELECT plan(50);
+SELECT plan(62);
 --SELECT * FROM no_plan();
 
 -- Run the tests.
 
 -- Core
+
+-- Check that schema exists
 SELECT has_schema('pgstac'::name);
+
+-- Check that PostGIS and PG_Partman extensions are installed and available on the path
 SELECT has_extension('postgis');
 SELECT has_extension('pg_partman');
+
+SELECT has_table('pgstac'::name, 'migrations'::name);
+
+
 SELECT has_function('pgstac'::name, 'textarr', ARRAY['jsonb']);
 SELECT results_eq(
     $$ SELECT textarr('["a","b","c"]'::jsonb) $$,
@@ -46,48 +55,67 @@ SELECT results_eq(
     $$ SELECT '{"a":1,"b":"b"}'::jsonb $$,
     'properties_idx returns slimmed lower case jsonb'
 );
+
+SELECT has_function('pgstac'::name, 'stac_geom', ARRAY['jsonb']);
+SELECT has_function('pgstac'::name, 'stac_datetime', ARRAY['jsonb']);
+SELECT has_function('pgstac'::name, 'jsonb_paths', ARRAY['jsonb']);
+SELECT has_function('pgstac'::name, 'jsonb_obj_paths', ARRAY['jsonb']);
+SELECT has_function('pgstac'::name, 'jsonb_val_paths', ARRAY['jsonb']);
+SELECT has_function('pgstac'::name, 'path_includes', ARRAY['text[]','text[]']);
+SELECT has_function('pgstac'::name, 'path_excludes', ARRAY['text[]','text[]']);
+SELECT has_function('pgstac'::name, 'jsonb_obj_paths_filtered', ARRAY['jsonb','text[]','text[]']);
+SELECT has_function('pgstac'::name, 'empty_arr', ARRAY['anyarray']);
+SELECT has_function('pgstac'::name, 'filter_jsonb', ARRAY['jsonb','text[]','text[]']);
+
+
 -- Collections
 SELECT has_table('pgstac'::name, 'collections'::name);
 SELECT col_is_pk('pgstac'::name, 'collections'::name, 'id', 'collections has primary key');
+
 SELECT has_function('pgstac'::name, 'create_collection', ARRAY['jsonb']);
+SELECT has_function('pgstac'::name, 'update_collection', ARRAY['jsonb']);
+SELECT has_function('pgstac'::name, 'upsert_collection', ARRAY['jsonb']);
 SELECT has_function('pgstac'::name, 'get_collection', ARRAY['text']);
+SELECT has_function('pgstac'::name, 'delete_collection', ARRAY['text']);
 SELECT has_function('pgstac'::name, 'all_collections', NULL);
-SELECT has_function('pgstac'::name, 'collection_bbox', ARRAY['text']);
-SELECT has_function('pgstac'::name, 'collection_temporal_extent', ARRAY['text']);
-SELECT has_function('pgstac'::name, 'update_collection_extents', NULL);
-
-SELECT has_function('pgstac'::name, 'collections_trigger_func', 'collections trigger function exists');
-SELECT has_trigger('pgstac'::name, 'collections', 'collections_trigger', 'trigger exists on collections table');
 
 
 
 
 
 -- Items
 SELECT has_table('pgstac'::name, 'items'::name);
-SELECT col_is_pk('pgstac'::name, 'items', 'id', NULL);
 
-SELECT has_table('pgstac'::name, 'items_search'::name);
---SELECT col_is_pk('pgstac'::name, 'items_search'::name, 'id', 'id should be primary key');
-SELECT is_indexed('pgstac'::name, 'items_search'::name, ARRAY['datetime','id']);
-SELECT is_indexed('pgstac'::name, 'items_search'::name, 'properties');
-SELECT is_indexed('pgstac'::name, 'items_search'::name, 'geometry');
-SELECT is_indexed('pgstac'::name, 'items_search'::name, 'collection_id');
+SELECT is_indexed('pgstac'::name, 'items'::name, ARRAY['datetime','id']);
+SELECT is_indexed('pgstac'::name, 'items'::name, 'properties');
+SELECT is_indexed('pgstac'::name, 'items'::name, 'geometry');
+SELECT is_indexed('pgstac'::name, 'items'::name, 'collection_id');
 
 SELECT has_type('pgstac'::name, 'item'::name);
-SELECT is_partitioned('pgstac'::name,'items_search'::name);
+SELECT is_partitioned('pgstac'::name,'items'::name);
 
 
-SELECT has_function('pgstac'::name, 'feature_to_item', ARRAY['jsonb']);
-SELECT has_function('pgstac'::name, 'features_to_items', ARRAY['jsonb']);
 SELECT has_function('pgstac'::name, 'get_item', ARRAY['text']);
 SELECT has_function('pgstac'::name, 'delete_item', ARRAY['text']);
 SELECT has_function('pgstac'::name, 'create_item', ARRAY['jsonb']);
+SELECT has_function('pgstac'::name, 'update_item', ARRAY['jsonb']);
+SELECT has_function('pgstac'::name, 'upsert_item', ARRAY['jsonb']);
+
 
-SELECT has_function('pgstac'::name, 'items_trigger_func', 'items trigger function exists');
-SELECT has_trigger('pgstac'::name, 'items'::name, 'items_trigger', 'items table has trigger');
 
-SELECT has_view('pgstac'::name, 'items_search_partitions'::name, 'items_search_partitions view exists');
+SELECT has_function('pgstac'::name, 'analyze_empty_partitions', NULL);
+SELECT has_function('pgstac'::name, 'backfill_partitions', NULL);
+SELECT has_function('pgstac'::name, 'items_trigger_stmt_func', NULL);
+
+
+SELECT has_view('pgstac'::name, 'all_items_partitions'::name, 'all_items_partitions view exists');
+SELECT has_view('pgstac'::name, 'items_partitions'::name, 'items_partitions view exists');
+
+-- tools to update collection extents based on extents in items
+SELECT has_function('pgstac'::name, 'collection_bbox', ARRAY['text']);
+SELECT has_function('pgstac'::name, 'collection_temporal_extent', ARRAY['text']);
+SELECT has_function('pgstac'::name, 'update_collection_extents', NULL);
+
 
 
 -- Search
@@ -107,7 +135,6 @@ SELECT has_function('pgstac'::name, 'filter_by_order', ARRAY['item','jsonb','tex
 SELECT has_function('pgstac'::name, 'search_dtrange', ARRAY['jsonb']);
 SELECT has_function('pgstac'::name, 'search', ARRAY['jsonb']);
 
-
 -- Finish the tests and clean up.
 SELECT * FROM finish();
-ROLLBACK;
+ROLLBACK;

test/test.sh

@@ -1,22 +1,13 @@
 #!/bin/bash
-tempid=$(cat /dev/urandom | tr -dc 'a-zA-Z' | fold -w 8 | head -n 1)
-tempdb="pgstac_tst_${tmpid}"
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-
-
-createdb $tempdb;
-export PGDATABASE=$tempdb
-
-cd $DIR/..
-psql -f pgstac.sql
-
-
-cat test/testdata/collections.ndjson | psql -c "copy pgstac.collections_staging FROM stdin"
-cat test/testdata/items.ndjson | psql -c "copy pgstac.items_staging FROM stdin"
-
-psql -X -f test/test.sql >/tmp/$tempdb.out
-
-cmp --silent /tmp/$tempdb.out test/test.out && echo '### SUCCESS: Files Tests Pass! ###' || echo '### WARNING: Tests did not pass! ###' && diff /tmp/$tempdb.out test/test.out
-
-dropdb $tempdb;
-rm /tmp/$tempdb.out
+TESTOUTPUT=$(psql -X -f $(dirname $0)/pgtap.sql)
+if [ $? != 0 ]; then
+    echo "Problem connecting to database to run tests."
+    exit 1
+fi
+echo "$TESTOUTPUT" | grep '^not'
+if [ $? == 0 ]; then
+    exit 1
+else
+    echo "All PGTap Tests Passed!"
+    exit 0
+fi