diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index 34faf19..050e9dd 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -6,6 +6,8 @@ on: paths: - ".github/workflows/test-eql.yml" - "sql/*.sql" + - "tests/**/*" + - "tasks/**/*" pull_request: branches: @@ -13,6 +15,8 @@ on: paths: - ".github/workflows/test-eql.yml" - "sql/*.sql" + - "tests/**/*" + - "tasks/**/*" workflow_dispatch: @@ -23,7 +27,7 @@ defaults: jobs: test: name: "Test EQL SQL components" - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest-m strategy: fail-fast: false @@ -31,21 +35,21 @@ jobs: postgres-version: [17, 16, 15, 14] env: - CS_DATABASE__PASSWORD: - CS_DATABASE__PORT: 5432 - CS_DATABASE__NAME: test + POSTGRES_VERSION: ${{ matrix.postgres-version }} steps: - uses: actions/checkout@v4 - - uses: extractions/setup-just@v1 - - - uses: ankane/setup-postgres@v1 + - uses: jdx/mise-action@v2 with: - postgres-version: ${{ matrix.postgres-version }} - database: ${{ env.CS_DATABASE__NAME }} + version: 2025.1.6 # [default: latest] mise version to install + install: true # [default: true] run `mise install` + cache: true # [default: true] cache mise using GitHub's cache - - name: Test EQL + - name: Setup database (Postgres ${{ matrix.postgres-version }}) run: | - just build test + mise run postgres:up postgres-${POSTGRES_VERSION} --extra-args "--detach --wait" + - name: Test EQL for Postgres ${{ matrix.postgres-version }} + run: | + mise run --output prefix test --postgres ${POSTGRES_VERSION} diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 185ff65..0000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -just 1.36.0 diff --git a/README.md b/README.md index c566068..476241d 100644 --- a/README.md +++ b/README.md @@ -392,3 +392,49 @@ To cut a [release](https://github.com/cipherstash/encrypt-query-language/release 1. Click `Publish release`. This will trigger the [Release EQL](https://github.com/cipherstash/encrypt-query-language/actions/workflows/release-eql.yml) workflow, which will build and attach artifacts to [the release](https://github.com/cipherstash/encrypt-query-language/releases/). + +## Testing + +There are tests for EQL for PostgreSQL versions 14–17. + +They easiest way to run them is in [GitHub Actions](https://github.com/cipherstash/encrypt-query-language/actions/workflows/test-eql.yml). + +### Running tests locally + +> [!IMPORTANT] +> **Before you run the tests** you need to have this software installed: +> - [mise](https://mise.jdx.dev/) β€” see the [installing mise](#installing-mise) instructions +> - [Docker](https://www.docker.com/) β€” see Docker's [documentation for installing](https://docs.docker.com/get-started/get-docker/) + +To run tests locally: + +``` shell +# Clone the repo +git clone https://github.com/cipherstash/encrypt-query-language +cd encrypt-query-language + +# Install dependencies +mise trust --yes + +# Start a postgres instance +mise run postgres:up postgres-17 --extra-args "--detach --wait" + +# Run the tests +mise run test --postgres 17 + +# Stop and remove all containers and networks +mise run postgres:down +``` + +You can run the same tasks for Postgres 14, 15, 16, and 17. + +The configuration for the Postgres containers in `tests/docker-compose.yml`. + +Limitations: + +- **Volumes for Postgres containers are not persistent.** + If you need to look at data in the container, uncomment a volume in + `tests/docker-compose.yml` +- **You can't run multiple Postgres containers at the same time.** + All the containers bind to the same port (`7543`). If you want to run + multiple containers at the same time, you'll have to change the ports. diff --git a/justfile b/justfile deleted file mode 100644 index 7f65462..0000000 --- a/justfile +++ /dev/null @@ -1,84 +0,0 @@ -set dotenv-load -set positional-arguments - - -test: - #!/usr/bin/env bash - set -euxo pipefail - cd "{{justfile_directory()}}" - - just build - just reset - - connection_url=postgresql://${CS_DATABASE__USERNAME:-$USER}:@localhost:$CS_DATABASE__PORT/$CS_DATABASE__NAME - - # tests - PGPASSWORD=$CS_DATABASE__PASSWORD psql $connection_url -f tests/core.sql - PGPASSWORD=$CS_DATABASE__PASSWORD psql $connection_url -f tests/config.sql - PGPASSWORD=$CS_DATABASE__PASSWORD psql $connection_url -f tests/encryptindex.sql - - # Uninstall - PGPASSWORD=$CS_DATABASE__PASSWORD psql $connection_url -f release/cipherstash-encrypt-uninstall.sql - - - -build: - #!/usr/bin/env bash - set -euxo pipefail - cd "{{justfile_directory()}}" - - mkdir -p release - - rm -f release/cipherstash-encrypt-uninstall.sql - rm -f release/cipherstash-encrypt.sql - - # Collect all the drops - # In reverse order (tac) so that we drop the constraints before the tables - grep -h -E '^(DROP)' sql/0*-*.sql | tac > release/cipherstash-encrypt-tmp-drop-install.sql - # types are always last - cat sql/666-drop_types.sql >> release/cipherstash-encrypt-tmp-drop-install.sql - - - # Build cipherstash-encrypt.sql - # drop everything first - cat release/cipherstash-encrypt-tmp-drop-install.sql > release/cipherstash-encrypt.sql - # cat the rest of the sql files - cat sql/0*-*.sql >> release/cipherstash-encrypt.sql - - # Collect all the drops - # In reverse order (tac) so that we drop the constraints before the tables - grep -h -E '^(DROP|ALTER DOMAIN [^ ]+ DROP CONSTRAINT)' sql/0*-*.sql | tac > release/cipherstash-encrypt-tmp-drop-uninstall.sql - # types are always last - cat sql/666-drop_types.sql >> release/cipherstash-encrypt-tmp-drop-uninstall.sql - - - # Build cipherstash-encrypt-uninstall.sql - # prepend the drops to the main sql file - cat release/cipherstash-encrypt-tmp-drop-uninstall.sql >> release/cipherstash-encrypt-uninstall.sql - # uninstall renames configuration table - cat sql/666-rename_configuration_table.sql >> release/cipherstash-encrypt-uninstall.sql - - # remove the drop file - rm release/cipherstash-encrypt-tmp-drop-install.sql - rm release/cipherstash-encrypt-tmp-drop-uninstall.sql - - -reset: - #!/usr/bin/env bash - set -euxo pipefail - cd "{{justfile_directory()}}" - - PGPASSWORD=$CS_DATABASE__PASSWORD dropdb --force --if-exists --username ${CS_DATABASE__USERNAME:-$USER} --port $CS_DATABASE__PORT $CS_DATABASE__NAME - PGPASSWORD=$CS_DATABASE__PASSWORD createdb --username ${CS_DATABASE__USERNAME:-$USER} --port $CS_DATABASE__PORT $CS_DATABASE__NAME - - connection_url=postgresql://${CS_DATABASE__USERNAME:-$USER}:@localhost:$CS_DATABASE__PORT/$CS_DATABASE__NAME - - PGPASSWORD=$CS_DATABASE__PASSWORD psql $connection_url -f release/cipherstash-encrypt.sql - - -psql: - psql postgresql://$CS_USERNAME:$CS_PASSWORD@localhost:$CS_PORT/$CS_DATABASE__NAME - - -psql_direct: - psql --user $CS_DATABASE__USERNAME --dbname $CS_DATABASE__NAME --port $CS_DATABASE__PORT diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..ccfd9ef --- /dev/null +++ b/mise.toml @@ -0,0 +1,20 @@ +[settings] +# Config for test environments +# Can be invoked with: mise --env tcp run +# trusted_config_paths = [ +# "./tests/mise.toml", +# "./tests/mise.tcp.toml", +# "./tests/mise.tls.toml", +# ] +[task_config] +includes = [ + "tasks", + "tasks/postgres.toml" +] + +[env] +POSTGRES_DB = "cipherstash" +POSTGRES_USER = "cipherstash" +POSTGRES_PASSWORD = "password" +POSTGRES_HOST = "localhost" +POSTGRES_PORT = "7432" diff --git a/sql/010-core-domain-types.sql b/sql/010-core-domain-types.sql new file mode 100644 index 0000000..8d9ba78 --- /dev/null +++ b/sql/010-core-domain-types.sql @@ -0,0 +1,159 @@ +DROP DOMAIN IF EXISTS cs_match_index_v1; +CREATE DOMAIN cs_match_index_v1 AS smallint[]; + +DROP DOMAIN IF EXISTS cs_unique_index_v1; +CREATE DOMAIN cs_unique_index_v1 AS text; + + +-- cs_encrypted_v1 is a column type and cannot be dropped if in use +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cs_encrypted_v1') THEN + CREATE DOMAIN cs_encrypted_v1 AS JSONB; + END IF; +END +$$; + + +-- Should include a kind field +DROP FUNCTION IF EXISTS _cs_encrypted_check_k(jsonb); +CREATE FUNCTION _cs_encrypted_check_k(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val->>'k' = ANY('{ct, sv}')) THEN + RETURN true; + END IF; + RAISE 'Invalid kind (%) in Encrypted column. Kind should be one of {ct, sv}', val; + END; +$$ LANGUAGE plpgsql; + + +-- +-- CT payload should include a c field +-- +DROP FUNCTION IF EXISTS _cs_encrypted_check_k_ct(jsonb); +CREATE FUNCTION _cs_encrypted_check_k_ct(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val->>'k' = 'ct') THEN + IF (val ? 'c') THEN + RETURN true; + END IF; + RAISE 'Encrypted column kind (k) of "ct" missing data field (c): %', val; + END IF; + RETURN true; + END; +$$ LANGUAGE plpgsql; + + +-- +-- SV payload should include an sv field +-- +DROP FUNCTION IF EXISTS _cs_encrypted_check_k_sv(jsonb); +CREATE FUNCTION _cs_encrypted_check_k_sv(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val->>'k' = 'sv') THEN + IF (val ? 'sv') THEN + RETURN true; + END IF; + RAISE 'Encrypted column kind (k) of "sv" missing data field (sv): %', val; + END IF; + RETURN true; + END; +$$ LANGUAGE plpgsql; + + +-- Plaintext field should never be present in an encrypted column +DROP FUNCTION IF EXISTS _cs_encrypted_check_p(jsonb); +CREATE FUNCTION _cs_encrypted_check_p(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF NOT val ? 'p' THEN + RETURN true; + END IF; + RAISE 'Encrypted column includes plaintext (p) field: %', val; + END; +$$ LANGUAGE plpgsql; + +-- Should include an ident field +DROP FUNCTION IF EXISTS _cs_encrypted_check_i(jsonb); +CREATE FUNCTION _cs_encrypted_check_i(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF val ? 'i' THEN + RETURN true; + END IF; + RAISE 'Encrypted column missing ident (i) field: %', val; + END; +$$ LANGUAGE plpgsql; + +-- Query field should never be present in an encrypted column +DROP FUNCTION IF EXISTS _cs_encrypted_check_q(jsonb); +CREATE FUNCTION _cs_encrypted_check_q(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF val ? 'q' THEN + RAISE 'Encrypted column includes query (q) field: %', val; + END IF; + RETURN true; + END; +$$ LANGUAGE plpgsql; + +-- Ident field should include table and column +DROP FUNCTION IF EXISTS _cs_encrypted_check_i_ct(jsonb); +CREATE FUNCTION _cs_encrypted_check_i_ct(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val->'i' ?& array['t', 'c']) THEN + RETURN true; + END IF; + RAISE 'Encrypted column ident (i) missing table (t) or column (c) fields: %', val; + END; +$$ LANGUAGE plpgsql; + +-- Should include a version field +DROP FUNCTION IF EXISTS _cs_encrypted_check_v(jsonb); +CREATE FUNCTION _cs_encrypted_check_v(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val ? 'v') THEN + RETURN true; + END IF; + RAISE 'Encrypted column missing version (v) field: %', val; + END; +$$ LANGUAGE plpgsql; + + +DROP FUNCTION IF EXISTS cs_check_encrypted_v1(val jsonb); + +CREATE FUNCTION cs_check_encrypted_v1(val jsonb) + RETURNS BOOLEAN +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN ( + _cs_encrypted_check_v(val) AND + _cs_encrypted_check_i(val) AND + _cs_encrypted_check_k(val) AND + _cs_encrypted_check_k_ct(val) AND + _cs_encrypted_check_k_sv(val) AND + _cs_encrypted_check_q(val) AND + _cs_encrypted_check_p(val) + ); +END; + +ALTER DOMAIN cs_encrypted_v1 DROP CONSTRAINT IF EXISTS cs_encrypted_v1_check; + +ALTER DOMAIN cs_encrypted_v1 + ADD CONSTRAINT cs_encrypted_v1_check CHECK ( + cs_check_encrypted_v1(VALUE) +); + diff --git a/sql/010-core.sql b/sql/011-core-functions.sql similarity index 54% rename from sql/010-core.sql rename to sql/011-core-functions.sql index ecf9b6c..003e699 100644 --- a/sql/010-core.sql +++ b/sql/011-core-functions.sql @@ -1,164 +1,3 @@ -DROP DOMAIN IF EXISTS cs_match_index_v1; -CREATE DOMAIN cs_match_index_v1 AS smallint[]; - -DROP DOMAIN IF EXISTS cs_unique_index_v1; -CREATE DOMAIN cs_unique_index_v1 AS text; - - --- cs_encrypted_v1 is a column type and cannot be dropped if in use -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cs_encrypted_v1') THEN - CREATE DOMAIN cs_encrypted_v1 AS JSONB; - END IF; -END -$$; - - --- Should include a kind field -DROP FUNCTION IF EXISTS _cs_encrypted_check_k(jsonb); -CREATE FUNCTION _cs_encrypted_check_k(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val->>'k' = ANY('{ct, sv}')) THEN - RETURN true; - END IF; - RAISE 'Invalid kind (%) in Encrypted column. Kind should be one of {ct, sv}', val; - END; -$$ LANGUAGE plpgsql; - - --- --- CT payload should include a c field --- -DROP FUNCTION IF EXISTS _cs_encrypted_check_k_ct(jsonb); -CREATE FUNCTION _cs_encrypted_check_k_ct(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val->>'k' = 'ct') THEN - IF (val ? 'c') THEN - RETURN true; - END IF; - RAISE 'Encrypted column kind (k) of "ct" missing data field (c): %', val; - END IF; - RETURN true; - END; -$$ LANGUAGE plpgsql; - - --- --- SV payload should include an sv field --- -DROP FUNCTION IF EXISTS _cs_encrypted_check_k_sv(jsonb); -CREATE FUNCTION _cs_encrypted_check_k_sv(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val->>'k' = 'sv') THEN - IF (val ? 'sv') THEN - RETURN true; - END IF; - RAISE 'Encrypted column kind (k) of "sv" missing data field (sv): %', val; - END IF; - RETURN true; - END; -$$ LANGUAGE plpgsql; - - --- Plaintext field should never be present in an encrypted column -DROP FUNCTION IF EXISTS _cs_encrypted_check_p(jsonb); -CREATE FUNCTION _cs_encrypted_check_p(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF NOT val ? 'p' THEN - RETURN true; - END IF; - RAISE 'Encrypted column includes plaintext (p) field: %', val; - END; -$$ LANGUAGE plpgsql; - --- Should include an ident field -DROP FUNCTION IF EXISTS _cs_encrypted_check_i(jsonb); -CREATE FUNCTION _cs_encrypted_check_i(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF val ? 'i' THEN - RETURN true; - END IF; - RAISE 'Encrypted column missing ident (i) field: %', val; - END; -$$ LANGUAGE plpgsql; - --- Query field should never be present in an encrypted column -DROP FUNCTION IF EXISTS _cs_encrypted_check_q(jsonb); -CREATE FUNCTION _cs_encrypted_check_q(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF val ? 'q' THEN - RAISE 'Encrypted column includes query (q) field: %', val; - END IF; - RETURN true; - END; -$$ LANGUAGE plpgsql; - --- Ident field should include table and column -DROP FUNCTION IF EXISTS _cs_encrypted_check_i_ct(jsonb); -CREATE FUNCTION _cs_encrypted_check_i_ct(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val->'i' ?& array['t', 'c']) THEN - RETURN true; - END IF; - RAISE 'Encrypted column ident (i) missing table (t) or column (c) fields: %', val; - END; -$$ LANGUAGE plpgsql; - --- Should include a version field -DROP FUNCTION IF EXISTS _cs_encrypted_check_v(jsonb); -CREATE FUNCTION _cs_encrypted_check_v(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val ? 'v') THEN - RETURN true; - END IF; - RAISE 'Encrypted column missing version (v) field: %', val; - END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS cs_check_encrypted_v1(val jsonb); - -CREATE FUNCTION cs_check_encrypted_v1(val jsonb) - RETURNS BOOLEAN -LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN ( - _cs_encrypted_check_v(val) AND - _cs_encrypted_check_i(val) AND - _cs_encrypted_check_k(val) AND - _cs_encrypted_check_k_ct(val) AND - _cs_encrypted_check_k_sv(val) AND - _cs_encrypted_check_q(val) AND - _cs_encrypted_check_p(val) - ); -END; - - -ALTER DOMAIN cs_encrypted_v1 DROP CONSTRAINT IF EXISTS cs_encrypted_v1_check; - -ALTER DOMAIN cs_encrypted_v1 - ADD CONSTRAINT cs_encrypted_v1_check CHECK ( - cs_check_encrypted_v1(VALUE) -); - - DROP FUNCTION IF EXISTS cs_ciphertext_v1_v0_0(val jsonb); CREATE FUNCTION cs_ciphertext_v1_v0_0(val jsonb) @@ -256,7 +95,7 @@ BEGIN ATOMIC END; -DROP FUNCTION IF EXISTS cs_unique_v1(val jsonb); +DROP FUNCTION IF EXISTS cs_unique_v1(val jsonb); CREATE FUNCTION cs_unique_v1(val jsonb) RETURNS cs_unique_index_v1 diff --git a/sql/015-operators-unique.sql b/sql/015-operators-unique.sql new file mode 100644 index 0000000..1111692 --- /dev/null +++ b/sql/015-operators-unique.sql @@ -0,0 +1,210 @@ +-- Operators for unique comparisons of cs_encrypted_v1 types +-- +-- Support for the following comparisons: +-- +-- cs_encrypted_v1 = cs_encrypted_v1 +-- cs_encrypted_v1 <> cs_encrypted_v1 +-- cs_encrypted_v1 = jsonb +-- cs_encrypted_v1 <> jsonb +-- cs_encrypted_v1 = text +-- cs_encrypted_v1 <> text +-- + +DROP OPERATOR IF EXISTS = (cs_encrypted_v1, cs_encrypted_v1); +DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a cs_encrypted_v1, b cs_encrypted_v1); + +CREATE FUNCTION cs_encrypted_eq_v1(a cs_encrypted_v1, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT cs_unique_v1(a) = cs_unique_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR = ( + PROCEDURE="cs_encrypted_eq_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=cs_encrypted_v1, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + +DROP OPERATOR IF EXISTS = (cs_encrypted_v1, jsonb); +DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a cs_encrypted_v1, b jsonb); + +CREATE FUNCTION cs_encrypted_eq_v1(a cs_encrypted_v1, b jsonb) +RETURNS boolean AS $$ + SELECT cs_unique_v1(a) = cs_unique_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR = ( + PROCEDURE="cs_encrypted_eq_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=jsonb, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + +DROP OPERATOR IF EXISTS = (jsonb, cs_encrypted_v1); +DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a jsonb, b cs_encrypted_v1); + +CREATE FUNCTION cs_encrypted_eq_v1(a jsonb, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT cs_unique_v1(a) = cs_unique_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR = ( + PROCEDURE="cs_encrypted_eq_v1", + LEFTARG=jsonb, + RIGHTARG=cs_encrypted_v1, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +DROP OPERATOR IF EXISTS = (cs_encrypted_v1, cs_unique_index_v1); +DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a cs_encrypted_v1, b cs_unique_index_v1); + +CREATE FUNCTION cs_encrypted_eq_v1(a cs_encrypted_v1, b cs_unique_index_v1) +RETURNS boolean AS $$ + SELECT cs_unique_v1(a) = b; +$$ LANGUAGE SQL; + +CREATE OPERATOR = ( + PROCEDURE="cs_encrypted_eq_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=cs_unique_index_v1, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + +DROP OPERATOR IF EXISTS = (cs_unique_index_v1, cs_encrypted_v1); +DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a cs_unique_index_v1, b cs_encrypted_v1); + +CREATE FUNCTION cs_encrypted_eq_v1(a cs_unique_index_v1, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT a = cs_unique_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR =( + PROCEDURE="cs_encrypted_eq_v1", + LEFTARG=cs_unique_index_v1, + RIGHTARG=cs_encrypted_v1, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + + +--- ------------------------------------------------------------ + +DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, cs_encrypted_v1); +DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a cs_encrypted_v1, b cs_encrypted_v1); + +CREATE FUNCTION cs_encrypted_neq_v1(a cs_encrypted_v1, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT cs_unique_v1(a) <> cs_unique_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR <> ( + PROCEDURE="cs_encrypted_neq_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=cs_encrypted_v1, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, jsonb); +DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a cs_encrypted_v1, b jsonb); + +CREATE FUNCTION cs_encrypted_neq_v1(a cs_encrypted_v1, b jsonb) +RETURNS boolean AS $$ + SELECT cs_unique_v1(a) <> cs_unique_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR <> ( + PROCEDURE="cs_encrypted_neq_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=jsonb, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +DROP OPERATOR IF EXISTS <> (jsonb, cs_encrypted_v1); +DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a jsonb, b cs_encrypted_v1); + +CREATE FUNCTION cs_encrypted_neq_v1(a jsonb, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT cs_unique_v1(a) <> cs_unique_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR <> ( + PROCEDURE="cs_encrypted_neq_v1", + LEFTARG=jsonb, + RIGHTARG=cs_encrypted_v1, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, cs_unique_index_v1); +DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a cs_encrypted_v1, b cs_unique_index_v1); + +CREATE FUNCTION cs_encrypted_neq_v1(a cs_encrypted_v1, b cs_unique_index_v1) +RETURNS boolean AS $$ + SELECT cs_unique_v1(a) <> b; +$$ LANGUAGE SQL; + +CREATE OPERATOR <> ( + PROCEDURE="cs_encrypted_neq_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=cs_unique_index_v1, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +DROP OPERATOR IF EXISTS <> (cs_unique_index_v1, cs_encrypted_v1); +DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a cs_unique_index_v1, b cs_encrypted_v1); + +CREATE FUNCTION cs_encrypted_neq_v1(a cs_unique_index_v1, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT a <> cs_unique_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR <> ( + PROCEDURE="cs_encrypted_neq_v1", + LEFTARG=cs_unique_index_v1, + RIGHTARG=cs_encrypted_v1, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); diff --git a/sql/016-operators-match.sql b/sql/016-operators-match.sql new file mode 100644 index 0000000..717aade --- /dev/null +++ b/sql/016-operators-match.sql @@ -0,0 +1,105 @@ +-- Operators for match comparisons of cs_encrypted_v1 types +-- +-- Support for the following comparisons: +-- +-- cs_encrypted_v1 @> cs_encrypted_v1 +-- cs_encrypted_v1 @> jsonb +-- cs_encrypted_v1 @> cs_match_index_v1 +-- + +DROP OPERATOR IF EXISTS @> (cs_encrypted_v1, cs_encrypted_v1); +DROP FUNCTION IF EXISTS cs_encrypted_contains_v1(a cs_encrypted_v1, b cs_encrypted_v1); + +CREATE FUNCTION cs_encrypted_contains_v1(a cs_encrypted_v1, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT cs_match_v1(a) @> cs_match_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR @>( + PROCEDURE="cs_encrypted_contains_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=cs_encrypted_v1, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +DROP OPERATOR IF EXISTS @> (cs_encrypted_v1, cs_match_index_v1); +DROP FUNCTION IF EXISTS cs_encrypted_contains_v1(a cs_encrypted_v1, b cs_match_index_v1); + +CREATE FUNCTION cs_encrypted_contains_v1(a cs_encrypted_v1, b cs_match_index_v1) +RETURNS boolean AS $$ + SELECT cs_match_v1(a) @> b; +$$ LANGUAGE SQL; + +CREATE OPERATOR @>( + PROCEDURE="cs_encrypted_contains_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=cs_match_index_v1, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +-- ------------------------------------------------------------------------------------ + + +DROP OPERATOR IF EXISTS <@ (cs_encrypted_v1, cs_encrypted_v1); +DROP FUNCTION IF EXISTS cs_encrypted_contained_v1(a cs_encrypted_v1, b cs_encrypted_v1); + +CREATE FUNCTION cs_encrypted_contained_v1(a cs_encrypted_v1, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT cs_match_v1(a) <@ cs_match_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR <@( + PROCEDURE="cs_encrypted_contained_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=cs_encrypted_v1, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +DROP OPERATOR IF EXISTS <@ (cs_match_index_v1, cs_encrypted_v1); +DROP FUNCTION IF EXISTS cs_encrypted_contained_v1(a cs_match_index_v1, b cs_encrypted_v1); + +CREATE FUNCTION cs_encrypted_contained_v1(a cs_match_index_v1, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT a <@ cs_match_v1(b); +$$ LANGUAGE SQL; + +CREATE OPERATOR <@ ( + PROCEDURE="cs_encrypted_contained_v1", + LEFTARG=cs_match_index_v1, + RIGHTARG=cs_encrypted_v1, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +DROP OPERATOR IF EXISTS <@ (cs_encrypted_v1, cs_match_index_v1); +DROP FUNCTION IF EXISTS cs_encrypted_contained_v1(a cs_encrypted_v1, b cs_match_index_v1); + +CREATE FUNCTION cs_encrypted_contained_v1(a cs_match_index_v1, b cs_encrypted_v1) +RETURNS boolean AS $$ + SELECT cs_match_v1(a) <@ b; +$$ LANGUAGE SQL; + +CREATE OPERATOR <@ ( + PROCEDURE="cs_encrypted_contained_v1", + LEFTARG=cs_encrypted_v1, + RIGHTARG=cs_match_index_v1, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); diff --git a/tasks/build.sh b/tasks/build.sh new file mode 100755 index 0000000..62a9ded --- /dev/null +++ b/tasks/build.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +#MISE description="Build SQL into single release file" +#MISE sources=["sql/*.sql"] +#MISE outputs=["release/cipherstash-encrypt.sql","release/cipherstash-encrypt-uninstall.sql"] + +#!/bin/bash + +set -euxo pipefail + +mkdir -p release + +rm -f release/cipherstash-encrypt-uninstall.sql +rm -f release/cipherstash-encrypt.sql + +# Collect all the drops +# In reverse order (tac) so that we drop the constraints before the tables +grep -h -E '^(DROP)' sql/0*-*.sql | tac > release/cipherstash-encrypt-tmp-drop-install.sql +# types are always last +cat sql/666-drop_types.sql >> release/cipherstash-encrypt-tmp-drop-install.sql + + +# Build cipherstash-encrypt.sql +# drop everything first +cat release/cipherstash-encrypt-tmp-drop-install.sql > release/cipherstash-encrypt.sql +# cat the rest of the sql files +cat sql/0*-*.sql >> release/cipherstash-encrypt.sql + +# Collect all the drops +# In reverse order (tac) so that we drop the constraints before the tables +grep -h -E '^(DROP|ALTER DOMAIN [^ ]+ DROP CONSTRAINT)' sql/0*-*.sql | tac > release/cipherstash-encrypt-tmp-drop-uninstall.sql +# types are always last +cat sql/666-drop_types.sql >> release/cipherstash-encrypt-tmp-drop-uninstall.sql + + +# Build cipherstash-encrypt-uninstall.sql +# prepend the drops to the main sql file +cat release/cipherstash-encrypt-tmp-drop-uninstall.sql >> release/cipherstash-encrypt-uninstall.sql +# uninstall renames configuration table +cat sql/666-rename_configuration_table.sql >> release/cipherstash-encrypt-uninstall.sql + +# remove the drop file +rm release/cipherstash-encrypt-tmp-drop-install.sql +rm release/cipherstash-encrypt-tmp-drop-uninstall.sql diff --git a/tasks/postgres.toml b/tasks/postgres.toml new file mode 100644 index 0000000..5df6d89 --- /dev/null +++ b/tasks/postgres.toml @@ -0,0 +1,12 @@ +["postgres:down"] +description = "Tear down Postgres containers" +dir = "{{config_root}}/tests" +run = "docker compose down" + +["postgres:up"] +description = "Run Postgres instances with docker compose" +dir = "{{config_root}}/tests" +run = """ +{% set default_service = "postgres-" ~ get_env(name="POSTGRES_VERSION",default="17") %} +echo docker compose up {{arg(name="service",default=default_service)}} {{option(name="extra-args",default="")}} | bash +""" diff --git a/tasks/reset.sh b/tasks/reset.sh new file mode 100755 index 0000000..e128801 --- /dev/null +++ b/tasks/reset.sh @@ -0,0 +1,30 @@ +#!/bin/bash +#MISE description="Uninstall and install EQL to local postgres" +#USAGE flag "--postgres " help="Run tests for specified Postgres version" default="17" { +#USAGE choices "14" "15" "16" "17" +#USAGE } + +set -euo pipefail + +fail_if_postgres_not_running () { + containers=$(docker ps --filter "name=^${container_name}$" --quiet) + if [ -z "${containers}" ]; then + echo "error: Docker container for PostgreSQL is not running" + echo "error: Try running 'mise run postgres:up ${container_name}' to start the container" + exit 65 + fi +} + +POSTGRES_VERSION=${usage_postgres} + +connection_url=postgresql://${POSTGRES_USER:-$USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} +container_name=postgres-${POSTGRES_VERSION} + +# Setup +fail_if_postgres_not_running + +# Uninstall +cat release/cipherstash-encrypt-uninstall.sql | docker exec -i ${container_name} psql ${connection_url} -f- + +# Install +cat release/cipherstash-encrypt.sql | docker exec -i ${container_name} psql ${connection_url} -f- diff --git a/tasks/test.sh b/tasks/test.sh new file mode 100755 index 0000000..0db783c --- /dev/null +++ b/tasks/test.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +#MISE description="Build, reset and run tests" +#USAGE flag "--postgres " help="Run tests for specified Postgres version" default="17" { +#USAGE choices "14" "15" "16" "17" +#USAGE } + +#!/bin/bash + +set -euo pipefail + +POSTGRES_VERSION=${usage_postgres} + +connection_url=postgresql://${POSTGRES_USER:-$USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} +container_name=postgres-${POSTGRES_VERSION} + +fail_if_postgres_not_running () { + containers=$(docker ps --filter "name=^${container_name}$" --quiet) + if [ -z "${containers}" ]; then + echo "error: Docker container for PostgreSQL is not running" + echo "error: Try running 'mise run postgres:up ${container_name}' to start the container" + exit 65 + fi +} + +run_test () { + echo + echo '###############################################' + echo "# ${1}" + echo '###############################################' + echo + cat $1 | docker exec -i ${container_name} psql $connection_url -f- +} + +# setup +fail_if_postgres_not_running +mise run build +mise run reset --postgres ${POSTGRES_VERSION} + +# tests +run_test tests/core.sql +run_test tests/core-functions.sql +run_test tests/config.sql +run_test tests/encryptindex.sql +run_test tests/operators.sql + +echo +echo '###############################################' +echo "# βœ…ALL TESTS PASSED " +echo '###############################################' +echo diff --git a/tests/config.sql b/tests/config.sql index 8274935..e165c76 100644 --- a/tests/config.sql +++ b/tests/config.sql @@ -6,7 +6,7 @@ -- -- Helper function for assertions -- -DROP FUNCTION IF EXISTS _index_exists(text, text, text); +DROP FUNCTION IF EXISTS _index_exists(text, text, text, text); CREATE FUNCTION _index_exists(table_name text, column_name text, index_name text, state text DEFAULT 'pending') RETURNS boolean LANGUAGE sql STRICT PARALLEL SAFE diff --git a/tests/core.sql b/tests/core.sql index 6349a4d..ec42e73 100644 --- a/tests/core.sql +++ b/tests/core.sql @@ -12,7 +12,7 @@ CREATE TABLE users TRUNCATE TABLE users; --- no version + INSERT INTO users (name_encrypted) VALUES ( '{ "v": 1, diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..13c28d4 --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,52 @@ +services: + postgres: &postgres + container_name: postgres + image: postgres + ports: + - 7432:7432 + environment: + - PGPORT=${POSTGRES_PORT} + - PGUSER=${POSTGRES_USER} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + networks: + - postgres + deploy: + resources: + limits: + cpus: "${CPU_LIMIT:-2}" + memory: 2048mb + restart: always + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 1s + timeout: 5s + retries: 10 + + postgres-17: + <<: *postgres + image: postgres:17 + container_name: postgres-17 + #volumes: # uncomment if you need to inspect the container contents + #- ./pg/data-17:/var/lib/postgresql/data + + postgres-16: + <<: *postgres + image: postgres:16 + container_name: postgres-16 + + postgres-15: + <<: *postgres + image: postgres:15 + container_name: postgres-15 + + postgres-14: + <<: *postgres + image: postgres:14 + container_name: postgres-14 + +networks: + postgres: + driver: bridge + diff --git a/tests/operators.sql b/tests/operators.sql new file mode 100644 index 0000000..45bc51f --- /dev/null +++ b/tests/operators.sql @@ -0,0 +1,185 @@ +\set ON_ERROR_STOP on + + +-- Create a table with a plaintext column +DROP TABLE IF EXISTS users; +CREATE TABLE users +( + id bigint GENERATED ALWAYS AS IDENTITY, + name_encrypted cs_encrypted_v1, + PRIMARY KEY(id) +); + +TRUNCATE TABLE users; + +INSERT INTO users (name_encrypted) VALUES ( + '{ + "v": 1, + "k": "ct", + "c": "ciphertext", + "i": { + "t": "users", + "c": "name" + }, + "m": [1, 2, 3], + "u": "unique-text", + "o": ["a"] + }'::jsonb +); + + + +-- MATCH @> OPERATORS +DO $$ + BEGIN + -- SANITY CHECK FOR UNIQUE payloads + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_match_v1(name_encrypted) @> cs_match_v1('{"m":[1,2]}'))); + + -- cs_encrypted_v1 = jsonb + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted @> '{"m":[1,2]}'::jsonb)); + + -- cs_encrypted_v1 = text + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted @> ARRAY[1,2]::smallint[])); + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted @> ARRAY[1,2]::cs_match_index_v1)); + + -- cs_encrypted_v1 = cs_encrypted_v1 + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted @> '{ + "v": 1, + "k": "ct", + "c": "ciphertext", + "i": { + "t": "users", + "c": "name" + }, + "m": [1, 2] + }'::cs_encrypted_v1)); + + END; +$$ LANGUAGE plpgsql; + + + +-- MATCH <@ OPERATORS +DO $$ + BEGIN + -- SANITY CHECK FOR UNIQUE payloads + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_match_v1('{"m":[1,2]}') <@ cs_match_v1(name_encrypted))); + + -- cs_encrypted_v1 = jsonb + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{"m":[1,2]}'::jsonb <@ name_encrypted)); + + -- cs_encrypted_v1 = text + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE ARRAY[1,2]::smallint[] <@ name_encrypted)); + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE ARRAY[1,2]::cs_match_index_v1 <@ name_encrypted)); + + -- cs_encrypted_v1 = cs_encrypted_v1 + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{ + "v": 1, + "k": "ct", + "c": "ciphertext", + "i": { + "t": "users", + "c": "name" + }, + "m": [1, 2] + }'::cs_encrypted_v1 <@ name_encrypted)); + + END; +$$ LANGUAGE plpgsql; + + + +-- UNIQUE eq = OPERATORS +DO $$ + BEGIN + -- SANITY CHECK FOR UNIQUE payloads + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_unique_v1(name_encrypted) = cs_unique_v1('{"u":"unique-text"}'))); + + ASSERT (SELECT EXISTS ( + SELECT id FROM users WHERE name_encrypted = '{ + "v": 1, + "k": "ct", + "c": "ciphertext", + "i": { + "t": "users", + "c": "name" + }, + "u": "unique-text" + }'::jsonb + )); + + -- cs_encrypted_v1 = jsonb + ASSERT (SELECT EXISTS ( + SELECT id FROM users WHERE name_encrypted = '{"u": "unique-text"}'::jsonb + )); + + -- jsonb = cs_encrypted_v1 + ASSERT (SELECT EXISTS ( + SELECT id FROM users WHERE '{"u": "unique-text"}'::jsonb = name_encrypted + )); + + -- cs_encrypted_v1 = text + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted = 'unique-text'::text)); + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted = 'unique-text'::cs_unique_index_v1)); + + -- text = cs_encrypted_v1 + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE 'unique-text'::text = name_encrypted)); + + -- cs_encrypted_v1 = cs_encrypted_v1 + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted = '{ + "v": 1, + "k": "ct", + "c": "ciphertext", + "i": { + "t": "users", + "c": "name" + }, + "u": "unique-text" + }'::cs_encrypted_v1)); + + END; +$$ LANGUAGE plpgsql; + +-- UNIQUE inequality <> OPERATORS +DO $$ + BEGIN + -- SANITY CHECK FOR UNIQUE payloads + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_unique_v1(name_encrypted) != cs_unique_v1('{"u":"random-text"}'))); + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_unique_v1(name_encrypted) <> cs_unique_v1('{"u":"random-text"}'))); + + -- cs_encrypted_v1 = jsonb + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted != '{"u":"random-text"}'::jsonb)); + ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <> '{"u":"random-text"}'::jsonb)); + + -- -- cs_encrypted_v1 = text + -- ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted != 'random-text'::text)); + -- ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <> 'random-text'::text)); + + -- -- cs_encrypted_v1 = cs_encrypted_v1 + -- ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted != '{ + -- "v": 1, + -- "k": "ct", + -- "c": "ciphertext", + -- "i": { + -- "t": "users", + -- "c": "name" + -- }, + -- "u": "random-text" + -- }'::cs_encrypted_v1)); + -- ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <> '{ + -- "v": 1, + -- "k": "ct", + -- "c": "ciphertext", + -- "i": { + -- "t": "users", + -- "c": "name" + -- }, + -- "u": "random-text" + -- }'::cs_encrypted_v1)); + + + + END; +$$ LANGUAGE plpgsql; + +