diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index 9d6e2a32..1d34d5ac 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -5,18 +5,17 @@ on: - main paths: - ".github/workflows/test-eql.yml" - - "src/*.sql" - - "sql/*.sql" + - "src/**/*.sql" + - "sql/**/*.sql" - "tests/**/*" - "tasks/**/*" pull_request: - branches: - - main + # run on all pull requests paths: - ".github/workflows/test-eql.yml" - - "src/*.sql" - - "sql/*.sql" + - "src/**/*.sql" + - "sql/**/*.sql" - "tests/**/*" - "tasks/**/*" diff --git a/SUPABASE.md b/SUPABASE.md new file mode 100644 index 00000000..21cbde4a --- /dev/null +++ b/SUPABASE.md @@ -0,0 +1,74 @@ +# Supabase + + +## No operators, no problems + +Supabase [does not currently support](https://github.com/supabase/supautils/issues/72) custom operators. +The EQL operator functions can be used in this situation. + +In EQL, PostgreSQL operators are an alias for a function, so the implementation and behaviour remains the same across operators and functions. + +| Operator | Function | Example | +| -------- | -------------------------------------------------- | ----------------------------------------------------------------- | +| `=` | `eql_v1.eq(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.eq(encrypted_email, $1)`
| +| `<>` | `eql_v1.neq(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.neq(encrypted_email, $1)`
| +| `<` | `eql_v1.lt(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.lt(encrypted_email, $1)`
| +| `<=` | `eql_v1.lte(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.lte(encrypted_email, $1)`
| +| `>` | `eql_v1.gt(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.gt(encrypted_email, $1)`
| +| `>=` | `eql_v1.gte(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.gte(encrypted_email, $1)`
| +| `~~` | `eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.like(encrypted_email, $1)`
| +| `~~*` | `eql_v1.ilike(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.ilike(encrypted_email, $1)`
| +| `LIKE` | `eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.like(encrypted_email, $1)`
| +| `ILIKE` | `eql_v1.ilike(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.ilike(encrypted_email, $1)`
| + +### Example SQL Statements + +#### Equality `=` + + +**Operator** +```sql +SELECT * FROM users WHERE encrypted_email = $1 +``` + +**Function** +```sql +SELECT * FROM users WHERE eql_v1.eq(encrypted_email, $1) +``` + + +#### Like & ILIKE `~~, ~~*` + + +**Operator** +```sql +SELECT * FROM users WHERE encrypted_email LIKE $1 +``` + +**Function** +```sql +SELECT * FROM users WHERE eql_v1.like(encrypted_email, $1) +``` + +#### Case Sensitivity + +The EQL `eql_v1.like` and `eql_v1.ilike` functions are equivalent. + +The behaviour of EQL's encrypted `LIKE` operators is slightly different to the behaviour of PostgreSQL's `LIKE` operator. +In EQL, the `LIKE` operator can be used on `match` indexes. +Case sensitivity is determined by the [index term configuration](./docs/reference/INDEX.md#options-for-match-indexes-opts) of `match` indexes. +A `match` index term can be configured to enable case sensitive searches with token filters (for example, `downcase` and `upcase`). +The data is encrypted based on the index term configuration. +The `LIKE` operation is always the same, even if the data is tokenised differently. +The different operators are kept to preserve the semantics of SQL statements in client applications. + +### `ORDER BY` + +Ordering requires wrapping the ordered column in the `eql_v1.order_by` function, like this: + +```sql +SELECT * FROM users ORDER BY eql_v1.order_by(encrypted_created_at) DESC +``` + +PostgreSQL uses operators when handling `ORDER BY` operations. The `eql_v1.order_by` function behaves in + diff --git a/src/encryptindex/functions_test.sql b/src/encryptindex/functions_test.sql index 9dd7cc78..5e89d196 100644 --- a/src/encryptindex/functions_test.sql +++ b/src/encryptindex/functions_test.sql @@ -8,7 +8,7 @@ TRUNCATE TABLE eql_v1_configuration; -- Create a table with a plaintext column --- DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS users; CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY, @@ -63,7 +63,7 @@ $$ LANGUAGE plpgsql; TRUNCATE TABLE eql_v1_configuration; -- Create a table with multiple plaintext columns --- DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS users; CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY, @@ -119,7 +119,7 @@ $$ LANGUAGE plpgsql; -- The schema should be validated first. -- Users table does not exist, so should fail. -- ----------------------------------------------- --- DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS users; TRUNCATE TABLE eql_v1_configuration; @@ -148,7 +148,7 @@ $$ LANGUAGE plpgsql; -- -- Schema validation is skipped -- ----------------------------------------------- --- DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS users; TRUNCATE TABLE eql_v1_configuration; DO $$ @@ -194,7 +194,7 @@ INSERT INTO eql_v1_configuration (state, data) VALUES ( ); -- Create a table with plaintext and encrypted columns --- DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS users; CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY, @@ -244,7 +244,7 @@ INSERT INTO eql_v1_configuration (state, data) VALUES ( ); -- Create a table with plaintext and jsonb column --- DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS users; CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY, @@ -295,7 +295,7 @@ INSERT INTO eql_v1_configuration (state, data) VALUES ( -- Create a table with multiple plaintext columns --- DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS users; CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY, diff --git a/src/operators/order_by.sql b/src/operators/order_by.sql new file mode 100644 index 00000000..bdad66a1 --- /dev/null +++ b/src/operators/order_by.sql @@ -0,0 +1,72 @@ +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql +-- REQUIRE: src/ore/operators.sql +-- REQUIRE: src/ore_cllw_u64_8/types.sql +-- REQUIRE: src/ore_cllw_u64_8/functions.sql +-- REQUIRE: src/ore_cllw_u64_8/operators.sql + +-- order_by function for ordering when operators are not available. +-- +-- There are multiple index terms that provide equality comparisons +-- - ore_cllw_u64_8 +-- - ore_cllw_var_8 +-- - ore_64_8_v1 +-- +-- We check these index terms in this order and use the first one that exists for both parameters +-- +-- + +-- DROP FUNCTION IF EXISTS eql_v1.order_by(a eql_v1_encrypted, b eql_v1_encrypted); + +CREATE FUNCTION eql_v1.order_by(a eql_v1_encrypted) + RETURNS eql_v1.ore_64_8_v1 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + BEGIN + RETURN eql_v1.ore_64_8_v1(a); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v1.log('No ore_64_8_v1 index'); + END; + + RETURN false; + END; +$$ LANGUAGE plpgsql; + +-- TODO: make this work +-- fails with jsonb format issue, which I think is due to the type casting +-- +CREATE FUNCTION eql_v1.order_by_any(a anyelement) + RETURNS anyelement + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + e eql_v1_encrypted; + result ALIAS FOR $0; + BEGIN + + e := a::eql_v1_encrypted; + + BEGIN + result := eql_v1.ore_cllw_u64_8(e); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v1.log('No ore_cllw_u64_8 index'); + END; + + BEGIN + result := eql_v1.ore_cllw_var_8(e); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v1.log('No ore_cllw_u64_8 index'); + END; + + BEGIN + result := eql_v1.ore_64_8_v1(e); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v1.log('No ore_64_8_v1 index'); + END; + + RETURN result; + END; +$$ LANGUAGE plpgsql; + diff --git a/src/operators/order_by_test.sql b/src/operators/order_by_test.sql new file mode 100644 index 00000000..865891d7 --- /dev/null +++ b/src/operators/order_by_test.sql @@ -0,0 +1,28 @@ +\set ON_ERROR_STOP on + +-- +-- ORE - ORDER BY ore_64_8_v1(eql_v1_encrypted) +-- +DO $$ +DECLARE + e eql_v1_encrypted; + ore_term eql_v1_encrypted; + BEGIN + SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + + PERFORM assert_count( + 'ORDER BY eql_v1.order_by(e) DESC', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.order_by(e) DESC', ore_term), + 41); + + PERFORM assert_result( + 'ORDER BY eql_v1.order_by(e) DESC returns correct record', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.order_by(e) DESC LIMIT 1', ore_term), + '41'); + + PERFORM assert_result( + 'ORDER BY eql_v1.order_by(e) ASC', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.order_by(e) ASC LIMIT 1', ore_term), + '1'); + END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/src/operators/~~.sql b/src/operators/~~.sql index 59b4e204..f3fbf447 100644 --- a/src/operators/~~.sql +++ b/src/operators/~~.sql @@ -15,12 +15,23 @@ -- DROP FUNCTION IF EXISTS eql_v1.match(a eql_v1_encrypted, b eql_v1_encrypted); -CREATE FUNCTION eql_v1.match(a eql_v1_encrypted, b eql_v1_encrypted) +CREATE FUNCTION eql_v1.like(a eql_v1_encrypted, b eql_v1_encrypted) RETURNS boolean AS $$ SELECT eql_v1.match(a) @> eql_v1.match(b); $$ LANGUAGE SQL; +-- +-- Case sensitivity depends on the index term configuration +-- Function preserves the SQL semantics +-- +CREATE FUNCTION eql_v1.ilike(a eql_v1_encrypted, b eql_v1_encrypted) +RETURNS boolean AS $$ + SELECT eql_v1.match(a) @> eql_v1.match(b); +$$ LANGUAGE SQL; + + + -- DROP OPERATOR BEFORE FUNCTION -- DROP OPERATOR IF EXISTS ~~ (eql_v1_encrypted, eql_v1_encrypted); -- DROP OPERATOR IF EXISTS ~~* (eql_v1_encrypted, eql_v1_encrypted); @@ -31,7 +42,7 @@ CREATE FUNCTION eql_v1."~~"(a eql_v1_encrypted, b eql_v1_encrypted) RETURNS boolean AS $$ BEGIN - RETURN eql_v1.match(a, b); + RETURN eql_v1.like(a, b); END; $$ LANGUAGE plpgsql; @@ -65,7 +76,7 @@ CREATE FUNCTION eql_v1."~~"(a eql_v1_encrypted, b jsonb) RETURNS boolean AS $$ BEGIN - RETURN eql_v1.match(a, b::eql_v1_encrypted); + RETURN eql_v1.like(a, b::eql_v1_encrypted); END; $$ LANGUAGE plpgsql; @@ -100,7 +111,7 @@ CREATE FUNCTION eql_v1."~~"(a jsonb, b eql_v1_encrypted) RETURNS boolean AS $$ BEGIN - RETURN eql_v1.match(a::eql_v1_encrypted, b); + RETURN eql_v1.like(a::eql_v1_encrypted, b); END; $$ LANGUAGE plpgsql; diff --git a/src/operators/~~_test.sql b/src/operators/~~_test.sql index 48f1a35b..6043d71c 100644 --- a/src/operators/~~_test.sql +++ b/src/operators/~~_test.sql @@ -87,8 +87,8 @@ DECLARE e := create_encrypted_json(i, 'm'); PERFORM assert_result( - format('eql_v1.match(eql_v1_encrypted, eql_v1_encrypted)', i), - format('SELECT e FROM encrypted WHERE eql_v1.match(e, %L);', e)); + format('eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)', i), + format('SELECT e FROM encrypted WHERE eql_v1.like(e, %L);', e)); end loop; @@ -96,8 +96,8 @@ DECLARE e := create_encrypted_json('m')::jsonb || '{"m": [10, 11]}'; PERFORM assert_result( - 'eql_v1.match(eql_v1_encrypted, eql_v1_encrypted)', - format('SELECT e FROM encrypted WHERE eql_v1.match(e, %L);', e)); + 'eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v1.like(e, %L);', e)); END; $$ LANGUAGE plpgsql; diff --git a/src/ore/functions_test.sql b/src/ore/functions_test.sql index 2c7554c3..6a9f9588 100644 --- a/src/ore/functions_test.sql +++ b/src/ore/functions_test.sql @@ -11,4 +11,31 @@ DO $$ 'SELECT eql_v1.ore_64_8_v1(''{}''::jsonb)'); END; +$$ LANGUAGE plpgsql; + +-- +-- ORE - ORDER BY ore_64_8_v1(eql_v1_encrypted) +-- +DO $$ +DECLARE + e eql_v1_encrypted; + ore_term eql_v1_encrypted; + BEGIN + SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + + PERFORM assert_count( + 'ORDER BY eql_v1.ore_64_8_v1(e) DESC', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.ore_64_8_v1(e) DESC', ore_term), + 41); + + PERFORM assert_result( + 'ORDER BY eql_v1.ore_64_8_v1(e) DESC returns correct record', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.ore_64_8_v1(e) DESC LIMIT 1', ore_term), + '41'); + + PERFORM assert_result( + 'ORDER BY eql_v1.ore_64_8_v1(e) ASC', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.ore_64_8_v1(e) ASC LIMIT 1', ore_term), + '1'); + END; $$ LANGUAGE plpgsql; \ No newline at end of file