diff --git a/README.md b/README.md index 86d6761..7993ed3 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,8 @@ SELECT eql_v2.add_column('users', 'encrypted_email'); After modifying configurations, activate them by running: ```sql -SELECT eql_v2.encrypt(); -SELECT eql_v2.activate(); +SELECT eql_v2.migrate_config(); +SELECT eql_v2.activate_config(); ``` **Important:** These functions must be run after any modifications to the configuration. @@ -219,8 +219,8 @@ SELECT eql_v2.add_search_config( After adding an index, you have to activate the configuration: ```sql -SELECT eql_v2.encrypt(); -SELECT eql_v2.activate(); +SELECT eql_v2.migrate_config(); +SELECT eql_v2.activate_config(); ``` ## Searching data with EQL diff --git a/src/blake3/functions.sql b/src/blake3/functions.sql index e4d24c8..b88db5d 100644 --- a/src/blake3/functions.sql +++ b/src/blake3/functions.sql @@ -10,8 +10,11 @@ CREATE FUNCTION eql_v2.blake3(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; - IF NOT (val ? 'b3') NULL THEN + IF NOT (val ? 'b3') THEN RAISE 'Expected a blake3 index (b3) value in json: %', val; END IF; @@ -34,3 +37,24 @@ AS $$ RETURN (SELECT eql_v2.blake3(val.data)); END; $$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.has_blake3(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN val ? 'b3'; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.has_blake3(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.has_blake3(val.data); + END; +$$ LANGUAGE plpgsql; + diff --git a/src/bloom_filter/functions.sql b/src/bloom_filter/functions.sql index 8d729c6..69e51dc 100644 --- a/src/bloom_filter/functions.sql +++ b/src/bloom_filter/functions.sql @@ -8,9 +8,14 @@ CREATE FUNCTION eql_v2.bloom_filter(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + IF val ? 'bf' THEN RETURN ARRAY(SELECT jsonb_array_elements(val->'bf'))::eql_v2.bloom_filter; END IF; + RAISE 'Expected a match index (bf) value in json: %', val; END; $$ LANGUAGE plpgsql; @@ -26,3 +31,23 @@ AS $$ RETURN (SELECT eql_v2.bloom_filter(val.data)); END; $$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.has_bloom_filter(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN val ? 'bf'; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.has_bloom_filter(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.has_bloom_filter(val.data); + END; +$$ LANGUAGE plpgsql; diff --git a/src/config/functions.sql b/src/config/functions.sql index 617960f..491ec76 100644 --- a/src/config/functions.sql +++ b/src/config/functions.sql @@ -144,7 +144,7 @@ $$ LANGUAGE plpgsql; -- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt. -- -CREATE FUNCTION eql_v2.encrypt() +CREATE FUNCTION eql_v2.migrate_config() RETURNS boolean AS $$ BEGIN @@ -168,7 +168,7 @@ $$ LANGUAGE plpgsql; -CREATE FUNCTION eql_v2.activate() +CREATE FUNCTION eql_v2.activate_config() RETURNS boolean AS $$ BEGIN diff --git a/src/encryptindex/functions_test.sql b/src/encryptindex/functions_test.sql index 6626678..18a77f6 100644 --- a/src/encryptindex/functions_test.sql +++ b/src/encryptindex/functions_test.sql @@ -155,7 +155,7 @@ CREATE TABLE users DO $$ BEGIN PERFORM eql_v2.add_search_config('users', 'name', 'match'); - PERFORM eql_v2.encrypt(); + PERFORM eql_v2.migrate_config(); ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active')); ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting')); @@ -205,7 +205,7 @@ CREATE TABLE users DO $$ BEGIN PERFORM eql_v2.add_search_config('users', 'name', 'match'); - PERFORM eql_v2.encrypt(); + PERFORM eql_v2.migrate_config(); ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active')); ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting')); @@ -256,8 +256,8 @@ DO $$ BEGIN PERFORM eql_v2.add_search_config('users', 'name', 'match'); - PERFORM eql_v2.encrypt(); -- need to encrypt first - PERFORM eql_v2.activate(); + PERFORM eql_v2.migrate_config(); -- need to encrypt first + PERFORM eql_v2.activate_config(); ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active')); ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'inactive')); diff --git a/src/hmac_256/functions.sql b/src/hmac_256/functions.sql index 5ca187d..4cf0a94 100644 --- a/src/hmac_256/functions.sql +++ b/src/hmac_256/functions.sql @@ -8,6 +8,10 @@ CREATE FUNCTION eql_v2.hmac_256(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + IF val ? 'hm' THEN RETURN val->>'hm'; END IF; @@ -16,6 +20,27 @@ AS $$ $$ LANGUAGE plpgsql; +CREATE FUNCTION eql_v2.has_hmac_256(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN val ? 'hm'; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.has_hmac_256(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.has_hmac_256(val.data); + END; +$$ LANGUAGE plpgsql; + + + -- extracts hmac_256 index from an encrypted column CREATE FUNCTION eql_v2.hmac_256(val eql_v2_encrypted) diff --git a/src/hmac_256/functions_test.sql b/src/hmac_256/functions_test.sql index f381d5c..b4ccfae 100644 --- a/src/hmac_256/functions_test.sql +++ b/src/hmac_256/functions_test.sql @@ -12,3 +12,15 @@ DO $$ END; $$ LANGUAGE plpgsql; + + +DO $$ + DECLARE + e eql_v2_encrypted; + BEGIN + e := create_encrypted_json(1, 'hm'); + + ASSERT eql_v2.has_hmac_256(e); + END; +$$ LANGUAGE plpgsql; + diff --git a/src/operators/operator_class.sql b/src/operators/operator_class.sql index 74cfe6c..efb3668 100644 --- a/src/operators/operator_class.sql +++ b/src/operators/operator_class.sql @@ -10,9 +10,66 @@ -- REQUIRE: src/operators/>.sql +-- +-- Compare two eql_v2_encrypted values +-- Uses `ore_block_u64_8_256` or `has_hmac_256` index terms for comparison if defined on ONE of the compared value +-- +-- Important note: -- Index order of operations is reversed from equality operator. +-- In equality operations, `has_hmac_256` is preferred as it reduces to a text comparison and is more efficient +-- As compare is used for ordering, `ore_block_u64_8_256` provides more complete ordering and is checked first. +-- THe assumption is that if you add ore you are adding it because you want to use it specifically for comparison. + +-- Thusly, the logic for determining which index term to use: +-- Use ORE if BOTH parameters have ore index +-- Fallback to hmac if BOTH parameters have hmac index +-- Fallback to ORE if ONE of the parameters has ore index (will compare against a NULL term for the other parameter) +-- Fallback to hmac if ONE of the parameters has hmac index (will compare against a NULL term term for the other parameter) +-- +-- As a general rule, columns should have the same index terms as they are encrypted with the same configuration. +-- Index terms should only be different during an encryption config migration. +-- eg, when adding an ore index to a column any existing values will NOT have the ore index until encryptindexed/migrated +-- CREATE FUNCTION eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS integer IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + + -- PERFORM eql_v2.log('eql_v2.has_hmac_256(a)', eql_v2.has_hmac_256(a)::text); + -- PERFORM eql_v2.log('eql_v2.has_hmac_256(b)', eql_v2.has_hmac_256(b)::text); + -- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text); + -- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text); + + + -- Use ORE if BOTH parameters have ore index + IF eql_v2.has_ore_block_u64_8_256(a) AND eql_v2.has_ore_block_u64_8_256(b) THEN + RETURN eql_v2.compare_ore_block_u64_8_256(a, b); + END IF; + + -- Fallback to hmac if BOTH parameters have hmac index + IF eql_v2.has_hmac_256(a) AND eql_v2.has_hmac_256(b) THEN + RETURN eql_v2.compare_hmac(a, b); + END IF; + + -- Fallback to ORE if one of the parameters has ore index + IF eql_v2.has_ore_block_u64_8_256(a) OR eql_v2.has_ore_block_u64_8_256(b) THEN + RETURN eql_v2.compare_ore_block_u64_8_256(a, b); + END IF; + + -- Fallback to hmac if ONE of the parameters has hmac index + IF eql_v2.has_hmac_256(a) OR eql_v2.has_hmac_256(b) THEN + RETURN eql_v2.compare_hmac(a, b); + END IF; + + RAISE 'Expected an hmac_256 (hm) or ore_block_u64_8_256 (ob) value in json: %', val; + END; +$$ LANGUAGE plpgsql; + +-------------------- + +CREATE FUNCTION eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE AS $$ DECLARE a_ore eql_v2.ore_block_u64_8_256; @@ -38,6 +95,51 @@ AS $$ END; $$ LANGUAGE plpgsql; + +-------------------- + +CREATE FUNCTION eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + a_hmac eql_v2.hmac_256; + b_hmac eql_v2.hmac_256; + BEGIN + + a_hmac = eql_v2.hmac_256(a); + b_hmac = eql_v2.hmac_256(b); + + IF a_hmac IS NULL AND b_hmac IS NULL THEN + RETURN 0; + END IF; + + IF a_hmac IS NULL THEN + RETURN -1; + END IF; + + IF b_hmac IS NULL THEN + RETURN 1; + END IF; + + IF a_hmac = b_hmac THEN + RETURN 0; + END IF; + + IF a_hmac < b_hmac THEN + RETURN -1; + END IF; + + IF a_hmac > b_hmac THEN + RETURN 1; + END IF; + + END; +$$ LANGUAGE plpgsql; + + +-------------------- + CREATE OPERATOR FAMILY eql_v2.encrypted_operator USING btree; CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator AS @@ -48,3 +150,28 @@ CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypte OPERATOR 5 >, FUNCTION 1 eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted); + +-------------------- + +-- CREATE OPERATOR FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 USING btree; + +-- CREATE OPERATOR CLASS eql_v2.encrypted_operator_ore_block_u64_8_256 FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 AS +-- OPERATOR 1 <, +-- OPERATOR 2 <=, +-- OPERATOR 3 =, +-- OPERATOR 4 >=, +-- OPERATOR 5 >, +-- FUNCTION 1 eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted); + +-- -------------------- + +-- CREATE OPERATOR FAMILY eql_v2.encrypted_hmac_256_operator USING btree; + +-- CREATE OPERATOR CLASS eql_v2.encrypted_hmac_256_operator FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_hmac_256_operator AS +-- OPERATOR 1 <, +-- OPERATOR 2 <=, +-- OPERATOR 3 =, +-- OPERATOR 4 >=, +-- OPERATOR 5 >, +-- FUNCTION 1 eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted); + diff --git a/src/operators/operator_class_test.sql b/src/operators/operator_class_test.sql index 31132f6..8502cba 100644 --- a/src/operators/operator_class_test.sql +++ b/src/operators/operator_class_test.sql @@ -58,5 +58,77 @@ DO $$ END; $$ LANGUAGE plpgsql; +SELECT * FROM encrypted; + +-- +-- ORE GROUP BY +-- +DO $$ + DECLARE + ore_term eql_v2_encrypted; + result text; + BEGIN + + PERFORM create_table_with_encrypted(); + + EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"hm\": \"abc\"}")'';' into result; + + PERFORM eql_v2.log('', result); + + IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN + RAISE EXCEPTION 'Unexpected Bitmap Heap Scan: %', result; + ELSE + ASSERT true; + END IF; + + EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"ob\": \"abc\"}")'';' into result; + + PERFORM eql_v2.log('', result); + + IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN + RAISE EXCEPTION 'Unexpected Bitmap Heap Scan: %', result; + ELSE + ASSERT true; + END IF; + + + -- Add index + CREATE INDEX ON encrypted (e); + + EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"hm\": \"abc\"}")'';' into result; + + PERFORM eql_v2.log(result); + + IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN + ASSERT true; + ELSE + RAISE EXCEPTION 'Expected Bitmap Heap Scan: %', result; + END IF; + + PERFORM seed_encrypted_json(); + + SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + EXECUTE format('EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = %L::eql_v2_encrypted;', ore_term) into result; + + PERFORM eql_v2.log(result); + + IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN + ASSERT true; + ELSE + RAISE EXCEPTION 'Expected Bitmap Heap Scan: %', result; + END IF; + + + -- --- + -- EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"blah\": \"vtha\"}")'';' into result; + + -- IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN + -- ASSERT true; + -- ELSE + -- RAISE EXCEPTION 'Expected Bitmap Heap Scan: %', result; + -- END IF; + + END; +$$ LANGUAGE plpgsql; SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/ore_block_u64_8_256/functions.sql b/src/ore_block_u64_8_256/functions.sql index 0b1ea87..a82aa17 100644 --- a/src/ore_block_u64_8_256/functions.sql +++ b/src/ore_block_u64_8_256/functions.sql @@ -1,10 +1,10 @@ -- REQUIRE: src/schema.sql +-- REQUIRE: src/crypto.sql -- REQUIRE: src/encrypted/types.sql -- REQUIRE: src/encrypted/functions.sql -- REQUIRE: src/ore_block_u64_8_256/types.sql - -- Casts a jsonb array of hex-encoded strings to the `ore_block_u64_8_256` composite type. -- In other words, this function takes the ORE index format sent through in the -- EQL payload from Proxy and decodes it as the composite type that we use for @@ -48,7 +48,6 @@ $$ LANGUAGE plpgsql; -- extracts ore index from jsonb - CREATE FUNCTION eql_v2.ore_block_u64_8_256(val jsonb) RETURNS eql_v2.ore_block_u64_8_256 IMMUTABLE STRICT PARALLEL SAFE @@ -78,6 +77,29 @@ AS $$ $$ LANGUAGE plpgsql; +-- +-- Checks if val contains an ore_block_u64_8_256 search term +-- +CREATE FUNCTION eql_v2.has_ore_block_u64_8_256(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN val ? 'ob'; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.has_ore_block_u64_8_256(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.has_ore_block_u64_8_256(val.data); + END; +$$ LANGUAGE plpgsql; + + -- This function uses lexicographic comparison CREATE FUNCTION eql_v2.compare_ore_block_u64_8_256(a eql_v2.ore_block_u64_8_256, b eql_v2.ore_block_u64_8_256) @@ -97,6 +119,8 @@ AS $$ eq boolean := true; unequal_block smallint := 0; hash_key bytea; + data_block bytea; + encrypt_block bytea; target_block bytea; left_block_size CONSTANT smallint := 16; @@ -150,13 +174,18 @@ AS $$ -- first right block is at right offset + nonce_size (ordinally indexed) target_block := substr(b.bytes, right_offset + 17 + (unequal_block * right_block_size), right_block_size); + data_block := substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size); + + -- PERFORM eql_v2.log('substr', data_block::text); + -- PERFORM eql_v2.log('hash_key', hash_key::text); + -- PERFORM eql_v2.log('data_block', pg_typeof(data_block)::text); + -- PERFORM eql_v2.log('hash_key', pg_typeof(hash_key)::text); + + encrypt_block := public.encrypt(data_block::bytea, hash_key::bytea, 'aes-ecb'); + indicator := ( get_bit( - encrypt( - substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size), - hash_key, - 'aes-ecb' - ), + encrypt_block, 0 ) + get_bit(target_block, get_byte(a.bytes, unequal_block))) % 2; diff --git a/src/ore_block_u64_8_256/functions_test.sql b/src/ore_block_u64_8_256/functions_test.sql index db03c59..e7e7035 100644 --- a/src/ore_block_u64_8_256/functions_test.sql +++ b/src/ore_block_u64_8_256/functions_test.sql @@ -12,6 +12,21 @@ DO $$ END; $$ LANGUAGE plpgsql; + + +DO $$ + DECLARE + ore_term eql_v2_encrypted; + BEGIN + SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + + ASSERT eql_v2.has_ore_block_u64_8_256(ore_term); + + END; +$$ LANGUAGE plpgsql; + + + -- -- ORE - ORDER BY ore_block_u64_8_256(eql_v2_encrypted) -- diff --git a/src/ore_cllw_u64_8/functions.sql b/src/ore_cllw_u64_8/functions.sql index 0906358..4a2593f 100644 --- a/src/ore_cllw_u64_8/functions.sql +++ b/src/ore_cllw_u64_8/functions.sql @@ -13,6 +13,9 @@ CREATE FUNCTION eql_v2.ore_cllw_u64_8(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; IF NOT (val ? 'ocf') THEN RAISE 'Expected a ore_cllw_u64_8 index (ocf) value in json: %', val; @@ -39,6 +42,26 @@ AS $$ $$ LANGUAGE plpgsql; +CREATE FUNCTION eql_v2.has_ore_cllw_u64_8(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN val ? 'ocf'; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.has_ore_cllw_u64_8(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.has_ore_cllw_u64_8(val.data); + END; +$$ LANGUAGE plpgsql; + + -- -- Compare ore cllw bytes diff --git a/src/ore_cllw_var_8/functions.sql b/src/ore_cllw_var_8/functions.sql index 5706308..8e33f87 100644 --- a/src/ore_cllw_var_8/functions.sql +++ b/src/ore_cllw_var_8/functions.sql @@ -13,6 +13,10 @@ CREATE FUNCTION eql_v2.ore_cllw_var_8(val jsonb) AS $$ BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + IF NOT (val ? 'ocv') THEN RAISE 'Expected a ore_cllw_var_8 index (ocv) value in json: %', val; END IF; @@ -38,6 +42,25 @@ AS $$ $$ LANGUAGE plpgsql; +CREATE FUNCTION eql_v2.has_ore_cllw_var_8(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN val ? 'ocv'; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.has_ore_cllw_var_8(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.has_ore_cllw_var_8(val.data); + END; +$$ LANGUAGE plpgsql; + CREATE FUNCTION eql_v2.compare_ore_cllw_var_8(a eql_v2.ore_cllw_var_8, b eql_v2.ore_cllw_var_8) RETURNS int AS $$ diff --git a/tests/test_helpers.sql b/tests/test_helpers.sql index ed514e8..3d7c5bb 100644 --- a/tests/test_helpers.sql +++ b/tests/test_helpers.sql @@ -18,7 +18,6 @@ AS $$ CREATE TABLE encrypted ( id bigint GENERATED ALWAYS AS IDENTITY, - -- name_encrypted eql_v2_encrypted, e eql_v2_encrypted, PRIMARY KEY(id) );