Enable simpler indexing on encrypted records with hmac_256

README.md

@@ -103,8 +103,8 @@ SELECT eql_v2.add_column('users', 'encrypted_email');
 After modifying configurations, activate them by running:
 
 ```sql
-SELECT eql_v2.encrypt();
-SELECT eql_v2.activate();
+SELECT eql_v2.migrate_config();
+SELECT eql_v2.activate_config();
 ```
 
 **Important:** These functions must be run after any modifications to the configuration.
@@ -219,8 +219,8 @@ SELECT eql_v2.add_search_config(
 After adding an index, you have to activate the configuration:
 
 ```sql
-SELECT eql_v2.encrypt();
-SELECT eql_v2.activate();
+SELECT eql_v2.migrate_config();
+SELECT eql_v2.activate_config();
 ```
 
 ## Searching data with EQL

src/config/functions.sql

@@ -144,7 +144,7 @@ $$ LANGUAGE plpgsql;
 -- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt.
 --
 
-CREATE FUNCTION eql_v2.encrypt()
+CREATE FUNCTION eql_v2.migrate_config()
   RETURNS boolean
 AS $$
 	BEGIN
@@ -168,7 +168,7 @@ $$ LANGUAGE plpgsql;
 
 
 
-CREATE FUNCTION eql_v2.activate()
+CREATE FUNCTION eql_v2.activate_config()
   RETURNS boolean
 AS $$
 	BEGIN

src/encryptindex/functions_test.sql

@@ -155,7 +155,7 @@ CREATE TABLE users
 DO $$
   BEGIN
     PERFORM eql_v2.add_search_config('users', 'name', 'match');
-    PERFORM eql_v2.encrypt();
+    PERFORM eql_v2.migrate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
@@ -205,7 +205,7 @@ CREATE TABLE users
 DO $$
   BEGIN
     PERFORM eql_v2.add_search_config('users', 'name', 'match');
-    PERFORM eql_v2.encrypt();
+    PERFORM eql_v2.migrate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
@@ -256,8 +256,8 @@ DO $$
   BEGIN
     PERFORM eql_v2.add_search_config('users', 'name', 'match');
 
-    PERFORM eql_v2.encrypt(); -- need to encrypt first
-    PERFORM eql_v2.activate();
+    PERFORM eql_v2.migrate_config(); -- need to encrypt first
+    PERFORM eql_v2.activate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'inactive'));

src/hmac_256/functions.sql

@@ -8,6 +8,10 @@ CREATE FUNCTION eql_v2.hmac_256(val jsonb)
   IMMUTABLE STRICT PARALLEL SAFE
 AS $$
 	BEGIN
+    IF val IS NULL THEN
+      RETURN NULL;
+    END IF;
+
     IF val ? 'hm' THEN
       RETURN val->>'hm';
     END IF;
@@ -16,6 +20,27 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
+CREATE FUNCTION eql_v2.has_hmac_256(val jsonb)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN val ? 'hm';
+  END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE FUNCTION eql_v2.has_hmac_256(val eql_v2_encrypted)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN eql_v2.has_hmac_256(val.data);
+  END;
+$$ LANGUAGE plpgsql;
+
+
+
 -- extracts hmac_256 index from an encrypted column
 
 CREATE FUNCTION eql_v2.hmac_256(val eql_v2_encrypted)

src/hmac_256/functions_test.sql

@@ -12,3 +12,15 @@ DO $$
 
   END;
 $$ LANGUAGE plpgsql;
+
+
+DO $$
+  DECLARE
+   e eql_v2_encrypted;
+  BEGIN
+    e := create_encrypted_json(1, 'hm');
+
+    ASSERT eql_v2.has_hmac_256(e);
+  END;
+$$ LANGUAGE plpgsql;
+

src/operators/operator_class.sql

@@ -10,9 +10,66 @@
 -- REQUIRE: src/operators/>.sql
 
 
+--
+-- Compare two eql_v2_encrypted values
+-- Uses `ore_block_u64_8_256` or `has_hmac_256` index terms for comparison if defined on ONE of the compared value
+--
+-- Important note: -- Index order of operations is reversed from equality operator.
+-- In equality operations, `has_hmac_256` is preferred as it reduces to a text comparison and is more efficient
+-- As compare is used for ordering, `ore_block_u64_8_256` provides more complete ordering and is checked first.
+-- THe assumption is that if you add ore you are adding it because you want to use it specifically for comparison.
+
+-- Thusly, the logic for determining which index term to use:
+--    Use ORE if BOTH parameters have ore index
+--    Fallback to hmac if BOTH parameters have hmac index
+--    Fallback to ORE if ONE of the parameters has ore index (will compare against a NULL term for the other parameter)
+--    Fallback to hmac if ONE of the parameters has hmac index (will compare against a NULL term term for the other parameter)
+--
+-- As a general rule, columns should have the same index terms as they are encrypted with the same configuration.
+-- Index terms should only be different during an encryption config migration.
+-- eg, when adding an ore index to a column any existing values will NOT have the ore index until encryptindexed/migrated
+--
 CREATE FUNCTION eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted)
   RETURNS integer
   IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  BEGIN
+
+    -- PERFORM eql_v2.log('eql_v2.has_hmac_256(a)', eql_v2.has_hmac_256(a)::text);
+    -- PERFORM eql_v2.log('eql_v2.has_hmac_256(b)', eql_v2.has_hmac_256(b)::text);
+    -- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text);
+    -- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text);
+
+
+    -- Use ORE if BOTH parameters have ore index
+    IF eql_v2.has_ore_block_u64_8_256(a) AND eql_v2.has_ore_block_u64_8_256(b) THEN
+      RETURN eql_v2.compare_ore_block_u64_8_256(a, b);
+    END IF;
+
+    -- Fallback to hmac if BOTH parameters have hmac index
+    IF eql_v2.has_hmac_256(a) AND eql_v2.has_hmac_256(b) THEN
+      RETURN eql_v2.compare_hmac(a, b);
+    END IF;
+
+    -- Fallback to ORE if one of the parameters has ore index
+    IF eql_v2.has_ore_block_u64_8_256(a) OR eql_v2.has_ore_block_u64_8_256(b) THEN
+      RETURN eql_v2.compare_ore_block_u64_8_256(a, b);
+    END IF;
+
+    -- Fallback to hmac if ONE of the parameters has hmac index
+    IF eql_v2.has_hmac_256(a) OR eql_v2.has_hmac_256(b) THEN
+      RETURN eql_v2.compare_hmac(a, b);
+    END IF;
+
+    RAISE 'Expected an hmac_256 (hm) or ore_block_u64_8_256 (ob) value in json: %', val;
+  END;
+$$ LANGUAGE plpgsql;
+
+--------------------
+
+CREATE FUNCTION eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted)
+  RETURNS integer
+  IMMUTABLE STRICT PARALLEL SAFE
 AS $$
   DECLARE
     a_ore eql_v2.ore_block_u64_8_256;
@@ -38,6 +95,51 @@ AS $$
   END;
 $$ LANGUAGE plpgsql;
 
+
+--------------------
+
+CREATE FUNCTION eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted)
+  RETURNS integer
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  DECLARE
+    a_hmac eql_v2.hmac_256;
+    b_hmac eql_v2.hmac_256;
+  BEGIN
+
+    a_hmac = eql_v2.hmac_256(a);
+    b_hmac = eql_v2.hmac_256(b);
+
+    IF a_hmac IS NULL AND b_hmac IS NULL THEN
+      RETURN 0;
+    END IF;
+
+    IF a_hmac IS NULL THEN
+      RETURN -1;
+    END IF;
+
+    IF b_hmac IS NULL THEN
+      RETURN 1;
+    END IF;
+
+    IF a_hmac = b_hmac THEN
+      RETURN 0;
+    END IF;
+
+    IF a_hmac < b_hmac THEN
+      RETURN -1;
+    END IF;
+
+    IF a_hmac > b_hmac THEN
+      RETURN 1;
+    END IF;
+
+  END;
+$$ LANGUAGE plpgsql;
+
+
+--------------------
+
 CREATE OPERATOR FAMILY eql_v2.encrypted_operator USING btree;
 
 CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator AS
@@ -48,3 +150,28 @@ CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypte
   OPERATOR 5 >,
   FUNCTION 1 eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted);
 
+
+--------------------
+
+-- CREATE OPERATOR FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 USING btree;
+
+-- CREATE OPERATOR CLASS eql_v2.encrypted_operator_ore_block_u64_8_256 FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 AS
+--   OPERATOR 1 <,
+--   OPERATOR 2 <=,
+--   OPERATOR 3 =,
+--   OPERATOR 4 >=,
+--   OPERATOR 5 >,
+--   FUNCTION 1 eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted);
+
+-- --------------------
+
+-- CREATE OPERATOR FAMILY eql_v2.encrypted_hmac_256_operator USING btree;
+
+-- CREATE OPERATOR CLASS eql_v2.encrypted_hmac_256_operator FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_hmac_256_operator AS
+--   OPERATOR 1 <,
+--   OPERATOR 2 <=,
+--   OPERATOR 3 =,
+--   OPERATOR 4 >=,
+--   OPERATOR 5 >,
+--   FUNCTION 1 eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted);
+

src/operators/operator_class_test.sql

@@ -58,5 +58,77 @@ DO $$
   END;
 $$ LANGUAGE plpgsql;
 
+SELECT * FROM encrypted;
+
+--
+-- ORE GROUP BY
+--
+DO $$
+  DECLARE
+    ore_term eql_v2_encrypted;
+    result text;
+  BEGIN
+
+    PERFORM create_table_with_encrypted();
+
+    EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"hm\": \"abc\"}")'';' into result;
+
+    PERFORM eql_v2.log('', result);
+
+    IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+      RAISE EXCEPTION 'Unexpected Bitmap Heap Scan: %', result;
+    ELSE
+      ASSERT true;
+    END IF;
+
+    EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"ob\": \"abc\"}")'';' into result;
+
+    PERFORM eql_v2.log('', result);
+
+    IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+      RAISE EXCEPTION 'Unexpected Bitmap Heap Scan: %', result;
+    ELSE
+      ASSERT true;
+    END IF;
+
+
+    -- Add index
+    CREATE INDEX ON encrypted (e);
+
+    EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"hm\": \"abc\"}")'';' into result;
+
+    PERFORM eql_v2.log(result);
+
+    IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+      ASSERT true;
+    ELSE
+      RAISE EXCEPTION 'Expected Bitmap Heap Scan: %', result;
+    END IF;
+
+    PERFORM seed_encrypted_json();
+
+    SELECT ore.e FROM ore WHERE id = 42 INTO ore_term;
+    EXECUTE format('EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = %L::eql_v2_encrypted;', ore_term) into result;
+
+    PERFORM eql_v2.log(result);
+
+    IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+      ASSERT true;
+    ELSE
+      RAISE EXCEPTION 'Expected Bitmap Heap Scan: %', result;
+    END IF;
+
+
+    -- ---
+    -- EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"blah\": \"vtha\"}")'';' into result;
+
+    -- IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+    --   ASSERT true;
+    -- ELSE
+    --   RAISE EXCEPTION 'Expected Bitmap Heap Scan: %', result;
+    -- END IF;
+
+  END;
+$$ LANGUAGE plpgsql;
 
 SELECT drop_table_with_encrypted();