Merge pull request #101 from cipherstash/eql-v2-0-jsonb

Eql v2.0 - Searchable Encrypted JSONB

Author: tobyhede

diagrams/overview-insert.drawio.svg

@@ -0,0 +1,38 @@
+-- REQUIRE: src/schema.sql
+
+-- extracts ste_vec index from a jsonb value
+-- DROP FUNCTION IF EXISTS  eql_v1.blake3(val jsonb);
+
+-- extracts blake3 index from a jsonb value
+-- DROP FUNCTION IF EXISTS  eql_v1.blake3(val jsonb);
+
+CREATE FUNCTION eql_v1.blake3(val jsonb)
+  RETURNS eql_v1.blake3
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+
+    IF NOT (val ? 'b') NULL THEN
+        RAISE 'Expected a blake3 index (b) value in json: %', val;
+    END IF;
+
+    IF val->>'b' IS NULL THEN
+      RETURN NULL;
+    END IF;
+
+    RETURN val->>'b';
+  END;
+$$ LANGUAGE plpgsql;
+
+
+-- extracts blake3 index from an eql_v1_encrypted value
+-- DROP FUNCTION IF EXISTS  eql_v1.blake3(val eql_v1_encrypted);
+
+CREATE FUNCTION eql_v1.blake3(val eql_v1_encrypted)
+  RETURNS eql_v1.blake3
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  BEGIN
+    RETURN (SELECT eql_v1.blake3(val.data));
+  END;
+$$ LANGUAGE plpgsql;

diagrams/overview-select.drawio.svg

@@ -0,0 +1,4 @@
+-- REQUIRE: src/schema.sql
+
+-- DROP DOMAIN IF EXISTS eql_v1.blake3;
+CREATE DOMAIN eql_v1.blake3 AS text;

src/blake3/functions.sql

@@ -2,11 +2,65 @@
 -- REQUIRE: src/schema.sql
 
 
+-- Constant time comparison of 2 bytea values
+
+
+
+
+
+
+-- DROP FUNCTION IF EXISTS eql_v1.bytea_eq(a bytea, b bytea);
+
+CREATE FUNCTION eql_v1.bytea_eq(a bytea, b bytea) RETURNS boolean AS $$
+DECLARE
+    result boolean;
+    differing bytea;
+BEGIN
+
+    -- Check if the bytea values are the same length
+    IF LENGTH(a) != LENGTH(b) THEN
+        RETURN false;
+    END IF;
+
+    -- Compare each byte in the bytea values
+    result := true;
+    FOR i IN 1..LENGTH(a) LOOP
+        IF SUBSTRING(a FROM i FOR 1) != SUBSTRING(b FROM i FOR 1) THEN
+            result := result AND false;
+        END IF;
+    END LOOP;
+
+    RETURN result;
+END;
+$$ LANGUAGE plpgsql;
+
+
+-- DROP FUNCTION IF EXISTS eql_v1.jsonb_array_to_bytea_array(val jsonb);
+
+-- Casts a jsonb array of hex-encoded strings to an array of bytea.
+CREATE FUNCTION eql_v1.jsonb_array_to_bytea_array(val jsonb)
+RETURNS bytea[] AS $$
+DECLARE
+  terms_arr bytea[];
+BEGIN
+  IF jsonb_typeof(val) = 'null' THEN
+    RETURN NULL;
+  END IF;
+
+  SELECT array_agg(decode(value::text, 'hex')::bytea)
+    INTO terms_arr
+  FROM jsonb_array_elements_text(val) AS value;
+
+  RETURN terms_arr;
+END;
+$$ LANGUAGE plpgsql;
+
+
 
 --
 -- Convenience function to log a message
 --
-DROP FUNCTION IF EXISTS eql_v1.log(text);
+-- DROP FUNCTION IF EXISTS eql_v1.log(text);
 CREATE FUNCTION eql_v1.log(s text)
     RETURNS void
 AS $$
@@ -19,7 +73,7 @@ $$ LANGUAGE plpgsql;
 --
 -- Convenience function to describe a test
 --
-DROP FUNCTION IF EXISTS eql_v1.log(text, text);
+-- DROP FUNCTION IF EXISTS eql_v1.log(text, text);
 CREATE FUNCTION eql_v1.log(ctx text, s text)
     RETURNS void
 AS $$

src/blake3/types.sql

@@ -4,7 +4,7 @@
 --
 -- Helper function for assertions
 --
-DROP FUNCTION IF EXISTS _index_exists(text, text, text, text);
+-- DROP FUNCTION IF EXISTS _index_exists(text, text, text, text);
 CREATE FUNCTION _index_exists(table_name text, column_name text, index_name text, state text DEFAULT 'pending')
   RETURNS boolean
 LANGUAGE sql STRICT PARALLEL SAFE

src/common.sql

@@ -5,7 +5,7 @@
 --
 -- Used by the eql_v1.config_check_indexes as part of the configuration_data_v1 constraint
 --
-DROP FUNCTION IF EXISTS eql_v1.config_get_indexes(jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.config_get_indexes(jsonb);
 CREATE FUNCTION eql_v1.config_get_indexes(val jsonb)
     RETURNS SETOF text
     LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
@@ -18,7 +18,7 @@ END;
 --
 -- Used by the cs_configuration_data_v1_check constraint
 --
-DROP FUNCTION IF EXISTS eql_v1.config_check_indexes(jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.config_check_indexes(jsonb);
 CREATE FUNCTION eql_v1.config_check_indexes(val jsonb)
   RETURNS BOOLEAN
   IMMUTABLE STRICT PARALLEL SAFE
@@ -36,7 +36,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-DROP FUNCTION IF EXISTS eql_v1.config_check_cast(jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.config_check_cast(jsonb);
 
 CREATE FUNCTION eql_v1.config_check_cast(val jsonb)
   RETURNS BOOLEAN
@@ -52,7 +52,7 @@ $$ LANGUAGE plpgsql;
 --
 -- Should include a tables field
 -- Tables should not be empty
-DROP FUNCTION IF EXISTS eql_v1.config_check_tables(jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.config_check_tables(jsonb);
 CREATE FUNCTION eql_v1.config_check_tables(val jsonb)
   RETURNS boolean
 AS $$
@@ -65,7 +65,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 -- Should include a version field
-DROP FUNCTION IF EXISTS eql_v1.config_check_version(jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.config_check_version(jsonb);
 CREATE FUNCTION eql_v1.config_check_version(val jsonb)
   RETURNS boolean
 AS $$

src/config/config_test.sql

@@ -4,7 +4,7 @@
 --
 --
 
-DROP FUNCTION IF EXISTS  eql_v1.config_default(config jsonb);
+-- DROP FUNCTION IF EXISTS  eql_v1.config_default(config jsonb);
 
 CREATE FUNCTION eql_v1.config_default(config jsonb)
   RETURNS jsonb
@@ -19,7 +19,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-DROP FUNCTION IF EXISTS eql_v1.config_add_table(table_name text, config jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.config_add_table(table_name text, config jsonb);
 
 CREATE FUNCTION eql_v1.config_add_table(table_name text, config jsonb)
   RETURNS jsonb
@@ -37,7 +37,7 @@ $$ LANGUAGE plpgsql;
 
 
 -- Add the column if it doesn't exist
-DROP FUNCTION IF EXISTS eql_v1.config_add_column(table_name text, column_name text, config jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.config_add_column(table_name text, column_name text, config jsonb);
 
 CREATE FUNCTION eql_v1.config_add_column(table_name text, column_name text, config jsonb)
   RETURNS jsonb
@@ -56,7 +56,7 @@ $$ LANGUAGE plpgsql;
 
 
 -- Set the cast
-DROP FUNCTION IF EXISTS eql_v1.config_add_cast(table_name text, column_name text, cast_as text, config jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.config_add_cast(table_name text, column_name text, cast_as text, config jsonb);
 
 CREATE FUNCTION eql_v1.config_add_cast(table_name text, column_name text, cast_as text, config jsonb)
   RETURNS jsonb
@@ -70,7 +70,7 @@ $$ LANGUAGE plpgsql;
 
 
 -- Add the column if it doesn't exist
-DROP FUNCTION IF EXISTS eql_v1.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb);
 
 CREATE FUNCTION eql_v1.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb)
   RETURNS jsonb
@@ -86,7 +86,7 @@ $$ LANGUAGE plpgsql;
 --
 -- Default options for match index
 --
-DROP FUNCTION IF EXISTS eql_v1.config_match_default();
+-- DROP FUNCTION IF EXISTS eql_v1.config_match_default();
 
 CREATE FUNCTION eql_v1.config_match_default()
   RETURNS jsonb
@@ -103,7 +103,7 @@ END;
 --
 -- Adds an index term to the configuration
 --
-DROP FUNCTION IF EXISTS eql_v1.add_index(table_name text, column_name text, index_name text, cast_as text, opts jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.add_index(table_name text, column_name text, index_name text, cast_as text, opts jsonb);
 
 CREATE FUNCTION eql_v1.add_index(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}')
   RETURNS jsonb
@@ -155,7 +155,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-DROP FUNCTION IF EXISTS eql_v1.remove_index(table_name text, column_name text, index_name text);
+-- DROP FUNCTION IF EXISTS eql_v1.remove_index(table_name text, column_name text, index_name text);
 
 CREATE FUNCTION eql_v1.remove_index(table_name text, column_name text, index_name text)
   RETURNS jsonb
@@ -216,7 +216,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-DROP FUNCTION IF EXISTS eql_v1.modify_index(table_name text, column_name text, index_name text, cast_as text, opts jsonb);
+-- DROP FUNCTION IF EXISTS eql_v1.modify_index(table_name text, column_name text, index_name text, cast_as text, opts jsonb);
 
 CREATE FUNCTION eql_v1.modify_index(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}')
   RETURNS jsonb
@@ -240,7 +240,7 @@ $$ LANGUAGE plpgsql;
 --
 -- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt.
 --
-DROP FUNCTION IF EXISTS eql_v1.encrypt();
+-- DROP FUNCTION IF EXISTS eql_v1.encrypt();
 
 CREATE FUNCTION eql_v1.encrypt(force boolean DEFAULT false)
   RETURNS boolean
@@ -267,7 +267,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-DROP FUNCTION IF EXISTS eql_v1.activate();
+-- DROP FUNCTION IF EXISTS eql_v1.activate();
 
 CREATE FUNCTION eql_v1.activate()
   RETURNS boolean
@@ -285,7 +285,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-DROP FUNCTION IF EXISTS eql_v1.discard();
+-- DROP FUNCTION IF EXISTS eql_v1.discard();
 
 CREATE FUNCTION eql_v1.discard()
   RETURNS boolean
@@ -301,7 +301,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-DROP FUNCTION IF EXISTS eql_v1.add_column(table_name text, column_name text, cast_as text);
+-- DROP FUNCTION IF EXISTS eql_v1.add_column(table_name text, column_name text, cast_as text);
 
 CREATE FUNCTION eql_v1.add_column(table_name text, column_name text, cast_as text DEFAULT 'text')
   RETURNS jsonb
@@ -340,7 +340,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-DROP FUNCTION IF EXISTS eql_v1.remove_column(table_name text, column_name text);
+-- DROP FUNCTION IF EXISTS eql_v1.remove_column(table_name text, column_name text);
 
 CREATE FUNCTION eql_v1.remove_column(table_name text, column_name text)
   RETURNS jsonb
@@ -396,7 +396,7 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-DROP FUNCTION IF EXISTS eql_v1.reload_config();
+-- DROP FUNCTION IF EXISTS eql_v1.reload_config();
 
 CREATE FUNCTION eql_v1.reload_config()
   RETURNS void
@@ -405,7 +405,7 @@ BEGIN ATOMIC
   RETURN NULL;
 END;
 
-DROP FUNCTION IF EXISTS eql_v1.config();
+-- DROP FUNCTION IF EXISTS eql_v1.config();
 
 -- A convenience function to return the configuration in a tabular format, allowing for easier filtering, and querying.
 -- Query using `SELECT * FROM cs_config();`

src/config/constraints.sql

@@ -2,7 +2,7 @@
 
 --
 --
--- CREATE the cs_configuration_v1 TABLE
+-- CREATE the eql_v1_configuration TABLE
 --
 CREATE TABLE IF NOT EXISTS public.eql_v1_configuration
 (

src/config/functions.sql

@@ -3,8 +3,8 @@
 -- REQUIRE: src/ore/functions.sql
 
 -- Aggregate functions for ORE
-DROP AGGREGATE IF EXISTS eql_v1.min(eql_v1_encrypted);
-DROP FUNCTION IF EXISTS eql_v1.min(a eql_v1_encrypted, b eql_v1_encrypted);
+-- DROP AGGREGATE IF EXISTS eql_v1.min(eql_v1_encrypted);
+-- DROP FUNCTION IF EXISTS eql_v1.min(a eql_v1_encrypted, b eql_v1_encrypted);
 
 CREATE FUNCTION eql_v1.min(a eql_v1_encrypted, b eql_v1_encrypted)
   RETURNS eql_v1_encrypted
@@ -27,8 +27,8 @@ CREATE AGGREGATE eql_v1.min(eql_v1_encrypted)
   stype = eql_v1_encrypted
 );
 
-DROP AGGREGATE IF EXISTS eql_v1.max(eql_v1_encrypted);
-DROP FUNCTION IF EXISTS eql_v1.max(a eql_v1_encrypted, b eql_v1_encrypted);
+-- DROP AGGREGATE IF EXISTS eql_v1.max(eql_v1_encrypted);
+-- DROP FUNCTION IF EXISTS eql_v1.max(a eql_v1_encrypted, b eql_v1_encrypted);
 
 CREATE FUNCTION eql_v1.max(a eql_v1_encrypted, b eql_v1_encrypted)
 RETURNS eql_v1_encrypted