diff --git a/README.md b/README.md index caf2cd6..ae7857d 100644 --- a/README.md +++ b/README.md @@ -191,10 +191,10 @@ In order to perform searchable operations on encrypted data, you must configure ### Adding an index -Add an index to an encrypted column using the `eql_v2.add_index` function: +Add an index to an encrypted column using the `eql_v2.add_search_config` function: ```sql -SELECT eql_v2.add_index( +SELECT eql_v2.add_search_config( 'table_name', -- Name of the table 'column_name', -- Name of the column 'index_name', -- Index kind ('unique', 'match', 'ore', 'ste_vec') @@ -208,7 +208,7 @@ You can read more about the index configuration options [here](docs/reference/IN **Example (Unique index):** ```sql -SELECT eql_v2.add_index( +SELECT eql_v2.add_search_config( 'users', 'encrypted_email', 'unique', @@ -236,7 +236,7 @@ Enable equality search on encrypted data using the `eql_v2.unique` function. **Index configuration example:** ```sql -SELECT eql_v2.add_index( +SELECT eql_v2.add_search_config( 'users', 'encrypted_email', 'unique', @@ -266,7 +266,7 @@ Enables basic full-text search on encrypted data using the `eql_v2.match` functi **Index configuration example:** ```sql -SELECT eql_v2.add_index( +SELECT eql_v2.add_search_config( 'users', 'encrypted_email', 'match', diff --git a/docs/reference/INDEX.md b/docs/reference/INDEX.md index 95d7cd4..762a64a 100644 --- a/docs/reference/INDEX.md +++ b/docs/reference/INDEX.md @@ -1,17 +1,17 @@ # EQL index configuration The following functions allow you to configure indexes for encrypted columns. -All these functions modify the `cs_configuration_v2` table in your database, and are added during the EQL installation. +All these functions modify the `eql_v2_configuration` table in your database, and are added during the EQL installation. -> **IMPORTANT:** When you modify or add an index, you must re-encrypt data that's already been stored in the database. +> **IMPORTANT:** When you modify or add search configuration index, you must re-encrypt data that's already been stored in the database. > The CipherStash encryption solution will encrypt the data based on the current state of the configuration. -### Adding an index (`cs_add_index`) +### Configuring search (`eql_v2.add_search_config`) Add an index to an encrypted column. ```sql -SELECT cs_add_index_v2( +SELECT eql_v2.add_search_config( 'table_name', -- Name of the table 'column_name', -- Name of the column 'index_name', -- Index kind ('unique', 'match', 'ore', 'ste_vec') diff --git a/docs/reference/JSON.md b/docs/reference/JSON.md index 7d63778..3f284ab 100644 --- a/docs/reference/JSON.md +++ b/docs/reference/JSON.md @@ -36,7 +36,7 @@ Similar to how you configure indexes for text data, you can configure indexes fo The only difference is that you need to specify the `cast_as` parameter as `json` or `jsonb`. ```sql -SELECT cs_add_index_v2( +SELECT eql_v2.add_search_config( 'users', 'encrypted_json', 'ste_vec', diff --git a/docs/tutorials/GETTINGSTARTED.md b/docs/tutorials/GETTINGSTARTED.md index 185e7fe..c633ff6 100644 --- a/docs/tutorials/GETTINGSTARTED.md +++ b/docs/tutorials/GETTINGSTARTED.md @@ -160,7 +160,7 @@ We now have our database schema setup to store encrypted data. In this part we will learn about why we need to add indexes and how to add them. -When you install EQL, a table called `cs_configuration_v2` is created in your database. +When you install EQL, a table called `eql_v2_configuration` is created in your database. Adding indexes updates this table with the details and configuration needed for CipherStash Proxy to know how to encrypt your data, and what types of queries are able to be performed @@ -196,7 +196,7 @@ SELECT cs_add_index_v2('users', 'email_encrypted', 'ore', 'text'); CREATE INDEX ON users (cs_ore_64_8_v2(email_encrypted)); ``` -After adding these indexes, our `cs_configuration_v2` table will look like this: +After adding these indexes, our `eql_v2_configuration` table will look like this: ```bash id | 1 @@ -228,7 +228,7 @@ Prerequisites: - [Database is setup](#setup-your-database) - [Indexes added](#adding-indexes) -Ensure CipherStash Proxy has the most up to date configuration from the `cs_configuration_v2` table. +Ensure CipherStash Proxy has the most up to date configuration from the `eql_v2_configuration` table. CipherStash Proxy pings the database every 60 seconds to refresh the configuration but we can force the refresh by running: diff --git a/src/config/config_test.sql b/src/config/config_test.sql index bf3ac40..da140d0 100644 --- a/src/config/config_test.sql +++ b/src/config/config_test.sql @@ -4,8 +4,8 @@ -- -- Helper function for assertions -- -DROP FUNCTION IF EXISTS _index_exists(text, text, text, text); -CREATE FUNCTION _index_exists(table_name text, column_name text, index_name text, state text DEFAULT 'pending') +DROP FUNCTION IF EXISTS _search_config_exists(text, text, text, text); +CREATE FUNCTION _search_config_exists(table_name text, column_name text, index_name text, state text DEFAULT 'pending') RETURNS boolean LANGUAGE sql STRICT PARALLEL SAFE BEGIN ATOMIC @@ -25,23 +25,23 @@ DO $$ BEGIN -- Add indexes - PERFORM eql_v2.add_index('users', 'name', 'match'); - ASSERT (SELECT _index_exists('users', 'name', 'match')); + PERFORM eql_v2.add_search_config('users', 'name', 'match'); + ASSERT (SELECT _search_config_exists('users', 'name', 'match')); -- Add index with cast - PERFORM eql_v2.add_index('users', 'name', 'unique', 'int'); - ASSERT (SELECT _index_exists('users', 'name', 'unique')); + PERFORM eql_v2.add_search_config('users', 'name', 'unique', 'int'); + ASSERT (SELECT _search_config_exists('users', 'name', 'unique')); ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND c.data #> array['tables', 'users', 'name'] ? 'cast_as')); -- Match index removed - PERFORM eql_v2.remove_index('users', 'name', 'match'); - ASSERT NOT (SELECT _index_exists('users', 'name', 'match')); + PERFORM eql_v2.remove_search_config('users', 'name', 'match'); + ASSERT NOT (SELECT _search_config_exists('users', 'name', 'match')); -- All indexes removed, delete the emtpty pending config - PERFORM eql_v2.remove_index('users', 'name', 'unique'); + PERFORM eql_v2.remove_search_config('users', 'name', 'unique'); ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); END; @@ -60,16 +60,16 @@ DO $$ BEGIN -- Add indexes - PERFORM eql_v2.add_index('users', 'name', 'match'); - ASSERT (SELECT _index_exists('users', 'name', 'match')); + PERFORM eql_v2.add_search_config('users', 'name', 'match'); + ASSERT (SELECT _search_config_exists('users', 'name', 'match')); ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND c.data #> array['tables', 'users', 'name', 'indexes'] ? 'match')); -- Add index with cast - PERFORM eql_v2.add_index('blah', 'vtha', 'unique', 'int'); - ASSERT (SELECT _index_exists('blah', 'vtha', 'unique')); + PERFORM eql_v2.add_search_config('blah', 'vtha', 'unique', 'int'); + ASSERT (SELECT _search_config_exists('blah', 'vtha', 'unique')); ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND @@ -82,12 +82,12 @@ DO $$ -- Match index removed - PERFORM eql_v2.remove_index('users', 'name', 'match'); - ASSERT NOT (SELECT _index_exists('users', 'name', 'match')); + PERFORM eql_v2.remove_search_config('users', 'name', 'match'); + ASSERT NOT (SELECT _search_config_exists('users', 'name', 'match')); -- Match index removed - PERFORM eql_v2.remove_index('blah', 'vtha', 'unique'); - ASSERT NOT (SELECT _index_exists('users', 'vtha', 'unique')); + PERFORM eql_v2.remove_search_config('blah', 'vtha', 'unique'); + ASSERT NOT (SELECT _search_config_exists('users', 'vtha', 'unique')); -- All indexes removed, delete the emtpty pending config ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); @@ -107,12 +107,12 @@ $$ LANGUAGE plpgsql; DO $$ BEGIN - PERFORM eql_v2.add_index('users', 'name', 'match'); - ASSERT (SELECT _index_exists('users', 'name', 'match')); + PERFORM eql_v2.add_search_config('users', 'name', 'match'); + ASSERT (SELECT _search_config_exists('users', 'name', 'match')); -- Pending configuration contains the path `user/name.match.option` - PERFORM eql_v2.modify_index('users', 'name', 'match', 'int', '{"option": "value"}'::jsonb); - ASSERT (SELECT _index_exists('users', 'name', 'match')); + PERFORM eql_v2.modify_search_config('users', 'name', 'match', 'int', '{"option": "value"}'::jsonb); + ASSERT (SELECT _search_config_exists('users', 'name', 'match')); ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND @@ -123,7 +123,7 @@ DO $$ c.data #> array['tables', 'users', 'name'] ? 'cast_as')); -- All indexes removed, delete the emtpty pending config - PERFORM eql_v2.remove_index('users', 'name', 'match'); + PERFORM eql_v2.remove_search_config('users', 'name', 'match'); ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); END; $$ LANGUAGE plpgsql; @@ -160,16 +160,16 @@ INSERT INTO eql_v2_configuration (state, data) VALUES ( -- An encrypting config should exist DO $$ BEGIN - ASSERT (SELECT _index_exists('users', 'blah', 'match', 'active')); + ASSERT (SELECT _search_config_exists('users', 'blah', 'match', 'active')); - PERFORM eql_v2.add_index('users', 'name', 'match'); + PERFORM eql_v2.add_search_config('users', 'name', 'match'); -- index added to name - ASSERT (SELECT _index_exists('users', 'name', 'match' )); + ASSERT (SELECT _search_config_exists('users', 'name', 'match' )); -- pending is a copy of the active config -- and the active index still exists - ASSERT (SELECT _index_exists('users', 'blah', 'match')); + ASSERT (SELECT _search_config_exists('users', 'blah', 'match')); END; $$ LANGUAGE plpgsql; diff --git a/src/config/functions.sql b/src/config/functions.sql index 0854410..617960f 100644 --- a/src/config/functions.sql +++ b/src/config/functions.sql @@ -1,104 +1,16 @@ -- REQUIRE: src/config/types.sql +-- REQUIRE: src/config/functions_private.sql -- --- Configuration functions +-- Customer-facing configuration functions +-- Depends on private functions for implemenation -- -- - -CREATE FUNCTION eql_v2.config_default(config jsonb) - RETURNS jsonb - IMMUTABLE PARALLEL SAFE -AS $$ - BEGIN - IF config IS NULL THEN - SELECT jsonb_build_object('v', 1, 'tables', jsonb_build_object()) INTO config; - END IF; - RETURN config; - END; -$$ LANGUAGE plpgsql; - - - -CREATE FUNCTION eql_v2.config_add_table(table_name text, config jsonb) - RETURNS jsonb - IMMUTABLE PARALLEL SAFE -AS $$ - DECLARE - tbl jsonb; - BEGIN - IF NOT config #> array['tables'] ? table_name THEN - SELECT jsonb_insert(config, array['tables', table_name], jsonb_build_object()) INTO config; - END IF; - RETURN config; - END; -$$ LANGUAGE plpgsql; - - --- Add the column if it doesn't exist - -CREATE FUNCTION eql_v2.config_add_column(table_name text, column_name text, config jsonb) - RETURNS jsonb - IMMUTABLE PARALLEL SAFE -AS $$ - DECLARE - col jsonb; - BEGIN - IF NOT config #> array['tables', table_name] ? column_name THEN - SELECT jsonb_build_object('indexes', jsonb_build_object()) into col; - SELECT jsonb_set(config, array['tables', table_name, column_name], col) INTO config; - END IF; - RETURN config; - END; -$$ LANGUAGE plpgsql; - - --- Set the cast - -CREATE FUNCTION eql_v2.config_add_cast(table_name text, column_name text, cast_as text, config jsonb) - RETURNS jsonb - IMMUTABLE PARALLEL SAFE -AS $$ - BEGIN - SELECT jsonb_set(config, array['tables', table_name, column_name, 'cast_as'], to_jsonb(cast_as)) INTO config; - RETURN config; - END; -$$ LANGUAGE plpgsql; - - --- Add the column if it doesn't exist - -CREATE FUNCTION eql_v2.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb) - RETURNS jsonb - IMMUTABLE PARALLEL SAFE -AS $$ - BEGIN - SELECT jsonb_insert(config, array['tables', table_name, column_name, 'indexes', index_name], opts) INTO config; - RETURN config; - END; -$$ LANGUAGE plpgsql; - - --- --- Default options for match index --- - -CREATE FUNCTION eql_v2.config_match_default() - RETURNS jsonb -LANGUAGE sql STRICT PARALLEL SAFE -BEGIN ATOMIC - SELECT jsonb_build_object( - 'k', 6, - 'm', 2048, - 'include_original', true, - 'tokenizer', json_build_object('kind', 'ngram', 'token_length', 3), - 'token_filters', json_build_array(json_build_object('kind', 'downcase'))); -END; - -- -- Adds an index term to the configuration -- -CREATE FUNCTION eql_v2.add_index(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') +CREATE FUNCTION eql_v2.add_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') RETURNS jsonb AS $$ @@ -149,7 +61,7 @@ $$ LANGUAGE plpgsql; -CREATE FUNCTION eql_v2.remove_index(table_name text, column_name text, index_name text) +CREATE FUNCTION eql_v2.remove_search_config(table_name text, column_name text, index_name text) RETURNS jsonb AS $$ DECLARE @@ -209,12 +121,12 @@ $$ LANGUAGE plpgsql; -CREATE FUNCTION eql_v2.modify_index(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') +CREATE FUNCTION eql_v2.modify_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') RETURNS jsonb AS $$ BEGIN - PERFORM eql_v2.remove_index(table_name, column_name, index_name); - RETURN eql_v2.add_index(table_name, column_name, index_name, cast_as, opts); + PERFORM eql_v2.remove_search_config(table_name, column_name, index_name); + RETURN eql_v2.add_search_config(table_name, column_name, index_name, cast_as, opts); END; $$ LANGUAGE plpgsql; diff --git a/src/config/functions_private.sql b/src/config/functions_private.sql new file mode 100644 index 0000000..1d50056 --- /dev/null +++ b/src/config/functions_private.sql @@ -0,0 +1,95 @@ +-- REQUIRE: src/config/types.sql +-- +-- Private configuration functions +-- Internal implemention details that customers should not need to worry about. +-- +-- + +CREATE FUNCTION eql_v2.config_default(config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + BEGIN + IF config IS NULL THEN + SELECT jsonb_build_object('v', 1, 'tables', jsonb_build_object()) INTO config; + END IF; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.config_add_table(table_name text, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + DECLARE + tbl jsonb; + BEGIN + IF NOT config #> array['tables'] ? table_name THEN + SELECT jsonb_insert(config, array['tables', table_name], jsonb_build_object()) INTO config; + END IF; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- Add the column if it doesn't exist + +CREATE FUNCTION eql_v2.config_add_column(table_name text, column_name text, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + DECLARE + col jsonb; + BEGIN + IF NOT config #> array['tables', table_name] ? column_name THEN + SELECT jsonb_build_object('indexes', jsonb_build_object()) into col; + SELECT jsonb_set(config, array['tables', table_name, column_name], col) INTO config; + END IF; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- Set the cast + +CREATE FUNCTION eql_v2.config_add_cast(table_name text, column_name text, cast_as text, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + BEGIN + SELECT jsonb_set(config, array['tables', table_name, column_name, 'cast_as'], to_jsonb(cast_as)) INTO config; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- Add the column if it doesn't exist + +CREATE FUNCTION eql_v2.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + BEGIN + SELECT jsonb_insert(config, array['tables', table_name, column_name, 'indexes', index_name], opts) INTO config; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- +-- Default options for match index +-- + +CREATE FUNCTION eql_v2.config_match_default() + RETURNS jsonb +LANGUAGE sql STRICT PARALLEL SAFE +BEGIN ATOMIC + SELECT jsonb_build_object( + 'k', 6, + 'm', 2048, + 'include_original', true, + 'tokenizer', json_build_object('kind', 'ngram', 'token_length', 3), + 'token_filters', json_build_array(json_build_object('kind', 'downcase'))); +END; diff --git a/src/encryptindex/functions_test.sql b/src/encryptindex/functions_test.sql index 2e35ebd..6626678 100644 --- a/src/encryptindex/functions_test.sql +++ b/src/encryptindex/functions_test.sql @@ -154,7 +154,7 @@ CREATE TABLE users -- An encrypting config should exist DO $$ BEGIN - PERFORM eql_v2.add_index('users', 'name', 'match'); + PERFORM eql_v2.add_search_config('users', 'name', 'match'); PERFORM eql_v2.encrypt(); ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active')); @@ -204,7 +204,7 @@ CREATE TABLE users -- An encrypting config should exist DO $$ BEGIN - PERFORM eql_v2.add_index('users', 'name', 'match'); + PERFORM eql_v2.add_search_config('users', 'name', 'match'); PERFORM eql_v2.encrypt(); ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active')); @@ -254,7 +254,7 @@ CREATE TABLE users -- An encrypting config should exist DO $$ BEGIN - PERFORM eql_v2.add_index('users', 'name', 'match'); + PERFORM eql_v2.add_search_config('users', 'name', 'match'); PERFORM eql_v2.encrypt(); -- need to encrypt first PERFORM eql_v2.activate();