Merge pull request #124 from cipherstash/fix-remvoe

fix: config functions remove process

Author: calvinbrewer

README.md

@@ -84,6 +84,37 @@ These frameworks use EQL to enable searchable encryption functionality in Postgr
 | Protect.php | [Protect.php](https://github.com/cipherstash/protectphp) |
 | CipherStash Proxy | [CipherStash Proxy](https://github.com/cipherstash/proxy) |
 
+## Versioning
+
+You can find the version of EQL installed in your database by running the following query:
+
+```sql
+SELECT eql_v2.version();
+```
+
+### Upgrading
+
+To upgrade to the latest version of EQL, you can simply run the install script again.
+
+1. Download the latest EQL install script:
+
+   ```sh
+   curl -sLo cipherstash-encrypt.sql https://github.com/cipherstash/encrypt-query-language/releases/latest/download/cipherstash-encrypt.sql
+   ```
+
+2. Run this command to install the custom types and functions:
+
+   ```sh
+   psql -f cipherstash-encrypt.sql
+   ```
+
+> [!NOTE]
+> The install script will not remove any existing configurations, so you can safely run it multiple times.
+
+#### Using dbdev?
+
+Follow the instructions in the [dbdev documentation](https://database.dev/cipherstash/eql) to upgrade the extension to your desired version.
+
 ## Developing
 
 See the [development guide](./DEVELOPMENT.md).

dbdev/eql.control

@@ -1,3 +1,3 @@
-default_version = 2.1.2
+default_version = 2.1.3
 comment = 'Index and search encrypted data in PostgreSQL with SQL'
 relocatable = true

docs/tutorials/proxy-configuration.md

@@ -1,14 +1,62 @@
 # CipherStash Proxy Configuration with EQL functions
 
-Initialize the column using the `eql_v2.add_column` function to enable encryption and decryption via CipherStash Proxy.
+## Prerequisites
+
+> [!IMPORTANT] 
+> Before using any EQL configuration functions, you must first create the encrypted column in your database table:
 
 ```sql
-SELECT eql_v2.add_column('users', 'encrypted_email'); -- where users is the table name and encrypted_email is the column name of type eql_v2_encrypted
+-- First, add the encrypted column to your table
+ALTER TABLE users ADD COLUMN encrypted_email eql_v2_encrypted;
+```
+
+The column **must** be of type `eql_v2_encrypted`.
+If you try to configure a column that doesn't exist in the database, you'll get the error:
+
+```
+ERROR: Some pending columns do not have an encrypted target
+```
+
+## Initializing column configuration
+
+After creating the encrypted column, initialize it for use with CipherStash Proxy using the `eql_v2.add_column` function:
+
+```sql
+SELECT eql_v2.add_column('users', 'encrypted_email', 'text'); -- Initialize the new encrypted column
+```
+
+**Full signature:**
+```sql
+SELECT eql_v2.add_column(
+  'table_name',       -- Name of the table
+  'column_name',      -- Name of the encrypted column (must already exist as type eql_v2_encrypted)
+  'cast_as',          -- PostgreSQL type to cast decrypted data [optional, defaults to 'text']
+  migrating           -- If true, stages changes without immediate activation [optional, defaults to false]
+);
 ```
 
 **Note:** This function allows you to encrypt and decrypt data but does not enable searchable encryption. See [Searching data with EQL](#searching-data-with-eql) for enabling searchable encryption.
 
-## Refreshing CipherStash Proxy Configuration
+## Complete setup workflow
+
+Here's the complete workflow to set up an encrypted column with search capabilities:
+
+```sql
+-- Step 1: Create the encrypted column in your table
+ALTER TABLE users ADD COLUMN encrypted_email eql_v2_encrypted;
+
+-- Step 2: Configure the column for encryption/decryption
+SELECT eql_v2.add_column('users', 'encrypted_email', 'text');
+
+-- Step 3: Add search indexes as needed
+SELECT eql_v2.add_search_config('users', 'encrypted_email', 'unique', 'text');
+SELECT eql_v2.add_search_config('users', 'encrypted_email', 'match', 'text');
+
+-- Step 4: Verify configuration
+SELECT * FROM eql_v2.config();
+```
+
+## Refreshing CipherStash Proxy configuration
 
 CipherStash Proxy refreshes the configuration every 60 seconds. To force an immediate refresh, run:
 
@@ -25,7 +73,7 @@ Encrypted data is stored as `jsonb` values in the PostgreSQL database, regardles
 
 You can read more about the data format [here](docs/reference/payload.md).
 
-### Inserting Data
+### Inserting data
 
 When inserting data into the encrypted column, wrap the plaintext in the appropriate EQL payload. These statements must be run through the CipherStash Proxy to **encrypt** the data.
 
@@ -54,7 +102,7 @@ Data is stored in the PostgreSQL database as:
 }
 ```
 
-### Reading Data
+### Reading data
 
 When querying data, select the encrypted column. CipherStash Proxy will **decrypt** the data automatically.
 
@@ -90,15 +138,18 @@ In order to perform searchable operations on encrypted data, you must configure
 
 ### Adding an index
 
+**Prerequisites:** The encrypted column must already exist in the database (see [Prerequisites](#prerequisites)) and be configured with `eql_v2.add_column`.
+
 Add an index to an encrypted column using the `eql_v2.add_search_config` function:
 
 ```sql
 SELECT eql_v2.add_search_config(
   'table_name',       -- Name of the table
   'column_name',      -- Name of the column
   'index_name',       -- Index kind ('unique', 'match', 'ore', 'ste_vec')
-  'cast_as',          -- PostgreSQL type to cast decrypted data ('text', 'int', etc.)
-  'opts'              -- Index options as JSONB (optional)
+  'cast_as',          -- PostgreSQL type to cast decrypted data ('text', 'int', etc.) [optional, defaults to 'text']
+  'opts',             -- Index options as JSONB [optional, defaults to '{}']
+  migrating           -- If true, stages changes without immediate activation [optional, defaults to false]
 );
 ```
 
@@ -115,7 +166,20 @@ SELECT eql_v2.add_search_config(
 );
 ```
 
-Configuration changes are automatically migrated and activated.
+**Example (With custom options and staging):**
+
+```sql
+SELECT eql_v2.add_search_config(
+  'users',
+  'encrypted_name',
+  'match',
+  'text',
+  '{"k": 6, "bf": 4096}',
+  true  -- Stage changes without immediate activation
+);
+```
+
+Configuration changes are automatically migrated and activated unless the `migrating` parameter is set to `true`.
 
 ## Searching data with EQL
 
@@ -322,9 +386,43 @@ EQL supports the following index types:
 
 Use these functions to manage your EQL configurations:
 
-- `eql_v2.add_column()` - Add a new encrypted column
-- `eql_v2.remove_column()` - Remove an encrypted column
-- `eql_v2.add_search_config()` - Add a search index
-- `eql_v2.remove_search_config()` - Remove a search index
-- `eql_v2.modify_search_config()` - Modify an existing search index
-- `eql_v2.config()` - View current configuration in tabular format
+**Column Management:**
+- `eql_v2.add_column(table_name, column_name, cast_as DEFAULT 'text', migrating DEFAULT false)` - Add a new encrypted column
+- `eql_v2.remove_column(table_name, column_name, migrating DEFAULT false)` - Remove an encrypted column completely
+
+**Index Management:**
+- `eql_v2.add_search_config(table_name, column_name, index_name, cast_as DEFAULT 'text', opts DEFAULT '{}', migrating DEFAULT false)` - Add a search index to a column
+- `eql_v2.remove_search_config(table_name, column_name, index_name, migrating DEFAULT false)` - Remove a specific search index (preserves column configuration)
+- `eql_v2.modify_search_config(table_name, column_name, index_name, cast_as DEFAULT 'text', opts DEFAULT '{}', migrating DEFAULT false)` - Modify an existing search index
+
+**Configuration Management:**
+- `eql_v2.migrate_config()` - Manually migrate pending configuration to encrypting state
+- `eql_v2.activate_config()` - Manually activate encrypting configuration
+- `eql_v2.discard()` - Discard pending configuration changes
+- `eql_v2.config()` - View current configuration in tabular format (returns a table with columns: state, relation, col_name, decrypts_as, indexes)
+
+> [!NOTE]  
+> All configuration functions automatically migrate and activate changes unless `migrating` is set to `true`. 
+>
+> When `migrating` is `true`, changes are staged but not immediately applied, allowing for batch configuration updates.
+
+**Important Behavior Differences:**
+- `remove_search_config()` removes only the specified index but preserves the column configuration (including `cast_as` setting)
+- `remove_column()` removes the entire column configuration including all its indexes
+- Empty configurations (no tables/columns) are automatically maintained as active to reflect the current state
+
+## Troubleshooting
+
+### Common errors
+
+**Error: "Some pending columns do not have an encrypted target"**
+- **Cause**: You're trying to configure a column that doesn't exist as `eql_v2_encrypted` type in the database
+- **Solution**: First create the encrypted column with `ALTER TABLE table_name ADD COLUMN column_name eql_v2_encrypted;`
+
+**Error: "Config exists for column: table_name column_name"**
+- **Cause**: You're trying to add a column that's already configured
+- **Solution**: Use `eql_v2.add_search_config()` to add indexes to existing columns, or `eql_v2.remove_column()` first if you want to reconfigure
+
+**Error: "No configuration exists for column: table_name column_name"**
+- **Cause**: You're trying to add search config to a column that hasn't been configured with `add_column` yet
+- **Solution**: First run `eql_v2.add_column()` to configure the column, then add search indexes

src/config/config_test.sql

@@ -37,12 +37,13 @@ DO $$
             c.data #> array['tables', 'users', 'name'] ? 'cast_as'));
 
     -- Match index removed
-    PERFORM eql_v2.remove_search_config('users', 'name', 'match');
+    PERFORM eql_v2.remove_search_config('users', 'name', 'match', migrating => true);
     ASSERT NOT (SELECT _search_config_exists('users', 'name', 'match'));
 
-    -- All indexes removed, delete the emtpty pending config
-    PERFORM eql_v2.remove_search_config('users', 'name', 'unique');
-    ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending'));
+    -- All indexes removed, but column config preserved
+    PERFORM eql_v2.remove_search_config('users', 'name', 'unique', migrating => true);
+    ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending'));
+    ASSERT (SELECT data #> array['tables', 'users', 'name', 'indexes'] = '{}' FROM eql_v2_configuration c WHERE c.state = 'pending');
 
   END;
 $$ LANGUAGE plpgsql;
@@ -82,15 +83,16 @@ DO $$
 
 
     -- Match index removed
-    PERFORM eql_v2.remove_search_config('users', 'name', 'match');
+    PERFORM eql_v2.remove_search_config('users', 'name', 'match', migrating => true);
     ASSERT NOT (SELECT _search_config_exists('users', 'name', 'match'));
 
     -- Match index removed
-    PERFORM eql_v2.remove_search_config('blah', 'vtha', 'unique');
+    PERFORM eql_v2.remove_search_config('blah', 'vtha', 'unique', migrating => true);
     ASSERT NOT (SELECT _search_config_exists('users', 'vtha', 'unique'));
 
-    -- All indexes removed, delete the emtpty pending config
-    ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending'));
+    -- All indexes removed, but column config preserved  
+    ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending'));
+    ASSERT (SELECT data #> array['tables', 'blah', 'vtha', 'indexes'] = '{}' FROM eql_v2_configuration c WHERE c.state = 'pending');
 
   END;
 $$ LANGUAGE plpgsql;
@@ -122,9 +124,10 @@ DO $$
             WHERE c.state = 'pending' AND
             c.data #> array['tables', 'users', 'name'] ? 'cast_as'));
 
-    -- All indexes removed, delete the emtpty pending config
-    PERFORM eql_v2.remove_search_config('users', 'name', 'match');
-    ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending'));
+    -- All indexes removed, but column config preserved
+    PERFORM eql_v2.remove_search_config('users', 'name', 'match', migrating => true);
+    ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending'));
+    ASSERT (SELECT data #> array['tables', 'users', 'name', 'indexes'] = '{}' FROM eql_v2_configuration c WHERE c.state = 'pending');
   END;
 $$ LANGUAGE plpgsql;
 
@@ -215,9 +218,13 @@ DO $$
 
     PERFORM eql_v2.remove_column('encrypted', 'e', migrating => true);
 
-    PERFORM assert_no_result(
-        'Pending configuration was removed',
-        'SELECT * FROM eql_v2_configuration c WHERE c.state = ''pending''');
+    PERFORM assert_count(
+        'Pending configuration exists but is empty',
+        'SELECT * FROM eql_v2_configuration c WHERE c.state = ''pending''',
+        1);
+    
+    -- Verify the config is empty
+    ASSERT (SELECT data #> array['tables'] = '{}' FROM eql_v2_configuration c WHERE c.state = 'pending');
 
   END;
 $$ LANGUAGE plpgsql;

src/config/constraints.sql

@@ -39,10 +39,16 @@ CREATE FUNCTION eql_v2.config_check_cast(val jsonb)
   RETURNS BOOLEAN
 AS $$
 	BEGIN
-    IF EXISTS (SELECT jsonb_array_elements_text(jsonb_path_query_array(val, '$.tables.*.*.cast_as')) = ANY('{text, int, small_int, big_int, real, double, boolean, date, jsonb}')) THEN
-      RETURN true;
+    -- If there are cast_as fields, validate them
+    IF EXISTS (SELECT jsonb_array_elements_text(jsonb_path_query_array(val, '$.tables.*.*.cast_as'))) THEN
+      IF (SELECT bool_and(cast_as = ANY('{text, int, small_int, big_int, real, double, boolean, date, jsonb}')) 
+          FROM (SELECT jsonb_array_elements_text(jsonb_path_query_array(val, '$.tables.*.*.cast_as')) AS cast_as) casts) THEN
+        RETURN true;
+      END IF;
+      RAISE 'Configuration has an invalid cast_as (%). Cast should be one of {text, int, small_int, big_int, real, double, boolean, date, jsonb}', val;
     END IF;
-    RAISE 'Configuration has an invalid cast_as (%). Cast should be one of {text, int, small_int, big_int, real, double, boolean, date, jsonb}', val;
+    -- If no cast_as fields exist (empty config), that's valid
+    RETURN true;
   END;
 $$ LANGUAGE plpgsql;
 
@@ -53,7 +59,7 @@ CREATE FUNCTION eql_v2.config_check_tables(val jsonb)
   RETURNS boolean
 AS $$
 	BEGIN
-    IF (val ? 'tables') AND (val->'tables' <> '{}'::jsonb) THEN
+    IF (val ? 'tables') THEN
       RETURN true;
     END IF;
     RAISE 'Configuration missing tables (tables) field: %', val;

src/config/functions.sql

@@ -70,7 +70,7 @@ $$ LANGUAGE plpgsql;
 
 
 
-CREATE FUNCTION eql_v2.remove_search_config(table_name text, column_name text, index_name text)
+CREATE FUNCTION eql_v2.remove_search_config(table_name text, column_name text, index_name text, migrating boolean DEFAULT false)
   RETURNS jsonb
 AS $$
   DECLARE
@@ -105,22 +105,12 @@ AS $$
     -- remove the index
     SELECT _config #- array['tables', table_name, column_name, 'indexes', index_name] INTO _config;
 
-    -- if column is now empty, remove the column
-    IF _config #> array['tables', table_name, column_name, 'indexes'] = '{}' THEN
-      SELECT _config #- array['tables', table_name, column_name] INTO _config;
-    END IF;
-
-    -- if table  is now empty, remove the table
-    IF _config #> array['tables', table_name] = '{}' THEN
-      SELECT _config #- array['tables', table_name] INTO _config;
-    END IF;
-
-    -- if config empty delete
-    -- or update the config
-    IF _config #> array['tables'] = '{}' THEN
-      DELETE FROM public.eql_v2_configuration WHERE state = 'pending';
-    ELSE
-      UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending';
+    -- update the config and migrate (even if empty)
+    UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending';
+    
+    IF NOT migrating THEN
+      PERFORM eql_v2.migrate_config();
+      PERFORM eql_v2.activate_config();
     END IF;
 
     -- exeunt
@@ -134,7 +124,7 @@ CREATE FUNCTION eql_v2.modify_search_config(table_name text, column_name text, i
   RETURNS jsonb
 AS $$
   BEGIN
-    PERFORM eql_v2.remove_search_config(table_name, column_name, index_name);
+    PERFORM eql_v2.remove_search_config(table_name, column_name, index_name, migrating);
     RETURN eql_v2.add_search_config(table_name, column_name, index_name, cast_as, opts, migrating);
   END;
 $$ LANGUAGE plpgsql;
@@ -293,19 +283,20 @@ AS $$
       SELECT _config #- array['tables', table_name] INTO _config;
     END IF;
 
-    -- if config empty delete
-    -- or update the config
-    IF _config #> array['tables'] = '{}' THEN
-      DELETE FROM public.eql_v2_configuration WHERE state = 'pending';
-    ELSE
-      UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending';
-    END IF;
-
     PERFORM eql_v2.remove_encrypted_constraint(table_name, column_name);
 
+    -- update the config (even if empty) and activate
+    UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending';
+    
     IF NOT migrating THEN
-      PERFORM eql_v2.migrate_config();
-      PERFORM eql_v2.activate_config();
+      -- For empty configs, skip migration validation and directly activate
+      IF _config #> array['tables'] = '{}' THEN
+        UPDATE public.eql_v2_configuration SET state = 'inactive' WHERE state = 'active';
+        UPDATE public.eql_v2_configuration SET state = 'active' WHERE state = 'pending';
+      ELSE
+        PERFORM eql_v2.migrate_config();
+        PERFORM eql_v2.activate_config();
+      END IF;
     END IF;
 
     -- exeunt