fix: config functions remove process

calvinbrewer · calvinbrewer · commit 29ff0fffd553 · 2025-07-15T08:04:46.000-06:00
diff --git a/README.md b/README.md
@@ -84,6 +84,37 @@ These frameworks use EQL to enable searchable encryption functionality in Postgr
 | Protect.php | [Protect.php](https://github.com/cipherstash/protectphp) |
 | CipherStash Proxy | [CipherStash Proxy](https://github.com/cipherstash/proxy) |
 
+## Versioning
+
+You can find the version of EQL installed in your database by running the following query:
+
+```sql
+SELECT eql_v2.version();
+```
+
+### Upgrading
+
+To upgrade to the latest version of EQL, you can simply run the install script again.
+
+1. Download the latest EQL install script:
+
+   ```sh
+   curl -sLo cipherstash-encrypt.sql https://github.com/cipherstash/encrypt-query-language/releases/latest/download/cipherstash-encrypt.sql
+   ```
+
+2. Run this command to install the custom types and functions:
+
+   ```sh
+   psql -f cipherstash-encrypt.sql
+   ```
+
+> [!NOTE]
+> The install script will not remove any existing configurations, so you can safely run it multiple times.
+
+#### Using dbdev?
+
+Follow the instructions in the [dbdev documentation](https://database.dev/cipherstash/eql) to upgrade the extension to your desired version.
+
 ## Developing
 
 See the [development guide](./DEVELOPMENT.md).
diff --git a/dbdev/eql.control b/dbdev/eql.control
@@ -1,3 +1,3 @@
-default_version = 2.1.2
+default_version = 2.1.3
 comment = 'Index and search encrypted data in PostgreSQL with SQL'
 relocatable = true
diff --git a/docs/tutorials/proxy-configuration.md b/docs/tutorials/proxy-configuration.md
@@ -3,7 +3,17 @@
 Initialize the column using the `eql_v2.add_column` function to enable encryption and decryption via CipherStash Proxy.
 
 ```sql
-SELECT eql_v2.add_column('users', 'encrypted_email'); -- where users is the table name and encrypted_email is the column name of type eql_v2_encrypted
+SELECT eql_v2.add_column('users', 'encrypted_email', 'text'); -- where users is the table name and encrypted_email is the column name of type eql_v2_encrypted
+```
+
+**Full signature:**
+```sql
+SELECT eql_v2.add_column(
+  'table_name',       -- Name of the table
+  'column_name',      -- Name of the column (must be of type eql_v2_encrypted)
+  'cast_as',          -- PostgreSQL type to cast decrypted data [optional, defaults to 'text']
+  migrating           -- If true, stages changes without immediate activation [optional, defaults to false]
+);
 ```
 
 **Note:** This function allows you to encrypt and decrypt data but does not enable searchable encryption. See [Searching data with EQL](#searching-data-with-eql) for enabling searchable encryption.
@@ -97,8 +107,9 @@ SELECT eql_v2.add_search_config(
   'table_name',       -- Name of the table
   'column_name',      -- Name of the column
   'index_name',       -- Index kind ('unique', 'match', 'ore', 'ste_vec')
-  'cast_as',          -- PostgreSQL type to cast decrypted data ('text', 'int', etc.)
-  'opts'              -- Index options as JSONB (optional)
+  'cast_as',          -- PostgreSQL type to cast decrypted data ('text', 'int', etc.) [optional, defaults to 'text']
+  'opts',             -- Index options as JSONB [optional, defaults to '{}']
+  migrating           -- If true, stages changes without immediate activation [optional, defaults to false]
 );
 ```
 
@@ -115,7 +126,20 @@ SELECT eql_v2.add_search_config(
 );
 ```
 
-Configuration changes are automatically migrated and activated.
+**Example (With custom options and staging):**
+
+```sql
+SELECT eql_v2.add_search_config(
+  'users',
+  'encrypted_name',
+  'match',
+  'text',
+  '{"k": 6, "bf": 4096}',
+  true  -- Stage changes without immediate activation
+);
+```
+
+Configuration changes are automatically migrated and activated unless the `migrating` parameter is set to `true`.
 
 ## Searching data with EQL
 
@@ -322,9 +346,24 @@ EQL supports the following index types:
 
 Use these functions to manage your EQL configurations:
 
-- `eql_v2.add_column()` - Add a new encrypted column
-- `eql_v2.remove_column()` - Remove an encrypted column
-- `eql_v2.add_search_config()` - Add a search index
-- `eql_v2.remove_search_config()` - Remove a search index
-- `eql_v2.modify_search_config()` - Modify an existing search index
-- `eql_v2.config()` - View current configuration in tabular format
+**Column Management:**
+- `eql_v2.add_column(table_name, column_name, cast_as DEFAULT 'text', migrating DEFAULT false)` - Add a new encrypted column
+- `eql_v2.remove_column(table_name, column_name, migrating DEFAULT false)` - Remove an encrypted column completely
+
+**Index Management:**
+- `eql_v2.add_search_config(table_name, column_name, index_name, cast_as DEFAULT 'text', opts DEFAULT '{}', migrating DEFAULT false)` - Add a search index to a column
+- `eql_v2.remove_search_config(table_name, column_name, index_name, migrating DEFAULT false)` - Remove a specific search index (preserves column configuration)
+- `eql_v2.modify_search_config(table_name, column_name, index_name, cast_as DEFAULT 'text', opts DEFAULT '{}', migrating DEFAULT false)` - Modify an existing search index
+
+**Configuration Management:**
+- `eql_v2.migrate_config()` - Manually migrate pending configuration to encrypting state
+- `eql_v2.activate_config()` - Manually activate encrypting configuration
+- `eql_v2.discard()` - Discard pending configuration changes
+- `eql_v2.config()` - View current configuration in tabular format (returns a table with columns: state, relation, col_name, decrypts_as, indexes)
+
+**Note:** All configuration functions automatically migrate and activate changes unless `migrating` is set to `true`. When `migrating` is `true`, changes are staged but not immediately applied, allowing for batch configuration updates.
+
+**Important Behavior Differences:**
+- `remove_search_config()` removes only the specified index but preserves the column configuration (including `cast_as` setting)
+- `remove_column()` removes the entire column configuration including all its indexes
+- Empty configurations (no tables/columns) are automatically maintained as active to reflect the current state
diff --git a/src/config/constraints.sql b/src/config/constraints.sql
@@ -39,10 +39,16 @@ CREATE FUNCTION eql_v2.config_check_cast(val jsonb)
   RETURNS BOOLEAN
 AS $$
 	BEGIN
-    IF EXISTS (SELECT jsonb_array_elements_text(jsonb_path_query_array(val, '$.tables.*.*.cast_as')) = ANY('{text, int, small_int, big_int, real, double, boolean, date, jsonb}')) THEN
-      RETURN true;
+    -- If there are cast_as fields, validate them
+    IF EXISTS (SELECT jsonb_array_elements_text(jsonb_path_query_array(val, '$.tables.*.*.cast_as'))) THEN
+      IF (SELECT bool_and(cast_as = ANY('{text, int, small_int, big_int, real, double, boolean, date, jsonb}')) 
+          FROM (SELECT jsonb_array_elements_text(jsonb_path_query_array(val, '$.tables.*.*.cast_as')) AS cast_as) casts) THEN
+        RETURN true;
+      END IF;
+      RAISE 'Configuration has an invalid cast_as (%). Cast should be one of {text, int, small_int, big_int, real, double, boolean, date, jsonb}', val;
     END IF;
-    RAISE 'Configuration has an invalid cast_as (%). Cast should be one of {text, int, small_int, big_int, real, double, boolean, date, jsonb}', val;
+    -- If no cast_as fields exist (empty config), that's valid
+    RETURN true;
   END;
 $$ LANGUAGE plpgsql;
 
@@ -53,7 +59,7 @@ CREATE FUNCTION eql_v2.config_check_tables(val jsonb)
   RETURNS boolean
 AS $$
 	BEGIN
-    IF (val ? 'tables') AND (val->'tables' <> '{}'::jsonb) THEN
+    IF (val ? 'tables') THEN
       RETURN true;
     END IF;
     RAISE 'Configuration missing tables (tables) field: %', val;
diff --git a/src/config/functions.sql b/src/config/functions.sql
@@ -70,7 +70,7 @@ $$ LANGUAGE plpgsql;
 
 
 
-CREATE FUNCTION eql_v2.remove_search_config(table_name text, column_name text, index_name text)
+CREATE FUNCTION eql_v2.remove_search_config(table_name text, column_name text, index_name text, migrating boolean DEFAULT false)
   RETURNS jsonb
 AS $$
   DECLARE
@@ -105,22 +105,12 @@ AS $$
     -- remove the index
     SELECT _config #- array['tables', table_name, column_name, 'indexes', index_name] INTO _config;
 
-    -- if column is now empty, remove the column
-    IF _config #> array['tables', table_name, column_name, 'indexes'] = '{}' THEN
-      SELECT _config #- array['tables', table_name, column_name] INTO _config;
-    END IF;
-
-    -- if table  is now empty, remove the table
-    IF _config #> array['tables', table_name] = '{}' THEN
-      SELECT _config #- array['tables', table_name] INTO _config;
-    END IF;
-
-    -- if config empty delete
-    -- or update the config
-    IF _config #> array['tables'] = '{}' THEN
-      DELETE FROM public.eql_v2_configuration WHERE state = 'pending';
-    ELSE
-      UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending';
+    -- update the config and migrate (even if empty)
+    UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending';
+    
+    IF NOT migrating THEN
+      PERFORM eql_v2.migrate_config();
+      PERFORM eql_v2.activate_config();
     END IF;
 
     -- exeunt
@@ -134,7 +124,7 @@ CREATE FUNCTION eql_v2.modify_search_config(table_name text, column_name text, i
   RETURNS jsonb
 AS $$
   BEGIN
-    PERFORM eql_v2.remove_search_config(table_name, column_name, index_name);
+    PERFORM eql_v2.remove_search_config(table_name, column_name, index_name, migrating);
     RETURN eql_v2.add_search_config(table_name, column_name, index_name, cast_as, opts, migrating);
   END;
 $$ LANGUAGE plpgsql;
@@ -293,16 +283,11 @@ AS $$
       SELECT _config #- array['tables', table_name] INTO _config;
     END IF;
 
-    -- if config empty delete
-    -- or update the config
-    IF _config #> array['tables'] = '{}' THEN
-      DELETE FROM public.eql_v2_configuration WHERE state = 'pending';
-    ELSE
-      UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending';
-    END IF;
-
     PERFORM eql_v2.remove_encrypted_constraint(table_name, column_name);
 
+    -- update the config and migrate (even if empty)
+    UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending';
+    
     IF NOT migrating THEN
       PERFORM eql_v2.migrate_config();
       PERFORM eql_v2.activate_config();
diff --git a/tasks/postgres.toml b/tasks/postgres.toml
@@ -21,6 +21,11 @@ mise run postgres:up --extra-args "--detach --wait"
 ["postgres:psql"]
 description = "Run psql"
 run = """
-{% set default_service = "postgres-" ~ get_env(name="POSTGRES_VERSION",default="17") %}
-psql -U {{arg(name="user",default="cipherstash")}} -d {{arg(name="db",default="cipherstash")}} -h localhost -p {{arg(name="port",default="7432")}} --service {{arg(name="service",default=default_service)}}
+psql -U {{arg(name="user",default="cipherstash")}} -d {{arg(name="db",default="cipherstash")}} -h localhost -p {{arg(name="port",default="7432")}}
+"""
+
+["eql:install"]
+description = "Install EQL to local postgres"
+run = """
+psql -U {{arg(name="user",default="cipherstash")}} -d {{arg(name="db",default="cipherstash")}} -h localhost -p {{arg(name="port",default="7432")}} -f {{arg(name="file",default="release/cipherstash-encrypt.sql")}}
 """
