diff --git a/README.md b/README.md index b816e14..86d6761 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ When inserting data into the encrypted column, wrap the plaintext in the appropr ```sql INSERT INTO users (encrypted_email) VALUES ( - '{"v":1,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"}}' + '{"v":2,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"}}' ); ``` @@ -151,7 +151,7 @@ Data is stored in the PostgreSQL database as: "bf": null, "ob": null, "u": null, - "v": 1 + "v": 2 } ``` @@ -175,7 +175,7 @@ Data is returned as: "t": "users", "c": "encrypted_email" }, - "v": 1, + "v": 2, "q": null } ``` @@ -249,7 +249,7 @@ SELECT eql_v2.add_search_config( ```sql SELECT * FROM users WHERE eql_v2.hmac_256(encrypted_email) = eql_v2.hmac_256( - '{"v":1,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"},"q":"hmac_256"}' + '{"v":2,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"},"q":"hmac_256"}' ); ``` @@ -280,7 +280,7 @@ SELECT eql_v2.add_search_config( ```sql SELECT * FROM users WHERE eql_v2.bloom_filter(encrypted_email) @> eql_v2.bloom_filter( - '{"v":1,"k":"pt","p":"test","i":{"t":"users","c":"encrypted_email"},"q":"match"}' + '{"v":2,"k":"pt","p":"test","i":{"t":"users","c":"encrypted_email"},"q":"match"}' ); ``` @@ -302,7 +302,7 @@ Enable range queries on encrypted data using the `eql_v2.ore_block_u64_8_256`, ` ```sql SELECT * FROM users WHERE eql_v2.ore_block_u64_8_256(encrypted_date) < eql_v2.ore_block_u64_8_256( - '{"v":1,"k":"pt","p":"2023-10-05","i":{"t":"users","c":"encrypted_date"},"q":"ore"}' + '{"v":2,"k":"pt","p":"2023-10-05","i":{"t":"users","c":"encrypted_date"},"q":"ore"}' ); ``` diff --git a/docs/reference/JSON.md b/docs/reference/JSON.md index 3f284ab..adee765 100644 --- a/docs/reference/JSON.md +++ b/docs/reference/JSON.md @@ -70,7 +70,7 @@ The EQL payload would be: ```sql INSERT INTO users (encrypted_json) VALUES ( - '{"v":1,"k":"pt","p":"{\"name\":\"John Doe\",\"metadata\":{\"age\":42}}","i":{"t":"users","c":"encrypted_json"}}' + '{"v":2,"k":"pt","p":"{\"name\":\"John Doe\",\"metadata\":{\"age\":42}}","i":{"t":"users","c":"encrypted_json"}}' ); ``` @@ -83,7 +83,7 @@ Data is stored in the database as: "t": "users" }, "k": "sv", - "v": 1, + "v": 2, "sv": [["ciphertext"]] } ``` @@ -108,7 +108,7 @@ Data is returned as: "t": "users", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": null } ``` @@ -141,7 +141,7 @@ We can query records that contain a specific structure. SELECT * FROM examples WHERE cs_ste_vec_v2(encrypted_json) @> cs_ste_vec_v2( '{ - "v":1, + "v":2, "k":"pt", "p":{"top":{"nested":["a"]}}, "i":{"t":"examples","c":"encrypted_json"}, @@ -169,7 +169,7 @@ If we query for a value that does not exist in the data: SELECT * FROM examples WHERE cs_ste_vec_v2(encrypted_json) @> cs_ste_vec_v2( '{ - "v":1, + "v":2, "k":"pt", "p":{"top":{"nested":["d"]}}, "i":{"t":"examples","c":"encrypted_json"}, @@ -203,7 +203,7 @@ We can extract the value of the `"top"` key. ```sql SELECT cs_ste_vec_value_v2(encrypted_json, '{ - "v":1, + "v":2, "k":"pt", "p":"$.top", "i":{"t":"examples","c":"encrypted_json"}, @@ -250,7 +250,7 @@ We can query records where the `"num"` field is greater than `2`. SELECT * FROM examples WHERE cs_ste_vec_term_v2(encrypted_json, '{ - "v":1, + "v":2, "k":"pt", "p":"$.num", "i":{"t":"examples","c":"encrypted_json"}, @@ -258,7 +258,7 @@ WHERE cs_ste_vec_term_v2(encrypted_json, }' ) > cs_ste_vec_term_v2( '{ - "v":1, + "v":2, "k":"pt", "p":"2", "i":{"t":"examples","c":"encrypted_json"}, @@ -298,7 +298,7 @@ We can group the data by the `"color"` field and count occurrences. ```sql SELECT cs_grouped_value_v2(cs_ste_vec_value_v2(encrypted_json, '{ - "v":1, + "v":2, "k":"pt", "p":"$.color", "i":{"t":"examples","c":"encrypted_json"}, @@ -308,7 +308,7 @@ SELECT cs_grouped_value_v2(cs_ste_vec_value_v2(encrypted_json, FROM examples GROUP BY cs_ste_vec_term_v2(encrypted_json, '{ - "v":1, + "v":2, "k":"pt", "p":"$.color", "i":{"t":"examples","c":"encrypted_json"}, @@ -408,7 +408,7 @@ SELECT cs_ste_vec_value_v2(encrypted_json, $1) FROM examples; "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ejson_path" } ``` @@ -436,7 +436,7 @@ WHERE cs_ste_vec_term_v2(examples.encrypted_json, $1) > cs_ste_vec_term_v2($2) "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ejson_path" } @@ -448,7 +448,7 @@ WHERE cs_ste_vec_term_v2(examples.encrypted_json, $1) > cs_ste_vec_term_v2($2) "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ste_vec" } ``` @@ -495,7 +495,7 @@ With the params: "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ejson_path" } ``` @@ -511,7 +511,7 @@ Would return the EQL plaintext payload with an array (`[1, 2, 3]` for example): "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": null } ``` @@ -547,7 +547,7 @@ WHERE (cs_ste_vec_terms_v2(examples.encrypted_json, $1))[1] > cs_ste_vec_term_v2 "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ejson_path" } @@ -559,7 +559,7 @@ WHERE (cs_ste_vec_terms_v2(examples.encrypted_json, $1))[1] > cs_ste_vec_term_v2 "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ste_vec" } ``` @@ -607,7 +607,7 @@ SELECT cs_ste_vec_value_v2(encrypted_json, $1) FROM examples; "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ejson_path" } ``` @@ -637,7 +637,7 @@ WHERE cs_ste_vec_term_v2(examples.encrypted_json, $1) > cs_ste_vec_term_v2($2) "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ejson_path" } @@ -649,7 +649,7 @@ WHERE cs_ste_vec_term_v2(examples.encrypted_json, $1) > cs_ste_vec_term_v2($2) "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ste_vec" } ``` @@ -700,7 +700,7 @@ Example params: "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ste_vec" } ``` @@ -755,7 +755,7 @@ With the params: "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ejson_path" } ``` @@ -771,7 +771,7 @@ Would return the EQL plaintext payload with an array (`[1, 2, 3]` for example): "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": null } ``` @@ -808,7 +808,7 @@ WHERE EXISTS ( "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ejson_path" } @@ -820,7 +820,7 @@ WHERE EXISTS ( "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ste_vec" } ``` @@ -874,7 +874,7 @@ Example data and params: "t": "examples", "c": "encrypted_json" }, - "v": 1, + "v": 2, "q": "ejson_path" } ``` diff --git a/docs/reference/PAYLOAD.md b/docs/reference/PAYLOAD.md index e3914c5..933c305 100644 --- a/docs/reference/PAYLOAD.md +++ b/docs/reference/PAYLOAD.md @@ -8,7 +8,7 @@ The plaintext json payload that is sent from the client to CipherStash Proxy in ```json { - "v": 1, + "v": 2, "k": "pt", "p": "plaintext value", "i": { @@ -25,7 +25,7 @@ CipherStash Proxy will handle the plaintext payload and create the encrypted pay ```json { - "v": 1, + "v": 2, "k": "ct", "c": "ciphertext value", "i": { diff --git a/docs/tutorials/GETTINGSTARTED.md b/docs/tutorials/GETTINGSTARTED.md index e891fff..bd02c06 100644 --- a/docs/tutorials/GETTINGSTARTED.md +++ b/docs/tutorials/GETTINGSTARTED.md @@ -201,7 +201,7 @@ After adding these indexes, our `eql_v2_configuration` table will look like this ```bash id | 1 state | pending -data | {"v": 1, "tables": {"users": {"email_encrypted": {"cast_as": "text", "indexes": {"ore": {}, "match": {"k": 6, "bf": 2048, "tokenizer": {"kind": "ngram", "token_length": 3}, "token_filters": [{"kind": "downcase"}], "include_original": true}, "unique": {"token_filters": [{"kind": "downcase"}]}}}}}} +data | {"v": 2, "tables": {"users": {"email_encrypted": {"cast_as": "text", "indexes": {"ore": {}, "match": {"k": 6, "bf": 2048, "tokenizer": {"kind": "ngram", "token_length": 3}, "token_filters": [{"kind": "downcase"}], "include_original": true}, "unique": {"token_filters": [{"kind": "downcase"}]}}}}}} ``` The initial `state` will be set as pending. @@ -218,7 +218,7 @@ The `cs_configured_v2` table will now have a state of `active`. ```bash id | 1 state | active -data | {"v": 1, "tables": {"users": {"email_encrypted": {"cast_as": "text", "indexes": {"ore": {}, "match": {"k": 6, "bf": 2048, "tokenizer": {"kind": "ngram", "token_length": 3}, "token_filters": [{"kind": "downcase"}], "include_original": true}, "unique": {"token_filters": [{"kind": "downcase"}]}}}}}} +data | {"v": 2, "tables": {"users": {"email_encrypted": {"cast_as": "text", "indexes": {"ore": {}, "match": {"k": 6, "bf": 2048, "tokenizer": {"kind": "ngram", "token_length": 3}, "token_filters": [{"kind": "downcase"}], "include_original": true}, "unique": {"token_filters": [{"kind": "downcase"}]}}}}}} ``` ### Encrypting existing plaintext data @@ -292,7 +292,7 @@ An EQL payload will look like this: "t": "users", // The table "c": "email_encrypted" // The encrypted column }, - "v": 1, + "v": 2, "q": null // Used in queries only. } ``` @@ -308,7 +308,7 @@ INSERT INTO users (email) VALUES ('test@test.com'); The equivalent of this query to insert a plaintext email and encrypt it into the `email_encrypted` column using EQL: ```sql -INSERT INTO users (email_encrypted) VALUES ('{"v":1,"k":"pt","p":"test@test.com","i":{"t":"users","c":"email_encrypted"}}'); +INSERT INTO users (email_encrypted) VALUES ('{"v":2,"k":"pt","p":"test@test.com","i":{"t":"users","c":"email_encrypted"}}'); ``` **What is happening?** @@ -328,7 +328,7 @@ It creates an EQL payload that looks similar to this and inserts this into the e "bf": [42], // The ciphertext used for free text queries i.e match index "u": "unique ciphertext", // The ciphertext used for unique queries. i.e unique index "ob": ["a", "b", "c"], // The ciphertext used for order or comparison queries. i.e ore index - "v": 1 + "v": 2 } ``` @@ -368,10 +368,10 @@ Returns: ```bash email_encrypted ------------------------------------------------------------------------------------------------- - {"k":"pt","p":"adalovelace@example.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} - {"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} - {"k":"pt","p":"edithclarke@email.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} - {"k":"pt","p":"test@test.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} + {"k":"pt","p":"adalovelace@example.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} + {"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} + {"k":"pt","p":"edithclarke@email.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} + {"k":"pt","p":"test@test.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} ``` **What is happening?** @@ -389,7 +389,7 @@ The json stored in the database looks similar to this: "bf": [42], // The ciphertext used for free text queries i.e match index "u": "unique ciphertext", // The ciphertext used for unique queries. i.e unique index "ob": ["a", "b", "c"], // The ciphertext used for order or comparison queries. i.e ore index - "v": 1 + "v": 2 } ``` @@ -403,7 +403,7 @@ The Proxy decrypts the json above and returns a plaintext json payload that look "t": "users", "c": "email_encrypted" }, - "v": 1, + "v": 2, "q": null } ``` @@ -435,7 +435,7 @@ EQL query payload for a match query: "t": "users", "c": "email_encrypted" }, - "v": 1, + "v": 2, "q": "match" // This field is required on queries. This specifies the type of query we are executing. } ``` @@ -450,7 +450,7 @@ The EQL equivalent of this query is: ```sql SELECT * FROM users WHERE cs_match_v2(email_encrypted) @> cs_match_v2( - '{"v":1,"k":"pt","p":"grace","i":{"t":"users","c":"email_encrypted"},"q":"match"}' + '{"v":2,"k":"pt","p":"grace","i":{"t":"users","c":"email_encrypted"},"q":"match"}' ); ``` @@ -458,7 +458,7 @@ This query returns: | id | email_encrypted | | --- | -------------------------------------------------------------------------------------------- | -| 2 | {"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} | +| 2 | {"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} | #### Equality query @@ -478,7 +478,7 @@ EQL query payload for a match query: "t": "users", "c": "email_encrypted" }, - "v": 1, + "v": 2, "q": "unique" // This field is required on queries. This specifies the type of query we are executing. } ``` @@ -493,7 +493,7 @@ The EQL equivalent of this query is: ```sql SELECT * FROM users WHERE cs_unique_v2(email_encrypted) = cs_unique_v2( - '{"v":1,"k":"pt","p":"adalovelace@example.com","i":{"t":"users","c":"email_encrypted"},"q":"unique"}' + '{"v":2,"k":"pt","p":"adalovelace@example.com","i":{"t":"users","c":"email_encrypted"},"q":"unique"}' ); ``` @@ -501,7 +501,7 @@ This query returns: | id | email_encrypted | | --- | ----------------------------------------------------------------------------------------------- | -| 1 | {"k":"pt","p":"adalovelace@example.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} | +| 1 | {"k":"pt","p":"adalovelace@example.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} | #### Order by query @@ -527,10 +527,10 @@ This query returns: | id | email_encrypted | | --- | ----------------------------------------------------------------------------------------------- | -| 1 | {"k":"pt","p":"adalovelace@example.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} | -| 3 | {"k":"pt","p":"edithclarke@email.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} | -| 2 | {"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} | -| 4 | {"k":"pt","p":"test@test.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} | +| 1 | {"k":"pt","p":"adalovelace@example.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} | +| 3 | {"k":"pt","p":"edithclarke@email.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} | +| 2 | {"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} | +| 4 | {"k":"pt","p":"test@test.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} | #### Comparison query @@ -550,7 +550,7 @@ EQL query payload for a comparison query: "t": "users", "c": "email_encrypted" }, - "v": 1, + "v": 2, "q": "ore" // This field is required on queries. This specifies the type of query we are executing. } ``` @@ -565,7 +565,7 @@ The EQL equivalent of this query is: ```sql SELECT * FROM users WHERE ore_block_u64_8_256(email_encrypted) > ore_block_u64_8_256( - '{"v":1,"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"q":"ore"}' + '{"v":2,"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"q":"ore"}' ); ``` @@ -573,7 +573,7 @@ This query returns: | id | email_encrypted | | --- | ------------------------------------------------------------------------------------- | -| 4 | {"k":"pt","p":"test@test.com","i":{"t":"users","c":"email_encrypted"},"v":1,"q":null} | +| 4 | {"k":"pt","p":"test@test.com","i":{"t":"users","c":"email_encrypted"},"v":2,"q":null} | #### Summary diff --git a/src/encrypted/aggregates_test.sql b/src/encrypted/aggregates_test.sql index b6dcfe8..3e9670d 100644 --- a/src/encrypted/aggregates_test.sql +++ b/src/encrypted/aggregates_test.sql @@ -53,7 +53,7 @@ $$ LANGUAGE plpgsql; INSERT INTO agg_test (plain_int, enc_int) VALUES ( 3, - '{"c": "mBbLa7Cm?&jvpfcv1d3hep>s)76qzUbwUky&M&C3mjDG_os-_y0MRaMGl@&p#AOuusN|3Lu=mBCcg_V{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "bf": null, "v": 1}'::jsonb::eql_v2_encrypted + '{"c": "mBbLa7Cm?&jvpfcv1d3hep>s)76qzUbwUky&M&C3mjDG_os-_y0MRaMGl@&p#AOuusN|3Lu=mBCcg_V{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "bf": null, "v": 2}'::jsonb::eql_v2_encrypted ); -- run exceptional case diff --git a/src/encrypted/constraints.sql b/src/encrypted/constraints.sql index 6b02d06..8da1600 100644 --- a/src/encrypted/constraints.sql +++ b/src/encrypted/constraints.sql @@ -34,6 +34,12 @@ CREATE FUNCTION eql_v2._encrypted_check_v(val jsonb) AS $$ BEGIN IF (val ? 'v') THEN + + IF val->>'v' <> '2' THEN + RAISE 'Expected encrypted column version (v) 2'; + RETURN false; + END IF; + RETURN true; END IF; RAISE 'Encrypted column missing version (v) field: %', val; diff --git a/src/encrypted/constraints_test.sql b/src/encrypted/constraints_test.sql index 1ca4775..0dc88e5 100644 --- a/src/encrypted/constraints_test.sql +++ b/src/encrypted/constraints_test.sql @@ -43,6 +43,37 @@ DO $$ $$ LANGUAGE plpgsql; +-- EQL version is enforced +DO $$ + DECLARE + e eql_v2_encrypted; + BEGIN + + -- reset data + PERFORM create_table_with_encrypted(); + -- remove the version field + e := create_encrypted_json(1)::jsonb-'v'; + PERFORM assert_exception( + 'Insert with missing version fails', + format('INSERT INTO encrypted (e) VALUES (%s::jsonb::eql_v2_encrypted) RETURNING id', e)); + + -- set version to 1 + e := create_encrypted_json(1)::jsonb || '{"v": 1}'; + + PERFORM assert_exception( + 'Insert with invalid version fails', + format('INSERT INTO encrypted (e) VALUES (%s::jsonb::eql_v2_encrypted) RETURNING id', e)); + + -- set version to 1 + e := create_encrypted_json(1); + + PERFORM assert_result( + 'Insert with valid version is ok', + format('INSERT INTO encrypted (e) VALUES (%L) RETURNING id', e)); + + + END; +$$ LANGUAGE plpgsql; diff --git a/tasks/test.sh b/tasks/test.sh index c86909c..eb68300 100755 --- a/tasks/test.sh +++ b/tasks/test.sh @@ -38,10 +38,6 @@ fail_if_postgres_not_running mise run build --force mise run reset --force --postgres ${POSTGRES_VERSION} -echo '/////////////////////////////////////////////////////////' -cat release/cipherstash-encrypt.sql -echo '/////////////////////////////////////////////////////////' - # Install # cat release/cipherstash-encrypt.sql | docker exec -i ${container_name} psql ${connection_url} -f- diff --git a/tests/test_helpers.sql b/tests/test_helpers.sql index 7f04df0..ed514e8 100644 --- a/tests/test_helpers.sql +++ b/tests/test_helpers.sql @@ -319,7 +319,8 @@ AS $$ }, "hm": "unique.%s", "b3": "blake3.%s", - "bf": %s + "bf": %s, + "v": 2 }', random_key, random_val,