From da5072f9d07ad8f5abd1185661079111c0e419bb Mon Sep 17 00:00:00 2001 From: James Sadler Date: Tue, 29 Jul 2025 16:57:26 +1000 Subject: [PATCH 1/2] fix: make eql_v2.compare robust against null search terms ProtectJS was generating null search terms and EQL was only checking for the presence of an index *key*, which would then end up checking if two JSONB null values were equal, which would always return true. This commit changes checks of the form: ``` val ? '' ``` to ``` val ->> '' IS NOT NULL ``` --- src/blake3/functions.sql | 4 +- src/bloom_filter/functions.sql | 4 +- src/encrypted/aggregates_test.sql | 45 ---- src/hmac_256/functions.sql | 4 +- src/ore_block_u64_8_256/functions.sql | 4 +- src/ore_cllw_u64_8/functions.sql | 8 +- src/ore_cllw_u64_8/operator_class.sql.skip | 20 -- src/ore_cllw_u64_8/operators_test.sql.skip | 234 --------------------- src/ore_cllw_var_8/functions.sql | 8 +- 9 files changed, 12 insertions(+), 319 deletions(-) delete mode 100644 src/ore_cllw_u64_8/operator_class.sql.skip delete mode 100644 src/ore_cllw_u64_8/operators_test.sql.skip diff --git a/src/blake3/functions.sql b/src/blake3/functions.sql index b88db5d..f44321b 100644 --- a/src/blake3/functions.sql +++ b/src/blake3/functions.sql @@ -14,7 +14,7 @@ AS $$ RETURN NULL; END IF; - IF NOT (val ? 'b3') THEN + IF NOT eql_v2.has_blake3(val) THEN RAISE 'Expected a blake3 index (b3) value in json: %', val; END IF; @@ -44,7 +44,7 @@ CREATE FUNCTION eql_v2.has_blake3(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN - RETURN val ? 'b3'; + RETURN val ->> 'b3' IS NOT NULL; END; $$ LANGUAGE plpgsql; diff --git a/src/bloom_filter/functions.sql b/src/bloom_filter/functions.sql index 69e51dc..e6f5f4f 100644 --- a/src/bloom_filter/functions.sql +++ b/src/bloom_filter/functions.sql @@ -12,7 +12,7 @@ AS $$ RETURN NULL; END IF; - IF val ? 'bf' THEN + IF eql_v2.has_bloom_filter(val) THEN RETURN ARRAY(SELECT jsonb_array_elements(val->'bf'))::eql_v2.bloom_filter; END IF; @@ -38,7 +38,7 @@ CREATE FUNCTION eql_v2.has_bloom_filter(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN - RETURN val ? 'bf'; + RETURN val ->> 'bf' IS NOT NULL; END; $$ LANGUAGE plpgsql; diff --git a/src/encrypted/aggregates_test.sql b/src/encrypted/aggregates_test.sql index 3e9670d..68bd4ea 100644 --- a/src/encrypted/aggregates_test.sql +++ b/src/encrypted/aggregates_test.sql @@ -48,48 +48,3 @@ DO $$ ASSERT ((SELECT enc_int FROM agg_test WHERE plain_int = 5) = (SELECT eql_v2.max(enc_int) FROM agg_test)); END; $$ LANGUAGE plpgsql; - --- insert data without "ob" (ore index value) -INSERT INTO agg_test (plain_int, enc_int) VALUES -( - 3, - '{"c": "mBbLa7Cm?&jvpfcv1d3hep>s)76qzUbwUky&M&C3mjDG_os-_y0MRaMGl@&p#AOuusN|3Lu=mBCcg_V{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "bf": null, "v": 2}'::jsonb::eql_v2_encrypted -); - --- run exceptional case -DO $$ - DECLARE - error_message text; - BEGIN - -- min enc_int raises exception - SELECT eql_v2.min(enc_int) FROM agg_test; - EXCEPTION - WHEN others THEN - error_message := SQLERRM; - - IF error_message LIKE '%' || 'Expected an ore index (ob) value in json' || '%' THEN - ASSERT true; - ELSE - RAISE EXCEPTION 'Unexpected exception: %', error_message; - END IF; - END; -$$ LANGUAGE plpgsql; - --- run exceptional case -DO $$ - DECLARE - error_message text; - BEGIN - -- max enc_int raises exception - SELECT eql_v2.max(enc_int) FROM agg_test; - EXCEPTION - WHEN others THEN - error_message := SQLERRM; - - IF error_message LIKE '%' || 'Expected an ore index (ob) value in json' || '%' THEN - ASSERT true; - ELSE - RAISE EXCEPTION 'Unexpected exception: %', error_message; - END IF; - END; -$$ LANGUAGE plpgsql; diff --git a/src/hmac_256/functions.sql b/src/hmac_256/functions.sql index 4cf0a94..5888a5d 100644 --- a/src/hmac_256/functions.sql +++ b/src/hmac_256/functions.sql @@ -12,7 +12,7 @@ AS $$ RETURN NULL; END IF; - IF val ? 'hm' THEN + IF eql_v2.has_hmac_256(val) THEN RETURN val->>'hm'; END IF; RAISE 'Expected a hmac_256 index (hm) value in json: %', val; @@ -25,7 +25,7 @@ CREATE FUNCTION eql_v2.has_hmac_256(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN - RETURN val ? 'hm'; + RETURN val ->> 'hm' IS NOT NULL; END; $$ LANGUAGE plpgsql; diff --git a/src/ore_block_u64_8_256/functions.sql b/src/ore_block_u64_8_256/functions.sql index 2769367..9716c08 100644 --- a/src/ore_block_u64_8_256/functions.sql +++ b/src/ore_block_u64_8_256/functions.sql @@ -57,7 +57,7 @@ AS $$ RETURN NULL; END IF; - IF val ? 'ob' THEN + IF eql_v2.has_ore_block_u64_8_256(val) THEN RETURN eql_v2.jsonb_array_to_ore_block_u64_8_256(val->'ob'); END IF; RAISE 'Expected an ore index (ob) value in json: %', val; @@ -85,7 +85,7 @@ CREATE FUNCTION eql_v2.has_ore_block_u64_8_256(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN - RETURN val ? 'ob'; + RETURN val ->> 'ob' IS NOT NULL; END; $$ LANGUAGE plpgsql; diff --git a/src/ore_cllw_u64_8/functions.sql b/src/ore_cllw_u64_8/functions.sql index 3732f7c..00b74d3 100644 --- a/src/ore_cllw_u64_8/functions.sql +++ b/src/ore_cllw_u64_8/functions.sql @@ -17,14 +17,10 @@ AS $$ RETURN NULL; END IF; - IF NOT (val ? 'ocf') THEN + IF NOT (eql_v2.has_ore_cllw_u64_8(val)) THEN RAISE 'Expected a ore_cllw_u64_8 index (ocf) value in json: %', val; END IF; - IF val->>'ocf' IS NULL THEN - RETURN NULL; - END IF; - RETURN ROW(decode(val->>'ocf', 'hex')); END; $$ LANGUAGE plpgsql; @@ -47,7 +43,7 @@ CREATE FUNCTION eql_v2.has_ore_cllw_u64_8(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN - RETURN val ? 'ocf'; + RETURN val ->> 'ocf' IS NOT NULL; END; $$ LANGUAGE plpgsql; diff --git a/src/ore_cllw_u64_8/operator_class.sql.skip b/src/ore_cllw_u64_8/operator_class.sql.skip deleted file mode 100644 index 5567977..0000000 --- a/src/ore_cllw_u64_8/operator_class.sql.skip +++ /dev/null @@ -1,20 +0,0 @@ --- NOTE FILE IS DISABLED --- REPLACE `!REQUIRE` with `REQUIRE` to enable in the build - --- !REQUIRE: src/schema.sql --- !REQUIRE: src/ore_cllw_u64_8/types.sql --- !REQUIRE: src/ore_cllw_u64_8/functions.sql --- !REQUIRE: src/ore_cllw_u64_8/operators.sql - - - -CREATE OPERATOR FAMILY eql_v2.ore_cllw_u64_8_btree_ops USING btree; - - -CREATE OPERATOR CLASS eql_v2.ore_cllw_u64_8_btree_ops DEFAULT FOR TYPE eql_v2.ore_cllw_u64_8 USING btree FAMILY eql_v2.ore_cllw_u64_8_btree_ops AS - OPERATOR 1 <, - OPERATOR 2 <=, - OPERATOR 3 =, - OPERATOR 4 >=, - OPERATOR 5 >, - FUNCTION 1 eql_v2.compare_ore_cllw_u64_8(a eql_v2.ore_cllw_u64_8, b eql_v2.ore_cllw_u64_8); diff --git a/src/ore_cllw_u64_8/operators_test.sql.skip b/src/ore_cllw_u64_8/operators_test.sql.skip deleted file mode 100644 index e90833e..0000000 --- a/src/ore_cllw_u64_8/operators_test.sql.skip +++ /dev/null @@ -1,234 +0,0 @@ -\set ON_ERROR_STOP on - -SELECT create_table_with_encrypted(); -SELECT seed_encrypted_json(); - - --- ------------------------------------------------------------------------ --- ------------------------------------------------------------------------ --- --- ore_cllw_u64_8 < ore_cllw_u64_8 --- --- Test data is '{"hello": "world", "n": 42}' - --- Paths --- $ -> bca213de9ccce676fa849ff9c4807963 --- $.hello -> a7cea93975ed8c01f861ccb6bd082784 --- $.n -> 2517068c0d1f9d4d41d2c666211f785e --- --- -DO $$ -DECLARE - sv eql_v2_encrypted; - term eql_v2_encrypted; - BEGIN - - -- json n: 20 - sv := get_numeric_ste_vec_20()::eql_v2_encrypted; - -- extract the term at $.n - term := sv->'2517068c0d1f9d4d41d2c666211f785e'::text; - - PERFORM eql_v2.log('term', term::text); - - -- -- -- -- $.n - PERFORM assert_result( - format('eql_v2_encrypted < eql_v2_encrypted with ore_cllw_u64_8 index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) < eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - PERFORM assert_count( - format('eql_v2_encrypted < eql_v2_encrypted with ore index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) < eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), - 1); - - -- Check the $.hello path - -- Returned encrypted does not have ore_cllw_u64_8 - PERFORM assert_exception( - format('eql_v2_encrypted < eql_v2_encrypted with ore index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''a7cea93975ed8c01f861ccb6bd082784''::text) < eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - END; -$$ LANGUAGE plpgsql; - - - - --- ------------------------------------------------------------------------ --- ------------------------------------------------------------------------ --- --- ore_cllw_u64_8 <= ore_cllw_u64_8 --- --- Test data is '{"hello": "world", "n": 42}' - --- Paths --- $ -> bca213de9ccce676fa849ff9c4807963 --- $.hello -> a7cea93975ed8c01f861ccb6bd082784 --- $.n -> 2517068c0d1f9d4d41d2c666211f785e --- --- -DO $$ -DECLARE - sv eql_v2_encrypted; - term eql_v2_encrypted; - BEGIN - - -- json n: 20 - sv := get_numeric_ste_vec_20()::eql_v2_encrypted; - - -- extract the term at $.n - term := sv->'2517068c0d1f9d4d41d2c666211f785e'::text; - - -- -- -- -- $.n - PERFORM assert_result( - format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_u64_8 index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) <= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - PERFORM assert_count( - format('eql_v2_encrypted <= eql_v2_encrypted with ore index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) <= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), - 2); - - -- Check the $.hello path - -- Returned encrypted does not have ore_cllw_u64_8 - PERFORM assert_exception( - format('eql_v2_encrypted <= eql_v2_encrypted with ore index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''a7cea93975ed8c01f861ccb6bd082784''::text) <= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - END; -$$ LANGUAGE plpgsql; - - - --- ------------------------------------------------------------------------ --- ------------------------------------------------------------------------ --- --- ore_cllw_u64_8 >= ore_cllw_u64_8 --- --- Test data is '{"hello": "world", "n": 42}' - --- Paths --- $ -> bca213de9ccce676fa849ff9c4807963 --- $.hello -> a7cea93975ed8c01f861ccb6bd082784 --- $.n -> 2517068c0d1f9d4d41d2c666211f785e --- --- -DO $$ -DECLARE - sv eql_v2_encrypted; - term eql_v2_encrypted; - BEGIN - - -- json n: 30 - sv := get_numeric_ste_vec_30()::eql_v2_encrypted; - - -- extract the term at $.n - term := sv->'2517068c0d1f9d4d41d2c666211f785e'::text; - - -- -- -- -- $.n - PERFORM assert_result( - format('eql_v2_encrypted >= eql_v2_encrypted with ore_cllw_u64_8 index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) >= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - PERFORM assert_count( - format('eql_v2_encrypted >= eql_v2_encrypted with ore index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) >= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), - 1); - - -- Check the $ path - -- Returned encrypted does not have ore_cllw_u64_8 - PERFORM assert_exception( - format('eql_v2_encrypted >= eql_v2_encrypted with ore index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''bca213de9ccce676fa849ff9c4807963''::text) >= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - END; -$$ LANGUAGE plpgsql; - - - --- ------------------------------------------------------------------------ --- ------------------------------------------------------------------------ --- --- ore_cllw_u64_8 > ore_cllw_u64_8 --- --- Test data is '{"hello": "world", "n": 42}' - --- Paths --- $ -> bca213de9ccce676fa849ff9c4807963 --- $.hello -> a7cea93975ed8c01f861ccb6bd082784 --- $.n -> 2517068c0d1f9d4d41d2c666211f785e --- --- -DO $$ -DECLARE - sv eql_v2_encrypted; - term eql_v2_encrypted; - BEGIN - - -- json n: 20 - sv := get_numeric_ste_vec_20()::eql_v2_encrypted; - - -- extract the term at $.n - term := sv->'2517068c0d1f9d4d41d2c666211f785e'::text; - - -- -- -- -- $.n - PERFORM assert_result( - format('eql_v2_encrypted > eql_v2_encrypted with ore_cllw_u64_8 index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) > eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - PERFORM assert_count( - format('eql_v2_encrypted > eql_v2_encrypted with ore index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) > eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), - 1); - - -- Check the $ path - -- Returned encrypted does not have ore_cllw_u64_8 - PERFORM assert_exception( - format('eql_v2_encrypted > eql_v2_encrypted with ore index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''bca213de9ccce676fa849ff9c4807963''::text) > eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - END; -$$ LANGUAGE plpgsql; - - - - --- -- ------------------------------------------------------------------------ --- -- ------------------------------------------------------------------------ --- -- --- -- ore_cllw_u64_8 = ore_cllw_u64_8 --- -- --- -- Test data is '{"hello": "world", "n": 42}' - --- -- Paths --- -- $ -> bca213de9ccce676fa849ff9c4807963 --- -- $.hello -> a7cea93975ed8c01f861ccb6bd082784 --- -- $.n -> 2517068c0d1f9d4d41d2c666211f785e --- -- --- -- -DO $$ -DECLARE - sv eql_v2_encrypted; - term eql_v2_encrypted; - BEGIN - - -- json n: 20 - sv := get_numeric_ste_vec_10()::eql_v2_encrypted; - -- extract the term at $.n - term := sv->'2517068c0d1f9d4d41d2c666211f785e'::text; - - PERFORM assert_result( - format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) = eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - PERFORM assert_count( - format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e''::text) = eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), - 1); - - -- Check the $.n path - -- Returned encrypted does not have ore_cllw_u64_8 and raises exception - PERFORM assert_exception( - format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), - format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''a7cea93975ed8c01f861ccb6bd082784''::text) = eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); - - END; -$$ LANGUAGE plpgsql; diff --git a/src/ore_cllw_var_8/functions.sql b/src/ore_cllw_var_8/functions.sql index 483d7c9..4adbf0c 100644 --- a/src/ore_cllw_var_8/functions.sql +++ b/src/ore_cllw_var_8/functions.sql @@ -17,14 +17,10 @@ AS $$ RETURN NULL; END IF; - IF NOT (val ? 'ocv') THEN + IF NOT (eql_v2.has_ore_cllw_var_8(val)) THEN RAISE 'Expected a ore_cllw_var_8 index (ocv) value in json: %', val; END IF; - IF val->>'ocv' IS NULL THEN - RETURN NULL; - END IF; - RETURN ROW(decode(val->>'ocv', 'hex')); END; $$ LANGUAGE plpgsql; @@ -47,7 +43,7 @@ CREATE FUNCTION eql_v2.has_ore_cllw_var_8(val jsonb) IMMUTABLE STRICT PARALLEL SAFE AS $$ BEGIN - RETURN val ? 'ocv'; + RETURN val ->> 'ocv' IS NOT NULL; END; $$ LANGUAGE plpgsql; From 739e186b930bda8fbd2f9f693c6caf86d29878e0 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 29 Jul 2025 20:21:23 +1000 Subject: [PATCH 2/2] test: test for handling of null index terms --- src/operators/compare_test.sql | 43 +++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/operators/compare_test.sql b/src/operators/compare_test.sql index 63c3dbe..c55c1b3 100644 --- a/src/operators/compare_test.sql +++ b/src/operators/compare_test.sql @@ -163,4 +163,45 @@ DO $$ ASSERT eql_v2.compare(c, b) = 1; ASSERT eql_v2.compare(c, a) = 1; END; -$$ LANGUAGE plpgsql; \ No newline at end of file +$$ LANGUAGE plpgsql; + + +-- +-- Compare hmac_256 when record has a `null` index of higher precedence +-- TEST COVERAGE FOR BUG FIX +-- +-- ORE Block indexes `ob` are used in compare before hmac_256 indexes. +-- If the index term is null `{"ob": null}` it should not be used +-- Comparing two `null` values is evaluated as equality and hilarity ensues +-- + +DO $$ + DECLARE + a eql_v2_encrypted; + b eql_v2_encrypted; + c eql_v2_encrypted; + BEGIN + -- generate with `hm` index + a := create_encrypted_json(1, 'hm'); + -- append `null` index + a := '{"ob": null}'::jsonb || a::jsonb; + + b := create_encrypted_json(2, 'hm'); + b := '{"ob": null}'::jsonb || b::jsonb; + + c := create_encrypted_json(3, 'hm'); + c := '{"ob": null}'::jsonb || c::jsonb; + + ASSERT eql_v2.compare(a, a) = 0; + ASSERT eql_v2.compare(a, b) = -1; + ASSERT eql_v2.compare(a, c) = -1; + + ASSERT eql_v2.compare(b, b) = 0; + ASSERT eql_v2.compare(b, a) = 1; + ASSERT eql_v2.compare(b, c) = -1; + + ASSERT eql_v2.compare(c, c) = 0; + ASSERT eql_v2.compare(c, b) = 1; + ASSERT eql_v2.compare(c, a) = 1; + END; +$$ LANGUAGE plpgsql;