diff --git a/.gitignore b/.gitignore index 55d2f40..4d80e70 100644 --- a/.gitignore +++ b/.gitignore @@ -206,4 +206,7 @@ release/ __pycache__ # dbdev -eql--*.sql \ No newline at end of file +eql--*.sql + +# git worktrees +.worktrees/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..4b1f6b6 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,951 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "eql-build" +version = "0.1.0" +dependencies = [ + "anyhow", + "eql-core", + "eql-postgres", +] + +[[package]] +name = "eql-core" +version = "0.1.0" +dependencies = [ + "paste", + "serde", + "serde_json", + "thiserror", + "tokio-postgres", +] + +[[package]] +name = "eql-postgres" +version = "0.1.0" +dependencies = [ + "eql-core", + "eql-test", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "eql-test" +version = "0.1.0" +dependencies = [ + "eql-core", + "serde_json", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "libredox" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +dependencies = [ + "bitflags", + "libc", + "redox_syscall", +] + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "mio" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "postgres-protocol" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4605b7c057056dd35baeb6ac0c0338e4975b1f2bef0f65da953285eb007095" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", + "serde_core", + "serde_json", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b40d66d9b2cfe04b628173409368e58247e8eddbbd3b0e6c6ba1d09f20f6c9e" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-util" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" + +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", + "web-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f25c863 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,18 @@ +[workspace] +members = [ + "eql-core", + "eql-postgres", + "eql-test", + "eql-build", +] + +resolver = "2" + +[workspace.dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.35", features = ["full"] } +tokio-postgres = { version = "0.7", features = ["with-serde_json-1"] } +anyhow = "1.0" +thiserror = "1.0" +paste = "1.0" diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3240ea9..d65cf84 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -318,6 +318,17 @@ This produces two SQL files in `releases/`: ## Structure +### Adding SQL + +When adding new SQL files to the project, follow these guidelines: + +- Never drop the configuration table as it may contain customer data and needs to live across EQL versions +- Everything else should have a `DROP IF EXISTS` +- Functions should be `DROP` and `CREATE`, instead of `CREATE OR REPLACE` + - Data types cannot be changed once created, so dropping first is more flexible +- Keep `DROP` and `CREATE` together in the code +- Types need to be dropped last, add to the `666-drop_types.sql` + ### Schema EQL is installed into the `eql_v2` PostgreSQL schema. diff --git a/README.md b/README.md index 9c47e12..262d32c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,10 @@ Store encrypted data alongside your existing data: - [Enable encrypted columns](#enable-encrypted-columns) - [Encrypt configuration](#encrypt-configuration) - [CipherStash integrations using EQL](#cipherstash-integrations-using-eql) -- [Developing](#developing) +- [Versioning](#versioning) + - [Upgrading](#upgrading) +- [Troubleshooting](#troubleshooting) +- [Contributing](#contributing) --- @@ -169,7 +172,7 @@ You can find the EQL extension on [dbdev's extension catalog](https://database.d ## Getting started -Once the custom types and functions are installed in your PostgreSQL database, you can start using EQL in your queries. +Once EQL is installed in your PostgreSQL database, you can start using encrypted columns in your tables. ### Enable encrypted columns @@ -178,12 +181,22 @@ Define encrypted columns using the `eql_v2_encrypted` type, which stores encrypt **Example:** ```sql +-- Step 1: Create a table with an encrypted column CREATE TABLE users ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, encrypted_email eql_v2_encrypted ); + +-- Step 2: Configure the column for encryption/decryption +SELECT eql_v2.add_column('users', 'encrypted_email', 'text'); + +-- Step 3: (Optional) Add search indexes +SELECT eql_v2.add_search_config('users', 'encrypted_email', 'unique', 'text'); ``` +> [!NOTE] +> You must use [CipherStash Proxy](https://github.com/cipherstash/proxy) or [Protect.js](https://github.com/cipherstash/protectjs) to encrypt and decrypt data. EQL provides the database functions and types, while these tools handle the actual cryptographic operations. + ## Encrypt configuration In order to enable searchable encryption, you will need to configure your CipherStash integration appropriately. @@ -232,6 +245,28 @@ To upgrade to the latest version of EQL, you can simply run the install script a Follow the instructions in the [dbdev documentation](https://database.dev/cipherstash/eql) to upgrade the extension to your desired version. -## Developing +## Troubleshooting + +### Common Errors + +**Error: "Some pending columns do not have an encrypted target"** +- **Cause**: Trying to configure a column that doesn't exist as `eql_v2_encrypted` type +- **Solution**: First create the column: `ALTER TABLE table_name ADD COLUMN column_name eql_v2_encrypted;` + +**Error: "Config exists for column: table_name column_name"** +- **Cause**: Attempting to add a column configuration that already exists +- **Solution**: Use `eql_v2.add_search_config()` to add indexes, or `eql_v2.remove_column()` first to reconfigure + +**Error: "No configuration exists for column: table_name column_name"** +- **Cause**: Trying to add search configuration before configuring the column +- **Solution**: Run `eql_v2.add_column()` first, then add search indexes + +### Getting Help + +- Check the [full documentation](./docs/README.md) +- Review [CipherStash Proxy configuration guide](./docs/tutorials/proxy-configuration.md) +- Report issues at [https://github.com/cipherstash/encrypt-query-language/issues](https://github.com/cipherstash/encrypt-query-language/issues) + +## Contributing -See the [development guide](./DEVELOPMENT.md). +See the [development guide](./DEVELOPMENT.md) for information on developing and extending EQL. diff --git a/docs/README.md b/docs/README.md index 9660ba4..d3b7912 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,9 +8,11 @@ This directory contains the documentation for the Encrypt Query Language (EQL). ## Reference -- [EQL index configuration for CipherStash Proxy](reference/index-configuration.md) +- [EQL Functions Reference](reference/eql-functions.md) - Complete API reference for all EQL functions +- [Database Indexes for Encrypted Columns](reference/database-indexes.md) - PostgreSQL B-tree index creation and usage +- [EQL index configuration for CipherStash Proxy](reference/index-config.md) - [EQL with JSON and JSONB](reference/json-support.md) -- [EQL payload data format](reference/eql-payload.md) +- [EQL payload data format](reference/PAYLOAD.md) ## Tutorials diff --git a/docs/plans/2025-01-20-rust-sql-tooling-poc-v2.md b/docs/plans/2025-01-20-rust-sql-tooling-poc-v2.md new file mode 100644 index 0000000..146f12f --- /dev/null +++ b/docs/plans/2025-01-20-rust-sql-tooling-poc-v2.md @@ -0,0 +1,1914 @@ +# Rust-based SQL Development Tooling - Proof of Concept (v2) + +> **For Claude:** Use `${SUPERPOWERS_SKILLS_ROOT}/skills/collaboration/executing-plans/SKILL.md` to implement this plan task-by-task. + +**Goal:** Create a Rust-based development framework for EQL that provides testing, documentation generation, and multi-database support, using the Config module as a proof of concept. + +**Architecture:** Modular trait system with core traits (Schema, Config, etc.) and independent feature traits (Ore32Bit, Ore64Bit, etc.). Each database implements only supported features. SQL files remain the source of truth (one function per file), referenced via Rust. Build tool extracts SQL in dependency order using type-safe dependency graph. + +**Tech Stack:** Rust (workspace with multiple crates), PostgreSQL driver (tokio-postgres), rustdoc for documentation generation, thiserror for structured errors. + +**Working Directory:** All commands run from `.worktrees/rust-sql-tooling` unless otherwise specified. + +--- + +## Success Criteria + +- [ ] Rust workspace compiles successfully +- [ ] Core trait system defined (Component, Config, Dependencies traits) +- [ ] Structured error handling with thiserror from the start +- [ ] PostgreSQL implementation of Config module **fully functional** (add_column works end-to-end) +- [ ] Test harness provides transaction isolation +- [ ] Build tool uses automatic dependency resolution via Component::Dependencies +- [ ] Build tool generates valid `cipherstash-encrypt-postgres.sql` from Config module +- [ ] Rustdoc generates customer-facing API documentation (HTML with SQL examples) +- [ ] All Config functions migrated: types, add_column, and their dependencies +- [ ] Integration tests pass: add_column creates working configuration + +--- + +## Task 1: Initialize Rust Workspace + +**Working directory:** Start from main repo root, then move to worktree + +**Files:** +- Create: `Cargo.toml` (workspace root) +- Create: `eql-core/Cargo.toml` +- Create: `eql-postgres/Cargo.toml` +- Create: `eql-test/Cargo.toml` +- Create: `eql-build/Cargo.toml` + +**Step 1: Navigate to worktree** + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling +# All subsequent commands run from this directory +``` + +**Step 2: Create workspace Cargo.toml** + +Create the root workspace configuration: + +```toml +[workspace] +members = [ + "eql-core", + "eql-postgres", + "eql-test", + "eql-build", +] + +resolver = "2" + +[workspace.dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.35", features = ["full"] } +tokio-postgres = "0.7" +anyhow = "1.0" +thiserror = "1.0" +``` + +**Step 3: Create eql-core crate (trait definitions)** + +```toml +[package] +name = "eql-core" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio-postgres = { workspace = true } +``` + +**Step 4: Create eql-postgres crate (PostgreSQL implementation)** + +```toml +[package] +name = "eql-postgres" +version = "0.1.0" +edition = "2021" + +[dependencies] +eql-core = { path = "../eql-core" } +serde = { workspace = true } +serde_json = { workspace = true } + +[dev-dependencies] +eql-test = { path = "../eql-test" } +tokio = { workspace = true } +``` + +**Step 5: Create eql-test crate (test harness)** + +```toml +[package] +name = "eql-test" +version = "0.1.0" +edition = "2021" + +[dependencies] +eql-core = { path = "../eql-core" } +tokio = { workspace = true } +tokio-postgres = { workspace = true } +serde_json = { workspace = true } +``` + +**Step 6: Create eql-build crate (build tool)** + +```toml +[package] +name = "eql-build" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "eql-build" +path = "src/main.rs" + +[dependencies] +eql-core = { path = "../eql-core" } +eql-postgres = { path = "../eql-postgres" } +anyhow = { workspace = true } +``` + +**Step 7: Create directory structure** + +```bash +mkdir -p eql-core/src +mkdir -p eql-postgres/src +mkdir -p eql-test/src +mkdir -p eql-build/src +``` + +**Step 8: Create minimal lib.rs files** + +```bash +echo "// EQL Core" > eql-core/src/lib.rs +echo "// EQL PostgreSQL" > eql-postgres/src/lib.rs +echo "// EQL Test Harness" > eql-test/src/lib.rs +echo "// EQL Build Tool" > eql-build/src/main.rs +echo "fn main() {}" >> eql-build/src/main.rs +``` + +**Step 9: Verify workspace compiles** + +```bash +cargo build +``` + +Expected: Builds successfully (warnings about empty crates are fine) + +**Step 10: Commit** + +```bash +git add Cargo.toml eql-*/Cargo.toml eql-*/src/ +git commit -m "feat: initialize Rust workspace for EQL tooling + +Create workspace with four crates: +- eql-core: Trait definitions for EQL API +- eql-postgres: PostgreSQL implementation +- eql-test: Test harness with transaction isolation +- eql-build: Build tool for SQL extraction + +This is a proof of concept for Rust-based SQL development tooling +to improve testing, documentation, and multi-database support." +``` + +--- + +## Task 2: Define Error Handling with thiserror + +**Files:** +- Create: `eql-core/src/error.rs` +- Modify: `eql-core/src/lib.rs` + +**Why first:** Errors need to be designed upfront so subsequent tasks use them from the start (TDD principle). + +**Step 1: Write test for error hierarchy** + +Create `eql-core/src/lib.rs`: + +```rust +//! EQL Core - Trait definitions for multi-database SQL extension API + +pub mod error; + +pub use error::{ComponentError, DatabaseError, EqlError}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_error_types_display() { + let err = ComponentError::SqlFileNotFound { + path: "test.sql".to_string(), + }; + assert!(err.to_string().contains("SQL file not found")); + assert!(err.to_string().contains("test.sql")); + } + + #[test] + fn test_database_error_context() { + let err = DatabaseError::MissingJsonbKey { + key: "tables".to_string(), + actual: serde_json::json!({"wrong": "value"}), + }; + let err_string = err.to_string(); + assert!(err_string.contains("tables")); + assert!(err_string.contains("wrong")); + } +} +``` + +**Step 2: Run test to verify it fails** + +```bash +cargo test --package eql-core +``` + +Expected: FAIL with "module `error` not found" + +**Step 3: Implement error hierarchy** + +Create `eql-core/src/error.rs`: + +```rust +//! Error types for EQL operations + +use thiserror::Error; + +/// Top-level error type for all EQL operations +#[derive(Error, Debug)] +pub enum EqlError { + #[error("Component error: {0}")] + Component(#[from] ComponentError), + + #[error("Database error: {0}")] + Database(#[from] DatabaseError), +} + +/// Errors related to SQL components and dependencies +#[derive(Error, Debug)] +pub enum ComponentError { + #[error("SQL file not found: {path}")] + SqlFileNotFound { path: String }, + + #[error("Dependency cycle detected: {cycle}")] + DependencyCycle { cycle: String }, + + #[error("IO error reading SQL file {path}: {source}")] + IoError { + path: String, + #[source] + source: std::io::Error, + }, + + #[error("Missing dependency: {component} requires {missing}")] + MissingDependency { + component: String, + missing: String, + }, +} + +/// Errors related to database operations +#[derive(Error, Debug)] +pub enum DatabaseError { + #[error("Connection failed: {0}")] + Connection(#[source] tokio_postgres::Error), + + #[error("Transaction failed: {0}")] + Transaction(String), + + #[error("Query failed: {query}: {source}")] + Query { + query: String, + #[source] + source: tokio_postgres::Error, + }, + + #[error("Expected JSONB value to have key '{key}', got: {actual}")] + MissingJsonbKey { + key: String, + actual: serde_json::Value, + }, +} +``` + +**Step 4: Run tests to verify they pass** + +```bash +cargo test --package eql-core +``` + +Expected: PASS (2 tests) + +**Step 5: Commit** + +```bash +git add eql-core/ +git commit -m "feat(eql-core): add structured error handling with thiserror + +Add error hierarchy: +- EqlError: Top-level error type +- ComponentError: SQL file and dependency errors +- DatabaseError: Database operation errors + +Benefits: +- Clear error messages with context (e.g., which query failed) +- Type-safe error handling throughout the codebase +- Better debugging experience for tests and build tools + +Errors defined first (before other code) to enable TDD." +``` + +--- + +## Task 3: Define Core Trait System + +**Files:** +- Modify: `eql-core/src/lib.rs` +- Create: `eql-core/src/component.rs` +- Create: `eql-core/src/config.rs` + +**Step 1: Write test for Component trait** + +Update `eql-core/src/lib.rs`: + +```rust +//! EQL Core - Trait definitions for multi-database SQL extension API + +pub mod component; +pub mod config; +pub mod error; + +pub use component::{Component, Dependencies}; +pub use config::Config; +pub use error::{ComponentError, DatabaseError, EqlError}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_component_trait_compiles() { + // This test verifies the trait definition compiles + // Actual implementations will be in eql-postgres + struct TestComponent; + + impl Component for TestComponent { + type Dependencies = (); + + fn sql_file() -> &'static str { + "test.sql" + } + } + + assert_eq!(TestComponent::sql_file(), "test.sql"); + } + + // ... existing error tests ... +} +``` + +**Step 2: Run test to verify it fails** + +```bash +cargo test --package eql-core +``` + +Expected: FAIL with "module `component` not found" + +**Step 3: Implement Component trait** + +Create `eql-core/src/component.rs`: + +```rust +//! Component trait for SQL file dependencies + +use std::marker::PhantomData; + +/// Marker trait for dependency specifications +pub trait Dependencies { + /// Collect all dependency SQL files in dependency order (dependencies first) + fn collect_sql_files(files: &mut Vec<&'static str>); +} + +/// Unit type represents no dependencies +impl Dependencies for () { + fn collect_sql_files(_files: &mut Vec<&'static str>) { + // No dependencies + } +} + +/// Single dependency +impl Dependencies for T { + fn collect_sql_files(files: &mut Vec<&'static str>) { + // First collect transitive dependencies + T::Dependencies::collect_sql_files(files); + // Then add this dependency + if !files.contains(&T::sql_file()) { + files.push(T::sql_file()); + } + } +} + +/// Two dependencies +impl Dependencies for (A, B) { + fn collect_sql_files(files: &mut Vec<&'static str>) { + A::Dependencies::collect_sql_files(files); + if !files.contains(&A::sql_file()) { + files.push(A::sql_file()); + } + B::Dependencies::collect_sql_files(files); + if !files.contains(&B::sql_file()) { + files.push(B::sql_file()); + } + } +} + +/// Three dependencies +impl Dependencies for (A, B, C) { + fn collect_sql_files(files: &mut Vec<&'static str>) { + A::Dependencies::collect_sql_files(files); + if !files.contains(&A::sql_file()) { + files.push(A::sql_file()); + } + B::Dependencies::collect_sql_files(files); + if !files.contains(&B::sql_file()) { + files.push(B::sql_file()); + } + C::Dependencies::collect_sql_files(files); + if !files.contains(&C::sql_file()) { + files.push(C::sql_file()); + } + } +} + +/// Four dependencies +impl Dependencies for (A, B, C, D) { + fn collect_sql_files(files: &mut Vec<&'static str>) { + A::Dependencies::collect_sql_files(files); + if !files.contains(&A::sql_file()) { + files.push(A::sql_file()); + } + B::Dependencies::collect_sql_files(files); + if !files.contains(&B::sql_file()) { + files.push(B::sql_file()); + } + C::Dependencies::collect_sql_files(files); + if !files.contains(&C::sql_file()) { + files.push(C::sql_file()); + } + D::Dependencies::collect_sql_files(files); + if !files.contains(&D::sql_file()) { + files.push(D::sql_file()); + } + } +} + +/// A component represents a single SQL file with its dependencies +pub trait Component { + /// Type specifying what this component depends on + type Dependencies: Dependencies; + + /// Path to the SQL file containing this component's implementation + fn sql_file() -> &'static str; + + /// Collect this component and all its dependencies in load order + fn collect_dependencies() -> Vec<&'static str> { + let mut files = Vec::new(); + // First collect all transitive dependencies + Self::Dependencies::collect_sql_files(&mut files); + // Then add self + if !files.contains(&Self::sql_file()) { + files.push(Self::sql_file()); + } + files + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct A; + impl Component for A { + type Dependencies = (); + fn sql_file() -> &'static str { "a.sql" } + } + + struct B; + impl Component for B { + type Dependencies = A; + fn sql_file() -> &'static str { "b.sql" } + } + + struct C; + impl Component for C { + type Dependencies = (A, B); + fn sql_file() -> &'static str { "c.sql" } + } + + #[test] + fn test_no_dependencies() { + let deps = A::collect_dependencies(); + assert_eq!(deps, vec!["a.sql"]); + } + + #[test] + fn test_single_dependency() { + let deps = B::collect_dependencies(); + assert_eq!(deps, vec!["a.sql", "b.sql"]); + } + + #[test] + fn test_multiple_dependencies() { + let deps = C::collect_dependencies(); + assert_eq!(deps, vec!["a.sql", "b.sql", "c.sql"]); + } + + #[test] + fn test_deduplication() { + // C depends on both A and B, but A should only appear once + let deps = C::collect_dependencies(); + let a_count = deps.iter().filter(|&&f| f == "a.sql").count(); + assert_eq!(a_count, 1, "a.sql should only appear once"); + } +} +``` + +**Step 4: Implement Config trait** + +Create `eql-core/src/config.rs`: + +```rust +//! Configuration management trait + +use crate::Component; + +/// Configuration management functions for encrypted columns +pub trait Config { + /// Add a column for encryption/decryption. + /// + /// Initializes a column to work with CipherStash encryption. The column + /// must be of type `eql_v2_encrypted`. + /// + /// # Parameters + /// + /// - `table_name` - Name of the table containing the column + /// - `column_name` - Name of the column to configure + /// - `cast_as` - PostgreSQL type for decrypted data (default: 'text') + /// - `migrating` - Whether this is part of a migration (default: false) + /// + /// # Returns + /// + /// JSONB containing the updated configuration. + /// + /// # Examples + /// + /// ```sql + /// -- Configure a text column for encryption + /// SELECT eql_v2.add_column('users', 'encrypted_email', 'text'); + /// + /// -- Configure a JSONB column + /// SELECT eql_v2.add_column('users', 'encrypted_data', 'jsonb'); + /// ``` + fn add_column() -> &'static dyn Component; + + /// Remove column configuration completely. + /// + /// # Examples + /// + /// ```sql + /// SELECT eql_v2.remove_column('users', 'encrypted_email'); + /// ``` + fn remove_column() -> &'static dyn Component; + + /// Add a searchable index to an encrypted column. + /// + /// # Supported index types + /// + /// - `unique` - Exact equality (uses hmac_256 or blake3) + /// - `match` - Full-text search (uses bloom_filter) + /// - `ore` - Range queries and ordering (uses ore_block_u64_8_256) + /// - `ste_vec` - JSONB containment queries (uses structured encryption) + /// + /// # Examples + /// + /// ```sql + /// SELECT eql_v2.add_search_config('users', 'encrypted_email', 'unique', 'text'); + /// SELECT eql_v2.add_search_config('docs', 'encrypted_content', 'match', 'text'); + /// ``` + fn add_search_config() -> &'static dyn Component; +} +``` + +**Step 5: Run tests to verify they pass** + +```bash +cargo test --package eql-core +``` + +Expected: PASS (6 tests: 2 error tests + 1 component test + 4 dependency tests) + +**Step 6: Commit** + +```bash +git add eql-core/ +git commit -m "feat(eql-core): define Component and Config traits + +Add core trait system for EQL API: +- Component trait: Represents SQL file with type-safe dependencies +- Dependencies trait: Automatic dependency collection via type system +- Config trait: Configuration management API with rustdoc examples + +Key innovation: Component::collect_dependencies() walks the type graph +at compile time to resolve SQL load order automatically. + +The Config trait includes documentation that will be auto-generated +into customer-facing docs, preventing documentation drift." +``` + +--- + +## Task 4: Create Test Harness with Transaction Isolation + +**Files:** +- Create: `eql-test/src/lib.rs` + +**Step 1: Write test for TestDb** + +Create `eql-test/src/lib.rs`: + +```rust +//! Test harness providing transaction isolation for SQL tests + +use eql_core::error::DatabaseError; +use tokio_postgres::{Client, NoTls, Row}; + +pub struct TestDb { + client: Client, + in_transaction: bool, +} + +impl TestDb { + /// Create new test database with transaction isolation + pub async fn new() -> Result { + let (client, connection) = tokio_postgres::connect( + &Self::connection_string(), + NoTls, + ) + .await + .map_err(DatabaseError::Connection)?; + + // Spawn connection handler + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("Connection error: {}", e); + } + }); + + // Begin transaction for isolation + client.execute("BEGIN", &[]) + .await + .map_err(|e| DatabaseError::Query { + query: "BEGIN".to_string(), + source: e, + })?; + + Ok(Self { + client, + in_transaction: true, + }) + } + + fn connection_string() -> String { + std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| "host=localhost port=7432 user=cipherstash password=password dbname=postgres".to_string()) + } + + /// Execute SQL (for setup/implementation loading) + pub async fn execute(&self, sql: &str) -> Result { + self.client.execute(sql, &[]) + .await + .map_err(|e| DatabaseError::Query { + query: sql.to_string(), + source: e, + }) + } + + /// Query with single result + pub async fn query_one(&self, sql: &str) -> Result { + self.client.query_one(sql, &[]) + .await + .map_err(|e| DatabaseError::Query { + query: sql.to_string(), + source: e, + }) + } + + /// Assert JSONB result has key + pub fn assert_jsonb_has_key(&self, result: &Row, column_index: usize, key: &str) -> Result<(), DatabaseError> { + let json: serde_json::Value = result.get(column_index); + if json.get(key).is_none() { + return Err(DatabaseError::MissingJsonbKey { + key: key.to_string(), + actual: json, + }); + } + Ok(()) + } +} + +impl Drop for TestDb { + fn drop(&mut self) { + if self.in_transaction { + // Auto-rollback on drop + // Note: Can't use async in Drop, but connection will rollback anyway + // when client drops + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_testdb_transaction_isolation() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Create a temporary table + db.execute("CREATE TEMPORARY TABLE test_table (id int, value text)") + .await + .expect("Failed to create table"); + + // Insert data + db.execute("INSERT INTO test_table VALUES (1, 'test')") + .await + .expect("Failed to insert"); + + // Query data + let row = db.query_one("SELECT value FROM test_table WHERE id = 1") + .await + .expect("Failed to query"); + + let value: String = row.get(0); + assert_eq!(value, "test"); + + // Transaction will rollback on drop - table won't exist in next test + } + + #[tokio::test] + async fn test_database_error_includes_query() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + let result = db.execute("INVALID SQL SYNTAX").await; + assert!(result.is_err()); + + let err = result.unwrap_err(); + let err_string = err.to_string(); + assert!(err_string.contains("Query failed")); + assert!(err_string.contains("INVALID SQL SYNTAX")); + } +} +``` + +**Step 2: Run test to verify it compiles** + +```bash +cargo test --package eql-test +``` + +Expected: May fail if PostgreSQL container not running, but should compile + +**Step 3: Start PostgreSQL container for testing** + +From main repo root: + +```bash +cd /Users/tobyhede/src/encrypt-query-language +mise run postgres:up postgres-17 --extra-args "--detach --wait" +``` + +Expected: PostgreSQL container starts successfully on port 7432 + +**Step 4: Run test to verify it passes** + +```bash +cd .worktrees/rust-sql-tooling +cargo test --package eql-test +``` + +Expected: PASS (2 tests) - transaction isolation working, error messages include query context + +**Step 5: Commit** + +```bash +git add eql-test/ +git commit -m "feat(eql-test): add test harness with transaction isolation + +Create TestDb struct providing: +- Automatic transaction BEGIN on creation +- Auto-rollback on drop (clean slate for next test) +- Helper methods: execute(), query_one() +- Assertion helpers: assert_jsonb_has_key() +- Structured DatabaseError with query context + +This solves current testing pain points: +- No more manual database resets between tests +- Clear error messages (shows which query failed) +- Foundation for parallel test execution (future)" +``` + +--- + +## Task 5: Migrate Config SQL Dependencies + +**Files:** +- Create: `eql-postgres/src/sql/config/types.sql` +- Create: `eql-postgres/src/sql/config/functions_private.sql` +- Create: `eql-postgres/src/sql/encrypted/check_encrypted.sql` +- Create: `eql-postgres/src/sql/encrypted/add_encrypted_constraint.sql` +- Create: `eql-postgres/src/sql/config/migrate_activate.sql` +- Create: `eql-postgres/src/sql/config/add_column.sql` + +**Why this order:** We need all dependencies before we can test add_column working end-to-end. + +**Step 1: Create directory structure** + +```bash +mkdir -p eql-postgres/src/sql/config +mkdir -p eql-postgres/src/sql/encrypted +``` + +**Step 2: Copy config types** + +Copy from main repo (go up two levels from worktree): + +```bash +cp ../../src/config/types.sql eql-postgres/src/sql/config/types.sql +``` + +**Step 3: Copy config private functions** + +```bash +cp ../../src/config/functions_private.sql eql-postgres/src/sql/config/functions_private.sql +``` + +**Step 4: Create minimal check_encrypted stub** + +Create `eql-postgres/src/sql/encrypted/check_encrypted.sql`: + +```sql +-- Stub for check_encrypted function (minimal implementation for POC) +-- Full implementation would validate encrypted data structure + +CREATE FUNCTION eql_v2.check_encrypted(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ +BEGIN + -- For POC: Just check that it's a JSONB object + RETURN jsonb_typeof(val) = 'object'; +END; +$$ LANGUAGE plpgsql; +``` + +**Step 5: Extract add_encrypted_constraint function** + +Create `eql-postgres/src/sql/encrypted/add_encrypted_constraint.sql`: + +```sql +-- Add constraint to verify encrypted column structure +-- +-- Depends on: check_encrypted function + +CREATE FUNCTION eql_v2.add_encrypted_constraint(table_name TEXT, column_name TEXT) + RETURNS void +AS $$ +BEGIN + EXECUTE format( + 'ALTER TABLE %I ADD CONSTRAINT eql_v2_encrypted_check_%I CHECK (eql_v2.check_encrypted(%I))', + table_name, + column_name, + column_name + ); +END; +$$ LANGUAGE plpgsql; +``` + +**Step 6: Extract migrate_config and activate_config** + +Create `eql-postgres/src/sql/config/migrate_activate.sql`: + +```sql +-- Configuration migration and activation functions +-- +-- Depends on: config/types.sql (for eql_v2_configuration table) + +-- Stub for ready_for_encryption (POC only) +CREATE FUNCTION eql_v2.ready_for_encryption() + RETURNS boolean +AS $$ +BEGIN + -- POC: Always return true + -- Real implementation would validate all configured columns exist + RETURN true; +END; +$$ LANGUAGE plpgsql; + +-- Marks the currently pending configuration as encrypting +CREATE FUNCTION eql_v2.migrate_config() + RETURNS boolean +AS $$ +BEGIN + IF EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'encrypting') THEN + RAISE EXCEPTION 'An encryption is already in progress'; + END IF; + + IF NOT EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'pending') THEN + RAISE EXCEPTION 'No pending configuration exists to encrypt'; + END IF; + + IF NOT eql_v2.ready_for_encryption() THEN + RAISE EXCEPTION 'Some pending columns do not have an encrypted target'; + END IF; + + UPDATE public.eql_v2_configuration SET state = 'encrypting' WHERE state = 'pending'; + RETURN true; +END; +$$ LANGUAGE plpgsql; + +-- Activates the currently encrypting configuration +CREATE FUNCTION eql_v2.activate_config() + RETURNS boolean +AS $$ +BEGIN + IF EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'encrypting') THEN + UPDATE public.eql_v2_configuration SET state = 'inactive' WHERE state = 'active'; + UPDATE public.eql_v2_configuration SET state = 'active' WHERE state = 'encrypting'; + RETURN true; + ELSE + RAISE EXCEPTION 'No encrypting configuration exists to activate'; + END IF; +END; +$$ LANGUAGE plpgsql; +``` + +**Step 7: Extract add_column function** + +Create `eql-postgres/src/sql/config/add_column.sql`: + +```sql +-- Add a column for encryption/decryption +-- +-- This function initializes a column to work with CipherStash encryption. +-- The column must be of type eql_v2_encrypted. +-- +-- Depends on: config/types.sql, config/functions_private.sql, +-- config/migrate_activate.sql, encrypted/add_encrypted_constraint.sql + +CREATE FUNCTION eql_v2.add_column(table_name text, column_name text, cast_as text DEFAULT 'text', migrating boolean DEFAULT false) + RETURNS jsonb +AS $$ + DECLARE + _config jsonb; + BEGIN + -- set the active config + SELECT data INTO _config FROM public.eql_v2_configuration WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + + -- set default config + SELECT eql_v2.config_default(_config) INTO _config; + + -- if index exists + IF _config #> array['tables', table_name] ? column_name THEN + RAISE EXCEPTION 'Config exists for column: % %', table_name, column_name; + END IF; + + SELECT eql_v2.config_add_table(table_name, _config) INTO _config; + + SELECT eql_v2.config_add_column(table_name, column_name, _config) INTO _config; + + SELECT eql_v2.config_add_cast(table_name, column_name, cast_as, _config) INTO _config; + + -- create a new pending record if we don't have one + INSERT INTO public.eql_v2_configuration (state, data) VALUES ('pending', _config) + ON CONFLICT (state) + WHERE state = 'pending' + DO UPDATE + SET data = _config; + + IF NOT migrating THEN + PERFORM eql_v2.migrate_config(); + PERFORM eql_v2.activate_config(); + END IF; + + PERFORM eql_v2.add_encrypted_constraint(table_name, column_name); + + -- exeunt + RETURN _config; + END; +$$ LANGUAGE plpgsql; +``` + +**Step 8: Verify SQL files exist** + +```bash +ls -la eql-postgres/src/sql/config/ +ls -la eql-postgres/src/sql/encrypted/ +``` + +Expected: All 6 SQL files exist + +**Step 9: Commit SQL files** + +```bash +git add eql-postgres/src/sql/ +git commit -m "feat(eql-postgres): migrate Config module SQL files + +Add SQL implementations: +- config/types.sql: Configuration table and enum type +- config/functions_private.sql: Helper functions (config_default, etc.) +- config/migrate_activate.sql: Migration and activation functions +- config/add_column.sql: Main add_column function +- encrypted/check_encrypted.sql: Stub for encrypted data validation +- encrypted/add_encrypted_constraint.sql: Constraint helper + +All dependencies for add_column now present. Next task will wire +these up via Rust Component trait with automatic dependency resolution." +``` + +--- + +## Task 6: Implement PostgreSQL Config Components + +**Files:** +- Create: `eql-postgres/src/lib.rs` +- Create: `eql-postgres/src/config.rs` + +**Step 1: Design component hierarchy** + +We need components for each SQL file, with proper dependencies: +- `ConfigTypes` (no dependencies) +- `ConfigPrivateFunctions` (depends on ConfigTypes) +- `CheckEncrypted` (no dependencies) +- `AddEncryptedConstraint` (depends on CheckEncrypted) +- `MigrateActivate` (depends on ConfigTypes) +- `AddColumn` (depends on ConfigTypes, ConfigPrivateFunctions, AddEncryptedConstraint, MigrateActivate) + +**Step 2: Write test for component dependencies** + +Create `eql-postgres/src/lib.rs`: + +```rust +//! PostgreSQL implementation of EQL + +pub mod config; + +pub use config::PostgresEQL; + +#[cfg(test)] +mod tests { + use super::*; + use eql_core::{Component, Config}; + + #[test] + fn test_component_sql_files_exist() { + let add_column = PostgresEQL::add_column(); + let path = add_column.sql_file(); + assert!( + std::path::Path::new(path).exists(), + "add_column SQL file should exist at {}", + path + ); + } + + #[test] + fn test_add_column_dependencies_collected() { + use config::AddColumn; + + let deps = AddColumn::collect_dependencies(); + + // Should include all dependencies in order + assert!(deps.len() > 1, "AddColumn should have dependencies"); + + // Dependencies should come before AddColumn itself + let add_column_path = AddColumn::sql_file(); + let add_column_pos = deps.iter().position(|&f| f == add_column_path); + assert!(add_column_pos.is_some(), "Should include AddColumn itself"); + + // Verify no duplicates + let mut seen = std::collections::HashSet::new(); + for file in &deps { + assert!(seen.insert(file), "Dependency {} appears twice", file); + } + } +} +``` + +**Step 3: Run test to verify it fails** + +```bash +cargo test --package eql-postgres +``` + +Expected: FAIL - module `config` not found + +**Step 4: Implement component definitions** + +Create `eql-postgres/src/config.rs`: + +```rust +//! PostgreSQL implementation of Config trait + +use eql_core::{Component, Config, Dependencies}; + +// Base component: Configuration types +pub struct ConfigTypes; + +impl Component for ConfigTypes { + type Dependencies = (); + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/config/types.sql" + ) + } +} + +impl Dependencies for ConfigTypes {} + +// Private helper functions +pub struct ConfigPrivateFunctions; + +impl Component for ConfigPrivateFunctions { + type Dependencies = ConfigTypes; + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/config/functions_private.sql" + ) + } +} + +impl Dependencies for ConfigPrivateFunctions {} + +// Encrypted data validation stub +pub struct CheckEncrypted; + +impl Component for CheckEncrypted { + type Dependencies = (); + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/encrypted/check_encrypted.sql" + ) + } +} + +impl Dependencies for CheckEncrypted {} + +// Add encrypted constraint helper +pub struct AddEncryptedConstraint; + +impl Component for AddEncryptedConstraint { + type Dependencies = CheckEncrypted; + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/encrypted/add_encrypted_constraint.sql" + ) + } +} + +impl Dependencies for AddEncryptedConstraint {} + +// Migration and activation functions +pub struct MigrateActivate; + +impl Component for MigrateActivate { + type Dependencies = ConfigTypes; + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/config/migrate_activate.sql" + ) + } +} + +impl Dependencies for MigrateActivate {} + +// Main add_column function +pub struct AddColumn; + +impl Component for AddColumn { + type Dependencies = ( + ConfigTypes, + ConfigPrivateFunctions, + MigrateActivate, + AddEncryptedConstraint, + ); + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/config/add_column.sql" + ) + } +} + +// PostgreSQL implementation of Config trait +pub struct PostgresEQL; + +impl Config for PostgresEQL { + fn add_column() -> &'static dyn Component { + &AddColumn + } + + fn remove_column() -> &'static dyn Component { + todo!("Not implemented in POC") + } + + fn add_search_config() -> &'static dyn Component { + todo!("Not implemented in POC") + } +} +``` + +**Step 5: Run tests to verify they pass** + +```bash +cargo test --package eql-postgres +``` + +Expected: PASS (2 tests) - SQL files exist, dependencies collected correctly + +**Step 6: Verify dependency order manually** + +Add a debug test: + +```rust +#[test] +fn test_print_dependency_order() { + use config::AddColumn; + + let deps = AddColumn::collect_dependencies(); + println!("Dependency order for AddColumn:"); + for (i, file) in deps.iter().enumerate() { + println!(" {}. {}", i + 1, file); + } + + // Expected order: + // 1. types.sql (no deps) + // 2. functions_private.sql (depends on types) + // 3. check_encrypted.sql (no deps) + // 4. add_encrypted_constraint.sql (depends on check_encrypted) + // 5. migrate_activate.sql (depends on types) + // 6. add_column.sql (depends on all above) +} +``` + +```bash +cargo test --package eql-postgres test_print_dependency_order -- --nocapture +``` + +Expected: Prints dependency order, verify it makes sense + +**Step 7: Commit** + +```bash +git add eql-postgres/ +git commit -m "feat(eql-postgres): implement Config components with dependencies + +Add PostgreSQL component implementations: +- ConfigTypes: Configuration table/enum (no dependencies) +- ConfigPrivateFunctions: Helper functions (depends on ConfigTypes) +- CheckEncrypted: Validation stub (no dependencies) +- AddEncryptedConstraint: Constraint helper (depends on CheckEncrypted) +- MigrateActivate: Migration functions (depends on ConfigTypes) +- AddColumn: Main function (depends on all above) + +Key achievement: Component::collect_dependencies() automatically +resolves load order via type-level dependency graph. + +Tests verify: +- SQL files exist at expected paths +- Dependencies collected without duplicates +- Dependency order respects constraints" +``` + +--- + +## Task 7: Write Integration Test for add_column + +**Files:** +- Create: `eql-postgres/tests/config_test.rs` + +**Step 1: Write failing test** + +Create `eql-postgres/tests/config_test.rs`: + +```rust +use eql_postgres::config::{AddColumn, PostgresEQL}; +use eql_core::{Component, Config}; +use eql_test::TestDb; + +#[tokio::test] +async fn test_add_column_creates_config() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Load schema (from main project - need eql_v2 schema + encrypted type) + let schema_sql = include_str!("../../../src/schema.sql"); + db.execute(schema_sql).await.expect("Failed to create schema"); + + // Create minimal encrypted type stub for POC + db.execute( + "CREATE TYPE eql_v2_encrypted AS (data jsonb);" + ).await.expect("Failed to create encrypted type"); + + // Load all dependencies in order + let deps = AddColumn::collect_dependencies(); + for sql_file in deps { + let sql = std::fs::read_to_string(sql_file) + .unwrap_or_else(|e| panic!("Failed to read {}: {}", sql_file, e)); + db.execute(&sql) + .await + .unwrap_or_else(|e| panic!("Failed to load {}: {}", sql_file, e)); + } + + // Setup: Create test table with encrypted column + db.execute( + "CREATE TABLE users ( + id int, + email eql_v2_encrypted + )" + ).await.expect("Failed to create table"); + + // Execute: Call add_column + let result = db.query_one( + "SELECT eql_v2.add_column('users', 'email', 'text')" + ) + .await + .expect("Failed to call add_column"); + + // Assert: Result has expected structure + db.assert_jsonb_has_key(&result, 0, "tables") + .expect("Expected 'tables' key in config"); + + db.assert_jsonb_has_key(&result, 0, "v") + .expect("Expected 'v' (version) key in config"); + + // Assert: Configuration was stored + let config_row = db.query_one( + "SELECT data FROM public.eql_v2_configuration WHERE state = 'active'" + ) + .await + .expect("Should have active config"); + + db.assert_jsonb_has_key(&config_row, 0, "tables") + .expect("Stored config should have 'tables' key"); + + // Assert: Constraint was added + let constraint_exists = db.query_one( + "SELECT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'eql_v2_encrypted_check_email' + )" + ).await.expect("Failed to check constraint"); + + let exists: bool = constraint_exists.get(0); + assert!(exists, "Encrypted constraint should exist"); +} + +#[tokio::test] +async fn test_add_column_rejects_duplicate() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Load schema and dependencies (same as above) + let schema_sql = include_str!("../../../src/schema.sql"); + db.execute(schema_sql).await.expect("Failed to create schema"); + + db.execute("CREATE TYPE eql_v2_encrypted AS (data jsonb);") + .await.expect("Failed to create encrypted type"); + + let deps = AddColumn::collect_dependencies(); + for sql_file in deps { + let sql = std::fs::read_to_string(sql_file).unwrap(); + db.execute(&sql).await.unwrap(); + } + + db.execute("CREATE TABLE users (id int, email eql_v2_encrypted)") + .await.expect("Failed to create table"); + + // First call succeeds + db.query_one("SELECT eql_v2.add_column('users', 'email', 'text')") + .await + .expect("First add_column should succeed"); + + // Second call should fail + let result = db.query_one("SELECT eql_v2.add_column('users', 'email', 'text')").await; + assert!(result.is_err(), "Duplicate add_column should fail"); + + let err = result.unwrap_err(); + let err_string = err.to_string(); + assert!( + err_string.contains("Config exists for column"), + "Error should mention column already exists: {}", + err_string + ); +} +``` + +**Step 2: Run test to verify it fails** + +```bash +cargo test --package eql-postgres --test config_test +``` + +Expected: Likely fails on schema loading or missing encrypted type + +**Step 3: Check if schema.sql creates eql_v2 schema** + +From main repo: + +```bash +head -20 ../../src/schema.sql +``` + +If it doesn't create the schema, update the test to add: + +```rust +db.execute("CREATE SCHEMA IF NOT EXISTS eql_v2;") + .await.expect("Failed to create schema"); +``` + +**Step 4: Run test until it passes** + +Debug any SQL errors by examining the DatabaseError output. + +```bash +cargo test --package eql-postgres --test config_test -- --nocapture +``` + +Expected: PASS (2 tests) - add_column works end-to-end, rejects duplicates + +**Step 5: Commit** + +```bash +git add eql-postgres/tests/ +git commit -m "test(eql-postgres): add integration tests for add_column + +Add comprehensive integration tests: +1. test_add_column_creates_config: Verifies complete workflow + - Loads all dependencies via Component::collect_dependencies() + - Calls add_column function + - Validates JSONB config structure + - Confirms config stored in database + - Checks encrypted constraint was added + +2. test_add_column_rejects_duplicate: Verifies error handling + - Ensures duplicate column config raises exception + - Validates error message includes helpful context + +Key achievement: add_column function works end-to-end in POC. +All dependencies loaded automatically via type-safe dependency graph." +``` + +--- + +## Task 8: Create Build Tool with Dependency Resolution + +**Files:** +- Create: `eql-build/src/main.rs` +- Create: `eql-build/src/builder.rs` + +**Step 1: Write test for build tool** + +Create `eql-build/src/main.rs`: + +```rust +//! Build tool for extracting SQL files in dependency order + +use anyhow::{Context, Result}; +use std::fs; + +mod builder; + +use builder::Builder; + +fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + + if args.len() < 2 { + eprintln!("Usage: eql-build "); + eprintln!(" database: postgres"); + std::process::exit(1); + } + + let database = &args[1]; + + match database.as_str() { + "postgres" => build_postgres()?, + _ => anyhow::bail!("Unknown database: {}", database), + } + + Ok(()) +} + +fn build_postgres() -> Result<()> { + use eql_postgres::config::AddColumn; + use eql_core::Component; + + println!("Building PostgreSQL installer..."); + + let mut builder = Builder::new("CipherStash EQL for PostgreSQL"); + + // Use automatic dependency resolution + let deps = AddColumn::collect_dependencies(); + println!("Resolved {} dependencies", deps.len()); + + for (i, sql_file) in deps.iter().enumerate() { + println!(" {}. {}", i + 1, sql_file.split('/').last().unwrap_or(sql_file)); + builder.add_sql_file(sql_file)?; + } + + // Write output + fs::create_dir_all("release")?; + let output = builder.build(); + fs::write("release/cipherstash-encrypt-postgres-poc.sql", output)?; + + println!("✓ Generated release/cipherstash-encrypt-postgres-poc.sql"); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_creates_output_file() { + // Clean up any previous output + let _ = std::fs::remove_file("release/cipherstash-encrypt-postgres-poc.sql"); + + // Run build + build_postgres().expect("Build should succeed"); + + // Verify output exists + assert!( + std::path::Path::new("release/cipherstash-encrypt-postgres-poc.sql").exists(), + "Build should create output file" + ); + + // Verify it contains expected SQL + let content = std::fs::read_to_string("release/cipherstash-encrypt-postgres-poc.sql") + .expect("Should be able to read output"); + + assert!(content.contains("eql_v2_configuration_state"), "Should contain config types"); + assert!(content.contains("CREATE FUNCTION eql_v2.add_column"), "Should contain add_column function"); + assert!(content.contains("CREATE FUNCTION eql_v2.config_default"), "Should contain helper functions"); + } + + #[test] + fn test_build_dependency_order() { + build_postgres().expect("Build should succeed"); + + let content = std::fs::read_to_string("release/cipherstash-encrypt-postgres-poc.sql") + .expect("Should be able to read output"); + + // types.sql should come before functions_private.sql + let types_pos = content.find("eql_v2_configuration_state") + .expect("Should contain types"); + let private_pos = content.find("CREATE FUNCTION eql_v2.config_default") + .expect("Should contain private functions"); + + assert!( + types_pos < private_pos, + "Types should be defined before functions that use them" + ); + + // check_encrypted should come before add_encrypted_constraint + let check_pos = content.find("CREATE FUNCTION eql_v2.check_encrypted") + .expect("Should contain check_encrypted"); + let constraint_pos = content.find("CREATE FUNCTION eql_v2.add_encrypted_constraint") + .expect("Should contain add_encrypted_constraint"); + + assert!( + check_pos < constraint_pos, + "check_encrypted should be defined before add_encrypted_constraint" + ); + } +} +``` + +**Step 2: Implement Builder** + +Create `eql-build/src/builder.rs`: + +```rust +//! SQL file builder with dependency management + +use anyhow::{Context, Result}; +use std::fs; + +pub struct Builder { + header: String, + files: Vec, +} + +impl Builder { + pub fn new(title: &str) -> Self { + Self { + header: format!("-- {}\n-- Generated by eql-build\n\n", title), + files: Vec::new(), + } + } + + pub fn add_sql_file(&mut self, path: &str) -> Result<()> { + let sql = fs::read_to_string(path) + .with_context(|| format!("Failed to read SQL file: {}", path))?; + + // Remove REQUIRE comments (they're metadata for old build system) + let cleaned = sql + .lines() + .filter(|line| !line.trim_start().starts_with("-- REQUIRE:")) + .collect::>() + .join("\n"); + + self.files.push(cleaned); + Ok(()) + } + + pub fn build(self) -> String { + let mut output = self.header; + + for (i, file) in self.files.iter().enumerate() { + if i > 0 { + output.push_str("\n\n"); + } + output.push_str(file); + } + + output + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_builder_basic() { + let mut builder = Builder::new("Test"); + assert!(builder.build().contains("Test")); + assert!(builder.build().contains("Generated by eql-build")); + } +} +``` + +**Step 3: Run test to verify it fails** + +```bash +cargo test --package eql-build +``` + +Expected: FAIL - release directory doesn't exist yet + +**Step 4: Run build to create output** + +```bash +cargo run --bin eql-build postgres +``` + +Expected: Creates `release/cipherstash-encrypt-postgres-poc.sql` with dependency-ordered SQL + +**Step 5: Run tests to verify they pass** + +```bash +cargo test --package eql-build +``` + +Expected: PASS (3 tests) - output created, dependencies in correct order + +**Step 6: Verify generated SQL is valid** + +From main repo: + +```bash +cd /Users/tobyhede/src/encrypt-query-language +psql -h localhost -p 7432 -U cipherstash -d postgres -f .worktrees/rust-sql-tooling/release/cipherstash-encrypt-postgres-poc.sql +``` + +Expected: SQL executes successfully (may need to manually add schema first) + +**Step 7: Commit** + +```bash +cd .worktrees/rust-sql-tooling +git add eql-build/ release/ +git commit -m "feat(eql-build): implement build tool with dependency resolution + +Create build tool that: +- Uses Component::collect_dependencies() for automatic ordering +- Reads SQL files in dependency order +- Generates release/cipherstash-encrypt-postgres-poc.sql +- Removes REQUIRE comments (metadata from old system) + +Key achievement: Build tool uses type-level dependency graph +to automatically resolve SQL load order. No manual topological +sort or configuration files needed. + +Tests verify: +- Output file created +- Contains all expected SQL +- Dependencies in correct order (types before functions using them)" +``` + +--- + +## Task 9: Generate Customer-Facing Documentation + +**Files:** +- Create: `.cargo/config.toml` (optional, for rustdoc settings) + +**Step 1: Generate rustdoc HTML** + +```bash +cargo doc --no-deps --open +``` + +Expected: Opens browser with generated documentation + +**Step 2: Verify Config trait documentation** + +In browser, navigate to: +- `eql_core` → `config` → `Config` trait + +Verify: +- ✅ SQL examples are visible (not Rust code examples) +- ✅ Function descriptions are customer-friendly +- ✅ No implementation details leaked +- ✅ Examples show actual usage patterns + +**Step 3: Add custom CSS (optional)** + +Create `.cargo/config.toml`: + +```toml +[doc] +# Additional rustdoc flags +rustdocflags = ["--html-in-header", "docs/doc-header.html"] +``` + +This is optional - only if you want custom styling. + +**Step 4: Take screenshots for verification** + +```bash +# Generate docs +cargo doc --no-deps + +# Docs are in target/doc/eql_core/trait.Config.html +open target/doc/eql_core/trait.Config.html +``` + +Screenshot the Config trait documentation to confirm it looks good. + +**Step 5: Commit (if config added)** + +```bash +git add .cargo/config.toml # Only if created +git commit -m "docs: configure rustdoc for customer-facing documentation + +Rustdoc generates HTML documentation directly from trait definitions. +Config trait includes SQL examples (not Rust usage) to show customers +how to use the database functions. + +Key benefit: Documentation lives in code, preventing drift. +No separate markdown files to maintain. + +View docs: cargo doc --no-deps --open" +``` + +--- + +## Verification Checklist + +Run these commands to verify POC is complete: + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling + +# 1. All tests pass +cargo test --all + +# 2. Build tool generates SQL +cargo run --bin eql-build postgres + +# 3. Generated SQL is valid +wc -l release/cipherstash-encrypt-postgres-poc.sql # Should be 100+ lines + +# 4. Verify SQL loads successfully +cd /Users/tobyhede/src/encrypt-query-language +psql -h localhost -p 7432 -U cipherstash -d postgres < **For Claude:** Use `${SUPERPOWERS_SKILLS_ROOT}/skills/collaboration/executing-plans/SKILL.md` to implement this plan task-by-task. + +**Goal:** Create a Rust-based development framework for EQL that provides testing, documentation generation, and multi-database support, using the Config module as a proof of concept. + +**Architecture:** Modular trait system with core traits (Schema, Config, etc.) and independent feature traits (Ore32Bit, Ore64Bit, etc.). Each database implements only supported features. SQL files remain the source of truth (one function per file), referenced via Rust. Build tool extracts SQL in dependency order to generate single installer file per database. + +**Tech Stack:** Rust (workspace with multiple crates), PostgreSQL driver (tokio-postgres), TOML for component metadata, rustdoc for documentation generation. + +--- + +## Success Criteria + +- [ ] Rust workspace compiles successfully +- [ ] Core trait system defined (Component, Config traits) +- [ ] PostgreSQL implementation of Config module functional +- [ ] Test harness provides transaction isolation +- [ ] Build tool generates valid `cipherstash-encrypt-postgres.sql` from Config module +- [ ] Rustdoc generates customer-facing API documentation (HTML with SQL examples) +- [ ] Structured error handling with thiserror provides clear error messages +- [ ] Config types and add_column migrated with tests passing + +--- + +## Task 1: Initialize Rust Workspace + +**Files:** +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/Cargo.toml` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-core/Cargo.toml` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-postgres/Cargo.toml` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-test/Cargo.toml` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-build/Cargo.toml` + +**Step 1: Create workspace Cargo.toml** + +Create the root workspace configuration: + +```toml +[workspace] +members = [ + "eql-core", + "eql-postgres", + "eql-test", + "eql-build", +] + +resolver = "2" + +[workspace.dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.35", features = ["full"] } +tokio-postgres = "0.7" +anyhow = "1.0" +``` + +**Step 2: Create eql-core crate (trait definitions)** + +```toml +[package] +name = "eql-core" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +``` + +**Step 3: Create eql-postgres crate (PostgreSQL implementation)** + +```toml +[package] +name = "eql-postgres" +version = "0.1.0" +edition = "2021" + +[dependencies] +eql-core = { path = "../eql-core" } +serde = { workspace = true } +serde_json = { workspace = true } + +[dev-dependencies] +eql-test = { path = "../eql-test" } +tokio = { workspace = true } +``` + +**Step 4: Create eql-test crate (test harness)** + +```toml +[package] +name = "eql-test" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokio = { workspace = true } +tokio-postgres = { workspace = true } +anyhow = { workspace = true } +serde_json = { workspace = true } +``` + +**Step 5: Create eql-build crate (build tool)** + +```toml +[package] +name = "eql-build" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "eql-build" +path = "src/main.rs" + +[dependencies] +eql-core = { path = "../eql-core" } +eql-postgres = { path = "../eql-postgres" } +anyhow = { workspace = true } +``` + +**Step 6: Verify workspace compiles** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo build` + +Expected: Builds successfully (may have warnings about empty crates) + +**Step 7: Commit** + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling +git add Cargo.toml eql-core/Cargo.toml eql-postgres/Cargo.toml eql-test/Cargo.toml eql-build/Cargo.toml +git commit -m "feat: initialize Rust workspace for EQL tooling + +Create workspace with four crates: +- eql-core: Trait definitions for EQL API +- eql-postgres: PostgreSQL implementation +- eql-test: Test harness with transaction isolation +- eql-build: Build tool for SQL extraction + +This is a proof of concept for Rust-based SQL development tooling +to improve testing, documentation, and multi-database support." +``` + +--- + +## Task 2: Define Core Trait System + +**Files:** +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-core/src/lib.rs` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-core/src/component.rs` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-core/src/config.rs` + +**Step 1: Write test for Component trait** + +Create `eql-core/src/lib.rs`: + +```rust +//! EQL Core - Trait definitions for multi-database SQL extension API + +pub mod component; +pub mod config; + +pub use component::{Component, Dependencies}; +pub use config::Config; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_component_trait_compiles() { + // This test verifies the trait definition compiles + // Actual implementations will be in eql-postgres + struct TestComponent; + + impl Component for TestComponent { + type Dependencies = (); + + fn sql_file() -> &'static str { + "test.sql" + } + } + + assert_eq!(TestComponent::sql_file(), "test.sql"); + } +} +``` + +**Step 2: Run test to verify it fails** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo test --package eql-core` + +Expected: FAIL with "module `component` not found" + +**Step 3: Implement Component trait** + +Create `eql-core/src/component.rs`: + +```rust +//! Component trait for SQL file dependencies + +use std::marker::PhantomData; + +/// Marker trait for dependency specifications +pub trait Dependencies {} + +/// Unit type represents no dependencies +impl Dependencies for () {} + +/// Tuple types represent multiple dependencies +impl Dependencies for (A,) {} +impl Dependencies for (A, B) {} +impl Dependencies for (A, B, C) {} +impl Dependencies for (A, B, C, D) {} + +/// A component represents a single SQL file with its dependencies +pub trait Component { + /// Type specifying what this component depends on + type Dependencies: Dependencies; + + /// Path to the SQL file containing this component's implementation + fn sql_file() -> &'static str; +} +``` + +**Step 4: Implement Config trait** + +Create `eql-core/src/config.rs`: + +```rust +//! Configuration management trait + +use crate::Component; + +/// Configuration management functions for encrypted columns +pub trait Config { + /// Add a column for encryption/decryption. + /// + /// Initializes a column to work with CipherStash encryption. The column + /// must be of type `eql_v2_encrypted`. + /// + /// # Parameters + /// + /// - `table_name` - Name of the table containing the column + /// - `column_name` - Name of the column to configure + /// - `cast_as` - PostgreSQL type for decrypted data (default: 'text') + /// - `migrating` - Whether this is part of a migration (default: false) + /// + /// # Returns + /// + /// JSONB containing the updated configuration. + /// + /// # Examples + /// + /// ```sql + /// -- Configure a text column for encryption + /// SELECT eql_v2.add_column('users', 'encrypted_email', 'text'); + /// + /// -- Configure a JSONB column + /// SELECT eql_v2.add_column('users', 'encrypted_data', 'jsonb'); + /// ``` + fn add_column() -> &'static dyn Component; + + /// Remove column configuration completely. + /// + /// # Examples + /// + /// ```sql + /// SELECT eql_v2.remove_column('users', 'encrypted_email'); + /// ``` + fn remove_column() -> &'static dyn Component; + + /// Add a searchable index to an encrypted column. + /// + /// # Supported index types + /// + /// - `unique` - Exact equality (uses hmac_256 or blake3) + /// - `match` - Full-text search (uses bloom_filter) + /// - `ore` - Range queries and ordering (uses ore_block_u64_8_256) + /// - `ste_vec` - JSONB containment queries (uses structured encryption) + /// + /// # Examples + /// + /// ```sql + /// SELECT eql_v2.add_search_config('users', 'encrypted_email', 'unique', 'text'); + /// SELECT eql_v2.add_search_config('docs', 'encrypted_content', 'match', 'text'); + /// ``` + fn add_search_config() -> &'static dyn Component; +} +``` + +**Step 5: Run test to verify it passes** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo test --package eql-core` + +Expected: PASS (1 test) + +**Step 6: Commit** + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling +git add eql-core/ +git commit -m "feat(eql-core): define Component and Config traits + +Add core trait system for EQL API: +- Component trait: Represents SQL file with type-safe dependencies +- Dependencies trait: Marker for dependency specifications +- Config trait: Configuration management API with rustdoc examples + +The Config trait includes documentation that will be auto-generated +into customer-facing docs, preventing documentation drift." +``` + +--- + +## Task 3: Create Test Harness with Transaction Isolation + +**Files:** +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-test/src/lib.rs` + +**Step 1: Write test for TestDb** + +Create `eql-test/src/lib.rs`: + +```rust +//! Test harness providing transaction isolation for SQL tests + +use anyhow::{Context, Result}; +use tokio_postgres::{Client, NoTls, Row}; + +pub struct TestDb { + client: Client, + in_transaction: bool, +} + +impl TestDb { + /// Create new test database with transaction isolation + pub async fn new() -> Result { + let (client, connection) = tokio_postgres::connect( + &Self::connection_string(), + NoTls, + ) + .await + .context("Failed to connect to test database")?; + + // Spawn connection handler + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("Connection error: {}", e); + } + }); + + // Begin transaction for isolation + client.execute("BEGIN", &[]).await?; + + Ok(Self { + client, + in_transaction: true, + }) + } + + fn connection_string() -> String { + std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| "host=localhost port=7432 user=cipherstash password=password dbname=postgres".to_string()) + } + + /// Execute SQL (for setup/implementation loading) + pub async fn execute(&self, sql: &str) -> Result { + self.client.execute(sql, &[]) + .await + .context("Failed to execute SQL") + } + + /// Query with single result + pub async fn query_one(&self, sql: &str) -> Result { + self.client.query_one(sql, &[]) + .await + .context("Failed to query") + } + + /// Assert JSONB result has key + pub fn assert_jsonb_has_key(&self, result: &Row, column_index: usize, key: &str) -> Result<()> { + let json: serde_json::Value = result.get(column_index); + anyhow::ensure!( + json.get(key).is_some(), + "Expected key '{}' in result, got: {:?}", + key, + json + ); + Ok(()) + } +} + +impl Drop for TestDb { + fn drop(&mut self) { + if self.in_transaction { + // Auto-rollback on drop + // Note: Can't use async in Drop, but connection will rollback anyway + // when client drops + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_testdb_transaction_isolation() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Create a temporary table + db.execute("CREATE TEMPORARY TABLE test_table (id int, value text)") + .await + .expect("Failed to create table"); + + // Insert data + db.execute("INSERT INTO test_table VALUES (1, 'test')") + .await + .expect("Failed to insert"); + + // Query data + let row = db.query_one("SELECT value FROM test_table WHERE id = 1") + .await + .expect("Failed to query"); + + let value: String = row.get(0); + assert_eq!(value, "test"); + + // Transaction will rollback on drop - table won't exist in next test + } +} +``` + +**Step 2: Run test to verify it compiles** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo test --package eql-test` + +Expected: May fail if PostgreSQL container not running, but should compile + +**Step 3: Start PostgreSQL container for testing** + +Run: `cd /Users/tobyhede/src/encrypt-query-language && mise run postgres:up postgres-17 --extra-args "--detach --wait"` + +Expected: PostgreSQL container starts successfully + +**Step 4: Run test to verify it passes** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo test --package eql-test` + +Expected: PASS (1 test) - transaction isolation working + +**Step 5: Commit** + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling +git add eql-test/ +git commit -m "feat(eql-test): add test harness with transaction isolation + +Create TestDb struct providing: +- Automatic transaction BEGIN on creation +- Auto-rollback on drop (clean slate for next test) +- Helper methods: execute(), query_one() +- Assertion helpers: assert_jsonb_has_key() + +This solves the current testing pain points: +- No more manual database resets between tests +- Clear error messages (no more block-level PostgreSQL ASSERT errors) +- Foundation for parallel test execution (future enhancement)" +``` + +--- + +## Task 4: Implement PostgreSQL Config Module + +**Files:** +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-postgres/src/lib.rs` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-postgres/src/config.rs` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-postgres/src/sql/config/types.sql` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-postgres/src/sql/config/add_column.sql` + +**Step 1: Copy existing SQL files** + +Copy the current implementation from main codebase: + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling +mkdir -p eql-postgres/src/sql/config +cp ../../src/config/types.sql eql-postgres/src/sql/config/types.sql +cp ../../src/config/functions.sql eql-postgres/src/sql/config/add_column_temp.sql +``` + +**Step 2: Extract add_column function into separate file** + +From `add_column_temp.sql`, extract just the `add_column` function: + +Create `eql-postgres/src/sql/config/add_column.sql`: + +```sql +-- Add a column for encryption/decryption +-- +-- This function initializes a column to work with CipherStash encryption. +-- The column must be of type eql_v2_encrypted. + +CREATE FUNCTION eql_v2.add_column(table_name text, column_name text, cast_as text DEFAULT 'text', migrating boolean DEFAULT false) + RETURNS jsonb + +AS $$ + DECLARE + o jsonb; + _config jsonb; + BEGIN + + -- set the active config + SELECT data INTO _config FROM public.eql_v2_configuration WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + + -- set default config + SELECT eql_v2.config_default(_config) INTO _config; + + -- if index exists + IF _config #> array['tables', table_name] ? column_name THEN + RAISE EXCEPTION 'Config exists for column: % %', table_name, column_name; + END IF; + + SELECT eql_v2.config_add_table(table_name, _config) INTO _config; + + SELECT eql_v2.config_add_column(table_name, column_name, _config) INTO _config; + + SELECT eql_v2.config_add_cast(table_name, column_name, cast_as, _config) INTO _config; + + -- create a new pending record if we don't have one + INSERT INTO public.eql_v2_configuration (state, data) VALUES ('pending', _config) + ON CONFLICT (state) + WHERE state = 'pending' + DO UPDATE + SET data = _config; + + IF NOT migrating THEN + PERFORM eql_v2.migrate_config(); + PERFORM eql_v2.activate_config(); + END IF; + + PERFORM eql_v2.add_encrypted_constraint(table_name, column_name); + + -- exeunt + RETURN _config; + END; +$$ LANGUAGE plpgsql; +``` + +**Step 3: Implement Rust component definitions** + +Create `eql-postgres/src/config.rs`: + +```rust +//! PostgreSQL implementation of Config trait + +use eql_core::{Component, Config, Dependencies}; + +pub struct ConfigTypes; + +impl Component for ConfigTypes { + type Dependencies = (); + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/config/types.sql" + ) + } +} + +impl Dependencies for ConfigTypes {} + +pub struct AddColumn; + +impl Component for AddColumn { + type Dependencies = ConfigTypes; + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/config/add_column.sql" + ) + } +} + +pub struct PostgresEQL; + +impl Config for PostgresEQL { + fn add_column() -> &'static dyn Component { + &AddColumn + } + + fn remove_column() -> &'static dyn Component { + todo!("Not implemented in POC") + } + + fn add_search_config() -> &'static dyn Component { + todo!("Not implemented in POC") + } +} +``` + +Create `eql-postgres/src/lib.rs`: + +```rust +//! PostgreSQL implementation of EQL + +pub mod config; + +pub use config::PostgresEQL; +``` + +**Step 4: Verify it compiles** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo build --package eql-postgres` + +Expected: Builds successfully + +**Step 5: Commit** + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling +git add eql-postgres/ +git commit -m "feat(eql-postgres): implement Config trait with SQL files + +Add PostgreSQL implementation: +- ConfigTypes component (wraps config/types.sql) +- AddColumn component (wraps config/add_column.sql) +- PostgresEQL struct implementing Config trait + +SQL files copied from existing implementation (src/config/). +Component system provides type-safe dependency: AddColumn depends on ConfigTypes. + +This proves the concept of Rust + SQL file references working together." +``` + +--- + +## Task 5: Write Integration Test for add_column + +**Files:** +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-postgres/tests/config_test.rs` + +**Step 1: Write failing test** + +Create `eql-postgres/tests/config_test.rs`: + +```rust +use eql_postgres::config::{AddColumn, ConfigTypes, PostgresEQL}; +use eql_core::{Component, Config}; +use eql_test::TestDb; + +#[tokio::test] +async fn test_add_column_creates_config() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Load schema (from main project) + let schema_sql = include_str!("../../../src/schema.sql"); + db.execute(schema_sql).await.expect("Failed to create schema"); + + // Load config types + let types_sql = std::fs::read_to_string(ConfigTypes::sql_file()) + .expect("Failed to read types.sql"); + db.execute(&types_sql).await.expect("Failed to load config types"); + + // Load add_column function + let add_column_sql = std::fs::read_to_string(AddColumn::sql_file()) + .expect("Failed to read add_column.sql"); + db.execute(&add_column_sql).await.expect("Failed to load add_column"); + + // Setup: Create test table + db.execute("CREATE TABLE users (id int)").await.expect("Failed to create table"); + + // Execute: Call add_column + let result = db.query_one("SELECT eql_v2.add_column('users', 'email', 'text')") + .await + .expect("Failed to call add_column"); + + // Assert: Result has 'tables' key + db.assert_jsonb_has_key(&result, 0, "tables") + .expect("Expected 'tables' key in config"); +} +``` + +**Step 2: Run test to verify it fails** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo test --package eql-postgres --test config_test` + +Expected: FAIL - SQL files likely have missing dependencies (functions_private, encrypted/functions, etc.) + +**Step 3: Fix dependencies by loading required SQL** + +This will fail because `add_column` depends on helper functions not yet migrated. For POC, we'll note this and create a simpler test: + +Update `eql-postgres/tests/config_test.rs`: + +```rust +use eql_postgres::config::{ConfigTypes, PostgresEQL}; +use eql_core::{Component, Config}; +use eql_test::TestDb; + +#[tokio::test] +async fn test_config_types_loads() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Load config types + let types_sql = std::fs::read_to_string(ConfigTypes::sql_file()) + .expect("Failed to read types.sql"); + db.execute(&types_sql).await.expect("Failed to load config types"); + + // Verify enum type exists + let result = db.query_one( + "SELECT EXISTS ( + SELECT 1 FROM pg_type + WHERE typname = 'eql_v2_configuration_state' + )" + ).await.expect("Failed to check type"); + + let exists: bool = result.get(0); + assert!(exists, "eql_v2_configuration_state type should exist"); +} + +#[tokio::test] +async fn test_component_sql_file_paths_valid() { + // Verify SQL files exist at the paths components claim + let types_path = ConfigTypes::sql_file(); + assert!( + std::path::Path::new(types_path).exists(), + "ConfigTypes SQL file should exist at {}", + types_path + ); +} +``` + +**Step 4: Run test to verify it passes** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo test --package eql-postgres --test config_test` + +Expected: PASS (2 tests) + +**Step 5: Commit** + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling +git add eql-postgres/tests/ +git commit -m "test(eql-postgres): add integration tests for Config module + +Add two tests: +1. test_config_types_loads: Verifies SQL file loads and creates enum type +2. test_component_sql_file_paths_valid: Verifies Component trait points to real files + +These tests demonstrate: +- TestDb transaction isolation working +- SQL file references from Rust components working +- Foundation for full integration testing + +Note: Full add_column test deferred - requires migrating helper functions +(config_default, config_add_table, etc.) which is beyond POC scope." +``` + +--- + +## Task 6: Create Build Tool Prototype + +**Files:** +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-build/src/main.rs` +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-build/src/graph.rs` + +**Step 1: Write test for build tool** + +Create `eql-build/src/main.rs`: + +```rust +//! Build tool for extracting SQL files in dependency order + +use anyhow::{Context, Result}; +use std::fs; + +mod graph; + +fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + + if args.len() < 2 { + eprintln!("Usage: eql-build "); + eprintln!(" database: postgres, mysql, etc."); + std::process::exit(1); + } + + let database = &args[1]; + + match database.as_str() { + "postgres" => build_postgres()?, + _ => anyhow::bail!("Unknown database: {}", database), + } + + Ok(()) +} + +fn build_postgres() -> Result<()> { + use eql_postgres::config::{ConfigTypes, AddColumn}; + use eql_core::Component; + + println!("Building PostgreSQL installer..."); + + // For POC: Simple sequential build + // Future: Use graph module for dependency resolution + let mut output = String::new(); + + // Add header + output.push_str("-- CipherStash EQL for PostgreSQL\n"); + output.push_str("-- Generated by eql-build\n\n"); + + // Add ConfigTypes + let types_sql = fs::read_to_string(ConfigTypes::sql_file()) + .context("Failed to read config types SQL")?; + output.push_str(&types_sql); + output.push_str("\n\n"); + + // Add AddColumn + let add_column_sql = fs::read_to_string(AddColumn::sql_file()) + .context("Failed to read add_column SQL")?; + output.push_str(&add_column_sql); + output.push_str("\n\n"); + + // Write output + fs::create_dir_all("release")?; + fs::write("release/cipherstash-encrypt-postgres-poc.sql", output)?; + + println!("✓ Generated release/cipherstash-encrypt-postgres-poc.sql"); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_creates_output_file() { + // Clean up any previous output + let _ = std::fs::remove_file("release/cipherstash-encrypt-postgres-poc.sql"); + + // Run build + build_postgres().expect("Build should succeed"); + + // Verify output exists + assert!( + std::path::Path::new("release/cipherstash-encrypt-postgres-poc.sql").exists(), + "Build should create output file" + ); + + // Verify it contains expected SQL + let content = std::fs::read_to_string("release/cipherstash-encrypt-postgres-poc.sql") + .expect("Should be able to read output"); + + assert!(content.contains("eql_v2_configuration_state"), "Should contain config types"); + assert!(content.contains("CREATE FUNCTION eql_v2.add_column"), "Should contain add_column function"); + } +} +``` + +Create `eql-build/src/graph.rs`: + +```rust +//! Dependency graph for topological sorting (future enhancement) + +// Placeholder for dependency graph implementation +// Will use Component::Dependencies to build DAG and topologically sort +``` + +**Step 2: Run test to verify it fails** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo test --package eql-build` + +Expected: FAIL - release directory doesn't exist yet, or SQL missing + +**Step 3: Run build to create output** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo run --bin eql-build postgres` + +Expected: Creates `release/cipherstash-encrypt-postgres-poc.sql` + +**Step 4: Run test to verify it passes** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo test --package eql-build` + +Expected: PASS (1 test) + +**Step 5: Verify generated SQL is valid** + +Run: `cd /Users/tobyhede/src/encrypt-query-language && psql -h localhost -p 7432 -U cipherstash -d postgres < .worktrees/rust-sql-tooling/release/cipherstash-encrypt-postgres-poc.sql` + +Expected: SQL executes without errors (though may be incomplete due to dependencies) + +**Step 6: Commit** + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling +git add eql-build/ release/ +git commit -m "feat(eql-build): add build tool for SQL extraction + +Create build tool that: +- Reads SQL files via Component::sql_file() paths +- Concatenates in dependency order (manual for POC) +- Generates release/cipherstash-encrypt-postgres-poc.sql + +Future enhancements: +- Automatic dependency graph resolution via Component::Dependencies +- Topological sort using graph module +- Support for multiple database targets + +This proves SQL extraction from Rust component system works." +``` + +--- + +## Task 7: Add Error Handling with thiserror + +**Files:** +- Create: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-core/src/error.rs` +- Modify: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-core/src/lib.rs` +- Modify: `/Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling/eql-test/src/lib.rs` +- Update: Workspace Cargo.toml to add thiserror dependency + +**Step 1: Add thiserror to workspace dependencies** + +Update root `Cargo.toml`: + +```toml +[workspace.dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.35", features = ["full"] } +tokio-postgres = "0.7" +anyhow = "1.0" +thiserror = "1.0" +``` + +**Step 2: Define error hierarchy** + +Create `eql-core/src/error.rs`: + +```rust +//! Error types for EQL operations + +use thiserror::Error; + +/// Top-level error type for all EQL operations +#[derive(Error, Debug)] +pub enum EqlError { + #[error("Component error: {0}")] + Component(#[from] ComponentError), + + #[error("Database error: {0}")] + Database(#[from] DatabaseError), +} + +/// Errors related to SQL components and dependencies +#[derive(Error, Debug)] +pub enum ComponentError { + #[error("SQL file not found: {path}")] + SqlFileNotFound { path: String }, + + #[error("Dependency cycle detected: {cycle}")] + DependencyCycle { cycle: String }, + + #[error("IO error reading SQL file {path}: {source}")] + IoError { + path: String, + #[source] + source: std::io::Error, + }, +} + +/// Errors related to database operations +#[derive(Error, Debug)] +pub enum DatabaseError { + #[error("Connection failed: {0}")] + Connection(#[source] tokio_postgres::Error), + + #[error("Transaction failed: {0}")] + Transaction(String), + + #[error("Query failed: {query}: {source}")] + Query { + query: String, + #[source] + source: tokio_postgres::Error, + }, + + #[error("Expected JSONB value to have key '{key}', got: {actual}")] + MissingJsonbKey { + key: String, + actual: serde_json::Value, + }, +} +``` + +**Step 3: Update eql-core to export error types** + +Update `eql-core/src/lib.rs`: + +```rust +//! EQL Core - Trait definitions for multi-database SQL extension API + +pub mod component; +pub mod config; +pub mod error; + +pub use component::{Component, Dependencies}; +pub use config::Config; +pub use error::{ComponentError, DatabaseError, EqlError}; + +// ... rest of file +``` + +Update `eql-core/Cargo.toml`: + +```toml +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio-postgres = { workspace = true } +``` + +**Step 4: Update TestDb to use structured errors** + +Update `eql-test/src/lib.rs`: + +```rust +use eql_core::error::DatabaseError; + +pub struct TestDb { + // ... fields unchanged +} + +impl TestDb { + // ... new() unchanged ... + + pub async fn execute(&self, sql: &str) -> Result { + self.client + .execute(sql, &[]) + .await + .map_err(|e| DatabaseError::Query { + query: sql.to_string(), + source: e, + }) + } + + pub async fn query_one(&self, sql: &str) -> Result { + self.client + .query_one(sql, &[]) + .await + .map_err(|e| DatabaseError::Query { + query: sql.to_string(), + source: e, + }) + } + + pub fn assert_jsonb_has_key(&self, result: &Row, column_index: usize, key: &str) -> Result<(), DatabaseError> { + let json: serde_json::Value = result.get(column_index); + if json.get(key).is_none() { + return Err(DatabaseError::MissingJsonbKey { + key: key.to_string(), + actual: json, + }); + } + Ok(()) + } +} +``` + +Update `eql-test/Cargo.toml`: + +```toml +[dependencies] +eql-core = { path = "../eql-core" } +tokio = { workspace = true } +tokio-postgres = { workspace = true } +serde_json = { workspace = true } +``` + +**Step 5: Run tests to verify error handling works** + +Run: `cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling && cargo test --all` + +Expected: All existing tests still pass, now with better error messages + +**Step 6: Test error messages manually** + +Add a test to `eql-test/src/lib.rs`: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + // ... existing test ... + + #[tokio::test] + async fn test_database_error_messages() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Test query error with helpful context + let result = db.execute("INVALID SQL SYNTAX").await; + assert!(result.is_err()); + + let err = result.unwrap_err(); + let err_string = err.to_string(); + assert!(err_string.contains("Query failed")); + assert!(err_string.contains("INVALID SQL SYNTAX")); + } +} +``` + +**Step 7: Commit** + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling +git add eql-core/src/error.rs eql-core/src/lib.rs eql-core/Cargo.toml eql-test/src/lib.rs eql-test/Cargo.toml Cargo.toml +git commit -m "feat: add structured error handling with thiserror + +Add error hierarchy: +- EqlError: Top-level error type +- ComponentError: SQL file and dependency errors +- DatabaseError: Database operation errors + +Benefits: +- Clear error messages with context (e.g., which query failed) +- Type-safe error handling throughout the codebase +- Better debugging experience for tests and build tools + +TestDb now returns structured DatabaseError instead of anyhow::Error, +providing detailed context about failed queries." +``` + +--- + +## Verification Checklist + +Run these commands to verify POC is complete: + +```bash +cd /Users/tobyhede/src/encrypt-query-language/.worktrees/rust-sql-tooling + +# All tests pass +cargo test --all + +# Build tool generates SQL +cargo run --bin eql-build postgres +ls -lh release/cipherstash-encrypt-postgres-poc.sql + +# Generated SQL is valid (basic check) +# Note: May have dependency errors but should parse +head -20 release/cipherstash-encrypt-postgres-poc.sql + +# Rustdoc generates customer-facing documentation +cargo doc --no-deps --open +# Verify in browser: Config trait docs show SQL examples and are customer-friendly +``` + +Expected results: +- [ ] All tests pass (5+ tests across crates) +- [ ] SQL installer generated (~50+ lines) +- [ ] Rustdoc HTML generated with Config trait documentation +- [ ] Config trait docs include SQL examples (not Rust usage) +- [ ] No Rust compilation errors + +--- + +## Future Work (Out of Scope for POC) + +The following are explicitly deferred to focus POC on proving the concept: + +1. **Full dependency graph resolution** - Currently manual ordering, should use Component::Dependencies for automatic topological sort +2. **Complete Config module migration** - Only types and add_column migrated, need full module +3. **Parallel test execution** - TestDb supports isolation, need test runner enhancements +4. **Multiple database support** - PostgreSQL only for POC, MySQL/SQLite deferred +5. **Feature trait system** - Ore32Bit, Ore64Bit, etc. not yet implemented +6. **Integration with existing build** - POC is standalone, needs integration with mise tasks +7. **Component-level error handling** - Expand error types as more components are added + +--- + +## Success Criteria Review + +After completing all tasks, verify: + +- [ ] Rust workspace compiles successfully +- [ ] Core trait system defined (Component, Config traits) +- [ ] PostgreSQL implementation of Config module functional +- [ ] Test harness provides transaction isolation +- [ ] Build tool generates valid SQL installer +- [ ] Rustdoc generates customer-facing API documentation from trait comments +- [ ] Config types and add_column migrated with tests passing +- [ ] Structured error handling with thiserror in place + +--- + +## Notes for Implementation + +**Testing approach:** +- Use TDD at task level (write test → verify fail → implement → verify pass) +- Transaction isolation ensures tests don't interfere +- Integration tests verify SQL loads correctly + +**Dependency management:** +- POC uses manual ordering in build tool +- Component::Dependencies types are defined but not yet used for graph walking +- Full implementation would traverse type graph at build time + +**Documentation:** +- Rustdoc comments in traits are written for customers (showing SQL examples, not Rust usage) +- `cargo doc` generates HTML documentation directly from trait definitions +- Key insight: Single source of truth prevents drift - no separate markdown files to maintain + +**Error handling:** +- Structured error types using thiserror +- DatabaseError provides query context for better debugging +- ComponentError will handle SQL file and dependency issues +- Expand error types organically as needs arise + +**Migration strategy:** +- Start with one module (Config) to prove concept +- Future modules can follow same pattern +- Existing SQL files are source of truth, just referenced differently diff --git a/docs/plans/2025-10-22-pgrx-investigation-report.md b/docs/plans/2025-10-22-pgrx-investigation-report.md new file mode 100644 index 0000000..fd15488 --- /dev/null +++ b/docs/plans/2025-10-22-pgrx-investigation-report.md @@ -0,0 +1,671 @@ +# pgrx Investigation Report + +**Date:** 2025-10-22 +**Investigator:** Toby Hede +**Status:** Complete - pgrx Not Feasible + +## Executive Summary + +**Question:** Can pgrx be used for EQL, given the constraint that EQL must remain a SQL file installer (not a compiled extension)? + +**Answer:** No. pgrx is fundamentally incompatible with EQL's SQL-only deployment model. + +**Recommendation:** Continue developing the custom Rust tooling framework in `feature/rust-sql-tooling`. The existing spike already provides superior solutions for EQL's specific needs. + +--- + +## Investigation Context + +### EQL's Current Pain Points + +1. **No test framework** - Manual Docker setup, no transaction isolation +2. **Brittle dependencies** - String parsing of `-- REQUIRE:` comments, `tsort` for ordering +3. **No documentation tooling** - Cannot embed docs with source +4. **Poor dev experience** - No type safety, no compile-time validation +5. **Minimal release support** - Basic build script with limited automation + +### Why pgrx Seemed Promising + +pgrx is a mature Rust framework for PostgreSQL extensions offering: +- Built-in testing framework (`#[pg_test]`) +- Automatic dependency management +- Multi-version PostgreSQL support (pg14, pg15, pg16, pg17) +- SQL schema generation +- Documentation via rustdoc +- Sophisticated build tooling + +### The Critical Constraint + +**EQL must ship as pure SQL files**, not compiled binary extensions. This is non-negotiable for: +- Compatibility with restrictive environments (Supabase, managed PostgreSQL) +- Simple installation via SQL script execution +- No platform-specific binaries to maintain +- Transparent, auditable source code + +--- + +## pgrx Analysis + +### How pgrx Works + +``` +┌─────────────────────────────────────────────────────────────┐ +│ pgrx Model │ +├─────────────────────────────────────────────────────────────┤ +│ Rust Code → Compile → .so Binary + SQL Schema → Install │ +│ │ +│ SQL = Interface (calls into Rust) │ +│ Rust = Implementation (compiled functions) │ +└─────────────────────────────────────────────────────────────┘ +``` + +**Key characteristics:** +- Generates compiled `.so` shared libraries +- SQL schemas reference functions implemented in the binary +- `cargo pgrx package` creates extension bundles +- Testing requires compiled extension to run +- All features assume Rust is the implementation language + +### SQL Generation Capability + +**Research finding:** pgrx does generate SQL files, but these are **not standalone**. + +From documentation: +> "The generated SQL files describe the extension's SQL interface (functions, types, etc.) but cannot function independently—they require the compiled binary component that pgrx produces from your Rust code." + +Example generated SQL: +```sql +CREATE FUNCTION my_function(arg text) RETURNS text +AS 'MODULE_PATHNAME', 'my_function_wrapper' +LANGUAGE C STRICT; +``` + +The `MODULE_PATHNAME` references the compiled `.so` file. **The SQL cannot run without the binary.** + +### Testing Framework (`#[pg_test]`) + +**Research finding:** Cannot be used standalone. + +- `#[pg_test]` requires a compiled extension +- Tests run "in-process inside Postgres during `cargo pgrx test`" +- The test macro depends on pgrx's extension loading mechanism +- No documented way to use testing infrastructure independently + +**Workaround suggested in docs:** Use standard `#[test]` instead of `#[pg_test]` - but this loses PostgreSQL integration. + +### Component Extraction Feasibility + +| pgrx Component | Can Use Standalone? | Reason | +|----------------|---------------------|--------| +| Testing (`#[pg_test]`) | ❌ No | Requires compiled extension | +| Schema generation | ❌ No | Generates SQL that calls compiled code | +| Build tooling | ❌ No | Designed for `.so` output, not SQL concatenation | +| Multi-version support | ⚠️ Conceptually only | Implementation tied to extension compilation | +| Type mappings | ❌ No | For Rust↔PostgreSQL FFI, not applicable to SQL | +| SQL macros | ❌ No | For embedding SQL in Rust, not managing SQL files | +| Dependency management | ❌ No | Uses Cargo, assumes Rust modules | + +**Conclusion:** No meaningful components can be extracted for SQL-only workflow. + +--- + +## Custom Spike Analysis + +### What `feature/rust-sql-tooling` Achieves + +The existing spike provides a **purpose-built Rust framework for SQL-first extensions**. + +#### Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ EQL Model │ +├─────────────────────────────────────────────────────────────┤ +│ SQL Files → Resolve Deps → Concatenate → Install │ +│ │ +│ SQL = Implementation (the actual extension) │ +│ Rust = Tooling (dependency mgmt, testing, docs) │ +└─────────────────────────────────────────────────────────────┘ +``` + +#### Crates + +**`eql-core`** - Core abstractions +- `Component` trait for SQL file dependencies +- `Dependencies` trait for type-level dependency tracking +- Compile-time dependency resolution via Rust's type system + +**`eql-postgres`** - PostgreSQL-specific components +- Concrete `Component` implementations for EQL SQL files +- Type-safe dependency declarations (e.g., `type Dependencies = (A, B, C)`) +- API trait (`Config`) describing EQL capabilities + +**`eql-test`** - Testing infrastructure +- `TestDb` with automatic transaction isolation +- Async PostgreSQL client via tokio-postgres +- Helper assertions (e.g., `assert_jsonb_has_key`) +- Automatic rollback on test completion + +**`eql-build`** - Build tooling +- SQL file concatenation in dependency order +- Removal of `-- REQUIRE:` metadata comments +- Single `.sql` file output to `release/` + +#### Example: Type-Safe Dependencies + +```rust +// Base component +pub struct ConfigTypes; +impl Component for ConfigTypes { + type Dependencies = (); // No dependencies + fn sql_file() -> &'static str { + "src/sql/config/types.sql" + } +} + +// Component depending on ConfigTypes +pub struct ConfigTables; +impl Component for ConfigTables { + type Dependencies = ConfigTypes; // Type-safe dependency + fn sql_file() -> &'static str { + "src/sql/config/tables.sql" + } +} + +// Component with multiple dependencies +pub struct AddColumn; +impl Component for AddColumn { + type Dependencies = ( + ConfigPrivateFunctions, + MigrateActivate, + AddEncryptedConstraint, + ConfigTypes, + ); + fn sql_file() -> &'static str { + "src/sql/config/add_column.sql" + } +} +``` + +**Key benefit:** Dependencies are validated at compile time. Missing or circular dependencies cause compiler errors. + +#### Example: Testing + +```rust +#[tokio::test] +async fn test_add_column_creates_config() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Load dependencies automatically + let deps = AddColumn::collect_dependencies(); + for sql_file in deps { + let sql = std::fs::read_to_string(sql_file).unwrap(); + db.batch_execute(&sql).await.unwrap(); + } + + // Setup test data + db.execute("CREATE TABLE users (id int, email eql_v2_encrypted)") + .await.expect("Failed to create table"); + + // Execute function under test + let result = db.query_one( + "SELECT eql_v2.add_column('users', 'email', 'text')" + ).await.expect("Failed to call add_column"); + + // Assert results + db.assert_jsonb_has_key(&result, 0, "tables") + .expect("Expected 'tables' key in config"); + + // Transaction auto-rolls back on drop +} +``` + +**Key benefits:** +- Transaction isolation (no test pollution) +- Automatic dependency loading +- Type-safe database interactions +- Standard Rust testing tools (`cargo test`) + +#### Example: Build Process + +```rust +fn build_postgres() -> Result<()> { + use eql_postgres::config::AddColumn; + use eql_core::Component; + + let mut builder = Builder::new("CipherStash EQL for PostgreSQL"); + + // Automatic dependency resolution + let deps = AddColumn::collect_dependencies(); + println!("Resolved {} dependencies", deps.len()); + + for sql_file in deps { + builder.add_sql_file(sql_file)?; + } + + // Write single SQL file + let output = builder.build(); + fs::write("release/cipherstash-encrypt-postgres-poc.sql", output)?; + + Ok(()) +} +``` + +**Output:** Single `release/cipherstash-encrypt-postgres-poc.sql` file with correct dependency ordering. + +--- + +## Comparison Matrix + +### Capabilities + +| Capability | EQL Needs | Custom Spike | pgrx | Winner | +|------------|-----------|--------------|------|--------| +| **SQL-only output** | ✅ Required | ✅ Yes | ❌ Requires `.so` | **Spike** | +| **Type-safe dependencies** | ✅ Required | ✅ `Component` trait | ✅ Cargo modules | **Tie** | +| **Auto dependency resolution** | ✅ Required | ✅ `collect_dependencies()` | ✅ Cargo | **Tie** | +| **Testing framework** | ✅ Required | ✅ `TestDb` + tokio | ✅ `#[pg_test]` | **Spike** (SQL-compatible) | +| **Transaction isolation** | ✅ Nice to have | ✅ Built-in | ✅ Built-in | **Tie** | +| **Multi-version PostgreSQL** | ✅ Required | ⚠️ Manual Docker | ✅ `cargo pgrx test pg14 pg15...` | **pgrx** | +| **Documentation** | ✅ Required | ✅ Can use rustdoc | ✅ rustdoc | **Tie** | +| **Build tool** | ✅ Required | ✅ `eql-build` | ✅ `cargo pgrx package` | **Spike** (for SQL) | +| **SQL preservation** | ✅ **BLOCKER** | ✅ SQL stays SQL | ❌ Rust replaces SQL | **Spike** | + +### Architecture Alignment + +| Aspect | Custom Spike | pgrx | +|--------|--------------|------| +| **Philosophy** | SQL-first, Rust as tooling | Rust-first, SQL as interface | +| **Output artifact** | `.sql` file | `.so` + `.sql` bundle | +| **Installation** | `psql < install.sql` | `CREATE EXTENSION` | +| **Implementation language** | SQL | Rust | +| **Deployment complexity** | Low (single SQL file) | Medium (platform-specific binaries) | +| **Supabase compatibility** | ✅ Yes | ❌ No (no binary extensions) | + +**Verdict:** Custom spike is architecturally aligned with EQL's needs. pgrx solves a different problem. + +--- + +## What We Learned from pgrx (Conceptual Inspiration) + +While pgrx's code is incompatible, we can steal ideas: + +### 1. Multi-Version Testing Ergonomics + +**pgrx approach:** `cargo pgrx test pg14 pg15 pg16 pg17` + +**Adaptation for EQL:** +```rust +// eql-test/src/lib.rs +pub enum PostgresVersion { + Pg14, Pg15, Pg16, Pg17 +} + +impl TestDb { + pub async fn new_with_version(version: PostgresVersion) -> Result { + let port = match version { + PostgresVersion::Pg14 => 7414, + PostgresVersion::Pg15 => 7415, + PostgresVersion::Pg16 => 7416, + PostgresVersion::Pg17 => 7417, + }; + // Connect to Docker container for that version + } +} + +// Test macro could expand to run across all versions +#[eql_test(all_versions)] +async fn test_add_column_works() { + // Automatically runs against pg14, pg15, pg16, pg17 +} +``` + +### 2. Schema Introspection + +**pgrx approach:** Extract function signatures from Rust code for SQL generation + +**Adaptation for EQL:** +```rust +trait Component { + // ... existing methods ... + + /// Extract documentation from SQL file + fn documentation() -> Option<&'static str> { + None + } + + /// List public API functions this component provides + fn public_functions() -> Vec { + vec![] + } +} + +// Could generate reference docs from this metadata +``` + +### 3. Dependency Visualization + +**Possible addition:** +```bash +$ cargo run --bin eql-deps -- --graph AddColumn +digraph { + AddColumn -> ConfigPrivateFunctions + AddColumn -> MigrateActivate + AddColumn -> AddEncryptedConstraint + ConfigPrivateFunctions -> ConfigTypes + MigrateActivate -> ConfigIndexes + ConfigIndexes -> ConfigTables + ConfigTables -> ConfigTypes +} +``` + +--- + +## Identified Gaps in Custom Spike + +| Gap | Current State | Impact | Priority | +|-----|---------------|--------|----------| +| **Multi-version PostgreSQL** | Manual Docker in `tests/docker-compose.yml` | High - testing across versions is tedious | High | +| **Test discovery** | Manual `#[tokio::test]` per file | Medium - no auto-discovery of SQL test files | Medium | +| **Documentation extraction** | Rustdoc for Rust types only | Medium - SQL functions not documented | Medium | +| **Release automation** | Basic `eql-build` | Low - versioning/changelog manual | Low | +| **Dependency visualization** | Implicit in type graph | Low - hard to understand dependency tree | Low | +| **Partial component loading** | All-or-nothing via `collect_dependencies()` | Low - can't test subsets easily | Low | + +--- + +## Alternative Tools Considered + +Since pgrx doesn't fit, these tools could address specific gaps: + +### pgTAP +**What:** PostgreSQL testing framework with TAP protocol +**How it could help:** SQL-native test assertions callable from Rust +**Integration idea:** +```rust +#[tokio::test] +async fn test_with_pgtap() { + let db = TestDb::new().await?; + db.batch_execute(include_str!("pgtap.sql")).await?; + + db.execute("SELECT plan(3);").await?; + db.execute("SELECT has_function('eql_v2', 'add_column');").await?; + db.execute("SELECT function_returns('eql_v2', 'add_column', 'jsonb');").await?; + db.execute("SELECT finish();").await?; +} +``` + +### sqitch +**What:** Database change management with dependency tracking +**How it could help:** Inspiration for migration/versioning metadata +**Not recommended:** Adds complexity for minimal benefit given existing `Component` approach + +### dbmate +**What:** Simple schema migration tool +**How it could help:** SQL file ordering patterns +**Not recommended:** Less powerful than custom `Component` system + +### pgx_scripts (Supabase dbdev) +**What:** Supabase's extension packaging for SQL-only extensions +**How it could help:** Learn from their SQL packaging approach +**Worth exploring:** Could inform release/distribution strategy + +--- + +## Recommendations + +### 1. ✅ Commit to Custom Rust Tooling + +**Decision:** Continue developing `feature/rust-sql-tooling` as the foundation for EQL development. + +**Rationale:** +- Already solves core problems (type-safe deps, testing, build) +- Architecturally aligned with SQL-first approach +- pgrx offers no viable path forward +- Investment already made in spike validates approach + +### 2. Address High-Priority Gaps + +**Immediate improvements:** + +#### Multi-Version PostgreSQL Testing + +Create `eql-test/src/postgres_version.rs`: +```rust +pub enum PostgresVersion { + Pg14, Pg15, Pg16, Pg17 +} + +impl PostgresVersion { + pub fn all() -> Vec { + vec![Self::Pg14, Self::Pg15, Self::Pg16, Self::Pg17] + } + + pub fn container_name(&self) -> &'static str { + match self { + Self::Pg14 => "postgres-14", + Self::Pg15 => "postgres-15", + Self::Pg16 => "postgres-16", + Self::Pg17 => "postgres-17", + } + } + + pub async fn ensure_running(&self) -> Result<()> { + // Docker container management + } +} +``` + +Add `eql-test/src/macros.rs`: +```rust +#[macro_export] +macro_rules! test_all_versions { + ($name:ident, $body:expr) => { + #[tokio::test] + async fn $name() { + for version in PostgresVersion::all() { + version.ensure_running().await.unwrap(); + let db = TestDb::new_with_version(version).await.unwrap(); + $body(db).await; + } + } + }; +} +``` + +Usage: +```rust +test_all_versions!(test_add_column, |db| async move { + // Test runs against all PostgreSQL versions +}); +``` + +#### Documentation Extraction + +Add to `Component` trait: +```rust +/// SQL documentation metadata +pub struct SqlDocs { + pub summary: &'static str, + pub functions: Vec, +} + +pub struct FunctionDoc { + pub name: &'static str, + pub signature: &'static str, + pub description: &'static str, + pub examples: Vec<&'static str>, +} + +trait Component { + // ... existing methods ... + + fn documentation() -> Option { + None + } +} +``` + +Generate docs with `cargo doc`: +```rust +impl Component for AddColumn { + fn documentation() -> Option { + Some(SqlDocs { + summary: "Add encrypted column configuration", + functions: vec![FunctionDoc { + name: "add_column", + signature: "add_column(table_name text, column_name text, source_type text) RETURNS jsonb", + description: "Configures an encrypted column...", + examples: vec![ + "SELECT eql_v2.add_column('users', 'email', 'text');" + ], + }], + }) + } +} +``` + +### 3. Medium-Priority Enhancements + +- **Test discovery:** Auto-discover `*_test.rs` files +- **Dependency visualization:** Add `eql-deps` binary with `--graph` flag +- **Error messages:** Better compile errors for circular dependencies + +### 4. Do NOT Pursue + +- ❌ pgrx integration (not feasible) +- ❌ Rewriting EQL in Rust (defeats purpose of SQL-first approach) +- ❌ Compiled UDFs (breaks Supabase compatibility) + +--- + +## Migration Plan + +### Phase 1: Validate Spike (Current State) + +**Goal:** Prove the approach works for a subset of EQL + +**Status:** ✅ Complete +- `eql-core` with `Component` trait +- `eql-postgres` with config components +- `eql-test` with transaction isolation +- `eql-build` generating SQL files +- Working tests demonstrating approach + +**Evidence:** `tests/config_test.rs` validates end-to-end workflow + +### Phase 2: Expand Component Coverage + +**Goal:** Map all EQL SQL files to Rust components + +**Tasks:** +- [ ] Create components for all `src/blake3/*.sql` files +- [ ] Create components for all `src/encrypted/*.sql` files +- [ ] Create components for all `src/operators/*.sql` files +- [ ] Create components for all `src/ore*/*.sql` files +- [ ] Create components for all `src/config/*.sql` files + +**Acceptance:** Can build full `cipherstash-encrypt.sql` via `eql-build` + +### Phase 3: Replace Build System + +**Goal:** Replace `mise` + `tsort` with `cargo` + +**Tasks:** +- [ ] Add multi-version test support +- [ ] Migrate all `*_test.sql` files to Rust tests +- [ ] Update CI to use `cargo test` instead of `mise run test` +- [ ] Update `mise run build` to call `cargo run --bin eql-build` + +**Acceptance:** CI passes using new build system + +### Phase 4: Enhance Developer Experience + +**Goal:** Make Rust tooling better than old system + +**Tasks:** +- [ ] Add dependency visualization +- [ ] Extract SQL documentation for rustdoc +- [ ] Add release automation +- [ ] Create developer documentation + +**Acceptance:** Team prefers new workflow over old + +--- + +## Success Metrics + +### Developer Experience +- [ ] Test execution time < 5 minutes for all PostgreSQL versions +- [ ] Compile-time dependency validation prevents build errors +- [ ] Zero manual `-- REQUIRE:` comment maintenance +- [ ] Rustdoc generates comprehensive API reference + +### Build Quality +- [ ] 100% test coverage (all SQL files have Rust tests) +- [ ] CI validates against PostgreSQL 14, 15, 16, 17 +- [ ] Generated SQL identical to current `release/cipherstash-encrypt.sql` +- [ ] No regressions in Supabase compatibility + +### Long-Term Maintainability +- [ ] New contributors can understand dependency graph via types +- [ ] Adding new SQL file requires single `Component` impl +- [ ] Breaking changes caught at compile time +- [ ] Documentation stays in sync with code + +--- + +## Conclusion + +**pgrx is not suitable for EQL** due to fundamental architectural incompatibility. pgrx assumes Rust is the implementation language with SQL as an interface layer. EQL requires SQL as the implementation with Rust providing development tooling. + +**The custom spike in `feature/rust-sql-tooling` is the correct path forward.** It already provides: +- ✅ Type-safe SQL dependency management +- ✅ Transaction-isolated testing +- ✅ SQL-only build output +- ✅ Foundation for documentation generation +- ✅ Better developer experience than current `tsort` approach + +**Next steps:** +1. Enhance multi-version PostgreSQL testing +2. Expand component coverage to all EQL SQL files +3. Migrate build system from `mise` to `cargo` +4. Add documentation extraction and dependency visualization + +This investigation successfully ruled out pgrx while validating the custom approach. The spike should be developed into EQL's production build/test infrastructure. + +--- + +## Appendix: Key Learnings + +### What pgrx Does Well +- Multi-version PostgreSQL testing ergonomics +- Automatic SQL schema generation from code +- In-process testing with transaction isolation +- Mature tooling ecosystem + +### Why pgrx Doesn't Fit EQL +- Requires compiled binary extensions +- SQL is interface, not implementation +- Cannot generate standalone SQL files +- Testing framework depends on compiled code +- Incompatible with Supabase/restricted environments + +### What Makes Custom Spike Superior +- Purpose-built for SQL-first extensions +- Type-safe dependencies at compile time +- Pure SQL output (no binaries) +- Supabase compatible +- Simpler mental model for SQL developers + +### Inspiration from pgrx +- Multi-version testing patterns +- Schema introspection concepts +- Documentation generation ideas +- Dependency resolution approaches + +--- + +**Report Author:** Claude (claude-sonnet-4-5) +**Date:** 2025-10-22 +**Branch:** feature/rust-sql-tooling diff --git a/docs/plans/2025-10-22-pgrx-investigation-summary.md b/docs/plans/2025-10-22-pgrx-investigation-summary.md new file mode 100644 index 0000000..d4680f0 --- /dev/null +++ b/docs/plans/2025-10-22-pgrx-investigation-summary.md @@ -0,0 +1,240 @@ +# pgrx Investigation - Executive Summary + +**Date:** 2025-10-22 +**Status:** Investigation Complete +**Decision:** Do not use pgrx. Continue with custom Rust tooling. + +--- + +## The Question + +Can we use pgrx (Rust framework for PostgreSQL extensions) to improve EQL's development experience, given that **EQL must remain a SQL file installer** (not a compiled extension)? + +--- + +## The Answer + +**No. pgrx is fundamentally incompatible with SQL-only deployment.** + +### Why pgrx Doesn't Work + +| What pgrx Does | What EQL Needs | +|----------------|----------------| +| Generates compiled `.so` libraries + SQL | Pure SQL files only | +| Rust is the implementation | SQL is the implementation | +| `CREATE EXTENSION` installation | `psql < install.sql` installation | +| Requires binary deployment | Must work in Supabase (no binaries allowed) | + +**Core incompatibility:** pgrx's generated SQL files call functions in compiled Rust code. They cannot run standalone. + +### Can We Use Parts of pgrx? + +We investigated extracting individual components: + +- ❌ **Testing framework** - Requires compiled extension +- ❌ **Schema generation** - Generates SQL that references `.so` files +- ❌ **Build tooling** - Designed for binary output +- ❌ **All other components** - Tightly coupled to extension model + +**Verdict:** No meaningful parts can be extracted. + +--- + +## What We Already Have (Better!) + +The `feature/rust-sql-tooling` spike provides a **custom framework purpose-built for SQL-first extensions**. + +### What It Achieves + +✅ **Type-safe dependencies** - Compile-time validation via Rust's type system +✅ **SQL-only output** - Generates pure `.sql` files +✅ **Transaction-isolated testing** - `TestDb` with automatic rollback +✅ **Automatic dependency resolution** - No more brittle `-- REQUIRE:` comments +✅ **Rustdoc-compatible** - Can document EQL API in standard Rust docs +✅ **Supabase compatible** - No compiled binaries required + +### Architecture Comparison + +``` +┌──────────────────────────────────────────────────┐ +│ pgrx: Rust → .so + SQL → CREATE EXTENSION │ +│ │ +│ EQL: SQL → Rust tooling → .sql → psql install │ +└──────────────────────────────────────────────────┘ +``` + +**Our spike is architecturally aligned. pgrx solves a different problem.** + +--- + +## What Needs Work + +The spike is solid but has gaps compared to a mature framework: + +| Gap | Impact | Priority | +|-----|--------|----------| +| **Multi-version PostgreSQL testing** | High - testing pg14-17 is manual | 🔴 High | +| **Documentation extraction** | Medium - SQL functions not in rustdoc | 🟡 Medium | +| **Test discovery** | Medium - manual test registration | 🟡 Medium | +| **Dependency visualization** | Low - hard to see dep graph | 🟢 Low | +| **Release automation** | Low - versioning is manual | 🟢 Low | + +--- + +## Recommendations + +### 1. ✅ Commit to Custom Rust Tooling + +Continue developing `feature/rust-sql-tooling` as EQL's build/test infrastructure. + +**Why:** +- Already solves core problems +- pgrx offers no viable alternative +- Investment validates the approach + +### 2. 🔴 Priority: Multi-Version PostgreSQL Testing + +Add ergonomic testing across PostgreSQL 14-17, inspired by pgrx's approach: + +```rust +// From: test_all_versions!(test_name, |db| async move { ... }) +// To: Automatically runs against all PostgreSQL versions + +#[eql_test(all_versions)] +async fn test_add_column_works() { + // Runs against pg14, pg15, pg16, pg17 +} +``` + +**Benefit:** Match pgrx's developer experience without the architectural mismatch. + +### 3. 🟡 Next: Expand Component Coverage + +Map all EQL SQL files to Rust `Component` implementations: +- [ ] `src/blake3/*.sql` +- [ ] `src/encrypted/*.sql` +- [ ] `src/operators/*.sql` +- [ ] `src/ore*/*.sql` +- [ ] `src/config/*.sql` (partially done) + +**Goal:** Replace current `mise` + `tsort` build system entirely. + +### 4. 🟡 Then: Documentation Extraction + +Extract SQL function signatures/docs for rustdoc generation: + +```rust +impl Component for AddColumn { + fn documentation() -> Option { + Some(SqlDocs { + summary: "Add encrypted column configuration", + functions: vec![/* ... */], + }) + } +} +``` + +**Benefit:** Auto-generated reference docs that stay in sync with code. + +--- + +## Migration Path + +### Phase 1: Validate (✅ Complete) +- Spike proves approach works +- Config subsystem as proof-of-concept +- Tests demonstrate transaction isolation + +### Phase 2: Expand (Next) +- Map all SQL files to Components +- Add multi-version test support +- Build full `cipherstash-encrypt.sql` + +### Phase 3: Replace (Future) +- Migrate CI to `cargo test` +- Update `mise` tasks to call Rust tooling +- Deprecate `tsort` dependency system + +### Phase 4: Enhance (Future) +- Documentation generation +- Dependency visualization +- Release automation + +--- + +## What We Learned from pgrx + +While we can't use pgrx's code, we stole good ideas: + +### Multi-Version Testing Patterns +```rust +// pgrx: cargo pgrx test pg14 pg15 pg16 pg17 +// EQL: Rust macro that runs test across Docker containers +``` + +### Schema Introspection +```rust +// Extract SQL function metadata for documentation +trait Component { + fn public_functions() -> Vec; +} +``` + +### Dependency Automation +```rust +// Automatic topological sorting via type system +AddColumn::collect_dependencies() // Returns ordered list +``` + +--- + +## Key Metrics for Success + +**Developer Experience:** +- [ ] Test execution < 5 min for all PostgreSQL versions +- [ ] Compile-time dependency validation +- [ ] Zero manual `-- REQUIRE:` maintenance + +**Build Quality:** +- [ ] 100% test coverage +- [ ] CI validates pg14-17 +- [ ] No Supabase regressions + +**Maintainability:** +- [ ] Dependency graph visible via types +- [ ] Documentation stays in sync +- [ ] Breaking changes caught at compile time + +--- + +## Conclusion + +### The Decision + +**Do not use pgrx.** It's a excellent framework for Rust-based PostgreSQL extensions, but EQL is a SQL-based extension with Rust tooling. + +**Continue with `feature/rust-sql-tooling`.** It already provides superior solutions for EQL's specific constraints. + +### Why This Is The Right Call + +1. **pgrx fundamentally requires compiled extensions** - EQL cannot ship binaries +2. **Custom spike is purpose-built for SQL-first** - Architecturally aligned +3. **Already provides core value** - Type-safe deps, testing, build automation +4. **Clear path forward** - Enhance multi-version testing, expand coverage + +### Next Steps + +1. 🔴 **Implement multi-version PostgreSQL testing** (high priority) +2. 🟡 **Map remaining SQL files to Components** (expand coverage) +3. 🟡 **Add documentation extraction** (improve DX) +4. 🟢 **Enhance with viz tools** (nice to have) + +### Questions? + +See full investigation report: `docs/plans/2025-10-22-pgrx-investigation-report.md` + +--- + +**Report Author:** Claude (claude-sonnet-4-5) +**Full Report:** `2025-10-22-pgrx-investigation-report.md` +**Branch:** feature/rust-sql-tooling diff --git a/docs/reference/database-indexes.md b/docs/reference/database-indexes.md new file mode 100644 index 0000000..d438603 --- /dev/null +++ b/docs/reference/database-indexes.md @@ -0,0 +1,390 @@ +# Database Indexes for Encrypted Columns + +EQL supports PostgreSQL B-tree indexes on `eql_v2_encrypted` columns to improve query performance. This guide explains how to create and use indexes effectively. + +## Table of Contents + +- [Creating Indexes](#creating-indexes) +- [Index Usage Requirements](#index-usage-requirements) +- [Query Patterns That Use Indexes](#query-patterns-that-use-indexes) +- [Query Patterns That Don't Use Indexes](#query-patterns-that-dont-use-indexes) +- [Index Limitations](#index-limitations) +- [Best Practices](#best-practices) + +--- + +## Creating Indexes + +### Basic Index Creation + +Create a B-tree index on an encrypted column using the `eql_v2.encrypted_operator_class`: + +```sql +CREATE INDEX ON table_name (encrypted_column eql_v2.encrypted_operator_class); +``` + +**Named index:** + +```sql +CREATE INDEX idx_users_email ON users (encrypted_email eql_v2.encrypted_operator_class); +``` + +### When to Create Indexes + +Create indexes on encrypted columns when: +- The table has a significant number of rows (typically > 1000) +- You frequently query by equality on that column +- Query performance is important +- The column contains searchable index terms (hmac_256, blake3, or ore) + +--- + +## Index Usage Requirements + +For PostgreSQL to use an index on encrypted columns, **all** of these conditions must be met: + +### 1. Column Must Have Appropriate Search Terms + +The encrypted data must contain the index term types that support the operation: + +- **Equality queries** - Require `unique` index config (adds `hm` hmac_256 or `b3` blake3 terms) +- **Range queries** - Require `ore` index config (adds `ob` ore_block_u64_8_256 terms) +- **Pattern matching** - Typically scans (bloom filters don't use B-tree indexes) + +**Example:** +```sql +-- This data HAS hmac_256 term - index will be used +'{"i":{"t":"users","c":"email"},"v":2,"hm":"abc123..."}' + +-- This data has ONLY bloom filter - index WON'T be used for equality +'{"i":{"t":"users","c":"email"},"v":2,"bf":[1,2,3]}' +``` + +### 2. Index Must Be Created AFTER Data Contains Required Terms + +If you: +1. Insert data without a search term (e.g., only `bf`) +2. Add the search term later (e.g., add `hm`) +3. Create an index + +**The index will NOT work** until you: +- Recreate the index, OR +- Truncate and repopulate the table + +**Correct order:** +```sql +-- 1. Configure the index type FIRST +SELECT eql_v2.add_search_config('users', 'encrypted_email', 'unique', 'text'); + +-- 2. Insert/update data through CipherStash Proxy (adds index terms) +INSERT INTO users (encrypted_email) VALUES (...); + +-- 3. Create the PostgreSQL index +CREATE INDEX ON users (encrypted_email eql_v2.encrypted_operator_class); +ANALYZE users; +``` + +### 3. Query Must Use Correct Type Casting + +The query value must be cast to `eql_v2_encrypted`: + +**✓ Index will be used:** +```sql +-- Literal row type +WHERE e = '("{\"hm\": \"abc\"}")'; + +-- Cast to eql_v2_encrypted +WHERE e = '{"hm": "abc"}'::eql_v2_encrypted; +WHERE e = '{"hm": "abc"}'::text::eql_v2_encrypted; +WHERE e = '{"hm": "abc"}'::jsonb::eql_v2_encrypted; + +-- Using helper function +WHERE e = eql_v2.to_encrypted('{"hm": "abc"}'::jsonb); +WHERE e = eql_v2.to_encrypted('{"hm": "abc"}'); + +-- Using parameterized query with encrypted value +WHERE e = $1::eql_v2_encrypted; +``` + +**✗ Index will NOT be used:** +```sql +-- Missing type cast +WHERE e = '{"hm": "abc"}'::jsonb; +``` + +--- + +## Query Patterns That Use Indexes + +### Equality Queries + +When encrypted column has `hm` (hmac_256) or `b3` (blake3) index terms: + +```sql +-- These will use the index +SELECT * FROM users +WHERE encrypted_email = $1::eql_v2_encrypted; + +SELECT * FROM users +WHERE encrypted_email = '{"hm": "abc123..."}'::eql_v2_encrypted; + +SELECT * FROM users +WHERE encrypted_email = eql_v2.to_encrypted('{"hm": "abc123..."}'::jsonb); +``` + +**Expected EXPLAIN output:** +``` +Index Only Scan using idx_users_email on users + Index Cond: (encrypted_email = '...'::eql_v2_encrypted) +``` + +Or: +``` +Bitmap Heap Scan on users + Recheck Cond: (encrypted_email = '...'::eql_v2_encrypted) + -> Bitmap Index Scan on idx_users_email + Index Cond: (encrypted_email = '...'::eql_v2_encrypted) +``` + +### Range Queries + +When encrypted column has `ob` (ore_block_u64_8_256) index terms: + +```sql +SELECT * FROM events +WHERE encrypted_date < $1::eql_v2_encrypted +ORDER BY encrypted_date DESC; +``` + +### GROUP BY + +Encrypted columns can be used in GROUP BY with indexes: + +```sql +SELECT encrypted_status, COUNT(*) +FROM orders +GROUP BY encrypted_status; +``` + +--- + +## Query Patterns That Don't Use Indexes + +### 1. Missing Type Cast + +```sql +-- ✗ No index usage - missing ::eql_v2_encrypted cast +SELECT * FROM users WHERE encrypted_email = '{"hm": "abc"}'::jsonb; +``` + +### 2. Data Without Required Index Terms + +```sql +-- ✗ Data only has bloom filter, not hmac_256 +-- Index won't be used even if query is correct +SELECT * FROM users +WHERE encrypted_email = $1::eql_v2_encrypted; +-- If column only has: '{"bf":[1,2,3]}' +``` + +### 3. Pattern Matching (LIKE) + +```sql +-- ✗ Bloom filter queries typically don't use B-tree indexes +SELECT * FROM users +WHERE encrypted_name ~~ $1::eql_v2_encrypted; +``` + +### 4. Index Created Before Data Population + +```sql +-- ✗ Wrong order +CREATE INDEX ON users (encrypted_email eql_v2.encrypted_operator_class); +-- Then add data with hm terms +-- Index won't work until recreated +``` + +--- + +## Index Limitations + +### 1. Index Term Requirement + +B-tree indexes **only work** with: +- `hm` (hmac_256) - for equality +- `b3` (blake3) - for equality +- `ob` (ore_block_u64_8_256) - for range queries + +They **do not work** with: +- `bf` (bloom_filter) - pattern matching +- `sv` (ste_vec) - JSONB containment +- Data without any index terms + +### 2. Index Creation Timing + +The index must be created **after** the data contains the required index terms. If you: + +1. Add `unique` config to existing column +2. Re-encrypt data to add `hm` terms +3. Create index + +You must create the index **after step 2**, not before. + +### 3. Index Doesn't Auto-Update + +If you modify the search configuration (e.g., change from `unique` to different config), you should: + +```sql +-- Drop and recreate the index +DROP INDEX idx_users_email; +CREATE INDEX idx_users_email ON users (encrypted_email eql_v2.encrypted_operator_class); +ANALYZE users; +``` + +--- + +## Best Practices + +### 1. Configure Search Indexes First + +Always configure EQL search indexes before creating PostgreSQL indexes: + +```sql +-- Step 1: Configure searchable encryption +SELECT eql_v2.add_column('users', 'encrypted_email', 'text'); +SELECT eql_v2.add_search_config('users', 'encrypted_email', 'unique', 'text'); + +-- Step 2: Populate data (through CipherStash Proxy) +INSERT INTO users (encrypted_email) VALUES (...); + +-- Step 3: Create PostgreSQL index +CREATE INDEX ON users (encrypted_email eql_v2.encrypted_operator_class); +ANALYZE users; +``` + +### 2. Run ANALYZE After Index Creation + +Always run `ANALYZE` after creating an index to update query planner statistics: + +```sql +CREATE INDEX idx_users_email ON users (encrypted_email eql_v2.encrypted_operator_class); +ANALYZE users; +``` + +### 3. Verify Index Usage + +Use `EXPLAIN ANALYZE` to verify the index is being used: + +```sql +EXPLAIN ANALYZE +SELECT * FROM users +WHERE encrypted_email = $1::eql_v2_encrypted; +``` + +Look for: +- `Index Only Scan using idx_name` +- `Bitmap Index Scan on idx_name` +- `Bitmap Heap Scan` with `Bitmap Index Scan` + +If you see `Seq Scan`, the index is not being used. + +### 4. Name Your Indexes + +Use descriptive names for easier management: + +```sql +CREATE INDEX idx_users_encrypted_email +ON users (encrypted_email eql_v2.encrypted_operator_class); + +CREATE INDEX idx_events_encrypted_date +ON events (encrypted_date eql_v2.encrypted_operator_class); +``` + +### 5. Consider Index Size + +Indexes on encrypted columns can be large. Monitor index size: + +```sql +SELECT + indexname, + pg_size_pretty(pg_relation_size(schemaname||'.'||indexname)) AS index_size +FROM pg_indexes +WHERE tablename = 'users'; +``` + +### 6. Drop Unused Indexes + +If you remove a search configuration, drop the corresponding PostgreSQL index: + +```sql +-- After removing search config +SELECT eql_v2.remove_search_config('users', 'encrypted_email', 'unique'); + +-- Drop the PostgreSQL index +DROP INDEX IF EXISTS idx_users_encrypted_email; +``` + +--- + +## Troubleshooting + +### Index Not Being Used + +**Check 1: Verify data has index terms** + +```sql +-- Check if data contains hm (hmac_256) or b3 (blake3) for equality +SELECT encrypted_email::jsonb ? 'hm' AS has_hmac, + encrypted_email::jsonb ? 'b3' AS has_blake3, + encrypted_email::jsonb ? 'ob' AS has_ore +FROM users LIMIT 1; +``` + +**Check 2: Verify query uses correct cast** + +```sql +-- ✓ Correct - will use index +WHERE encrypted_email = $1::eql_v2_encrypted + +-- ✗ Wrong - won't use index +WHERE encrypted_email = $1::jsonb +``` + +**Check 3: Recreate index if needed** + +```sql +DROP INDEX IF EXISTS idx_users_encrypted_email; +CREATE INDEX idx_users_encrypted_email +ON users (encrypted_email eql_v2.encrypted_operator_class); +ANALYZE users; +``` + +**Check 4: Verify index exists** + +```sql +SELECT indexname, indexdef +FROM pg_indexes +WHERE tablename = 'users' + AND indexname LIKE '%encrypted%'; +``` + +### Poor Query Performance + +1. **Ensure index exists and is being used** - Use `EXPLAIN ANALYZE` +2. **Check table has been ANALYZEd** - Run `ANALYZE table_name` +3. **Consider index selectivity** - Very small tables might not use indexes +4. **Check for appropriate search config** - Equality needs `unique`, ranges need `ore` + +--- + +## See Also + +- [EQL Functions Reference](./eql-functions.md) - Complete function API +- [Index Configuration](./index-config.md) - Searchable encryption index types +- [Configuration Tutorial](../tutorials/proxy-configuration.md) - Setting up encrypted columns + +--- + +### Didn't find what you wanted? + +[Click here to let us know what was missing from our docs.](https://github.com/cipherstash/encrypt-query-language/issues/new?template=docs-feedback.yml&title=[Docs:]%20Feedback%20on%20database-indexes.md) diff --git a/docs/reference/eql-functions.md b/docs/reference/eql-functions.md new file mode 100644 index 0000000..0547833 --- /dev/null +++ b/docs/reference/eql-functions.md @@ -0,0 +1,759 @@ +# EQL Functions Reference + +This document provides a comprehensive reference for all EQL (Encrypt Query Language) functions available for querying encrypted data in PostgreSQL. + +## Table of Contents + +- [Configuration Functions](#configuration-functions) +- [Query Functions](#query-functions) + - [Operators (Recommended)](#operators-recommended) + - [Function Equivalents](#function-equivalents) +- [Index Term Extraction Functions](#index-term-extraction-functions) +- [JSONB Path Functions](#jsonb-path-functions) +- [Array Functions](#array-functions) +- [Helper Functions](#helper-functions) +- [Aggregate Functions](#aggregate-functions) +- [Utility Functions](#utility-functions) + +--- + +## Configuration Functions + +These functions manage encrypted column configurations. See [Configuration Tutorial](../tutorials/proxy-configuration.md) for detailed usage. + +### `eql_v2.add_column()` + +Initialize a column for encryption/decryption. + +```sql +eql_v2.add_column( + table_name text, + column_name text, + cast_as text DEFAULT 'text', + migrating boolean DEFAULT false +) RETURNS jsonb +``` + +**Example:** +```sql +SELECT eql_v2.add_column('users', 'encrypted_email', 'text'); +``` + +### `eql_v2.add_search_config()` + +Add a searchable index to an encrypted column. + +```sql +eql_v2.add_search_config( + table_name text, + column_name text, + index_name text, -- 'unique', 'match', 'ore', 'ste_vec' + cast_as text DEFAULT 'text', + opts jsonb DEFAULT '{}', + migrating boolean DEFAULT false +) RETURNS jsonb +``` + +**Supported index types:** +- `unique` - Exact equality (uses hmac_256 or blake3) +- `match` - Full-text search (uses bloom_filter) +- `ore` - Range queries and ordering (uses ore_block_u64_8_256) +- `ste_vec` - JSONB containment queries (uses structured encryption) + +**Example:** +```sql +SELECT eql_v2.add_search_config('users', 'encrypted_email', 'unique', 'text'); +SELECT eql_v2.add_search_config('docs', 'encrypted_content', 'match', 'text'); +SELECT eql_v2.add_search_config('events', 'encrypted_data', 'ste_vec', 'jsonb', '{"prefix": "events/encrypted_data"}'); +``` + +### `eql_v2.remove_column()` + +Remove column configuration completely. + +```sql +eql_v2.remove_column( + table_name text, + column_name text, + migrating boolean DEFAULT false +) RETURNS jsonb +``` + +### `eql_v2.remove_search_config()` + +Remove a specific search index (preserves column configuration). + +```sql +eql_v2.remove_search_config( + table_name text, + column_name text, + index_name text, + migrating boolean DEFAULT false +) RETURNS jsonb +``` + +### `eql_v2.modify_search_config()` + +Modify an existing search index configuration. + +```sql +eql_v2.modify_search_config( + table_name text, + column_name text, + index_name text, + cast_as text DEFAULT 'text', + opts jsonb DEFAULT '{}', + migrating boolean DEFAULT false +) RETURNS jsonb +``` + +### `eql_v2.config()` + +View current configuration in tabular format. + +```sql +eql_v2.config() RETURNS TABLE ( + state eql_v2_configuration_state, + relation text, + col_name text, + decrypts_as text, + indexes jsonb +) +``` + +**Example:** +```sql +SELECT * FROM eql_v2.config(); +``` + +### `eql_v2.migrate_config()` + +Transition pending configuration to encrypting state. + +```sql +eql_v2.migrate_config() RETURNS boolean +``` + +**Description:** +- Validates that all configured columns exist with `eql_v2_encrypted` type +- Marks the pending configuration as 'encrypting' +- Required before activating a new configuration + +**Raises exception if:** +- An encryption is already in progress +- No pending configuration exists +- Some pending columns don't have encrypted targets + +**Example:** +```sql +-- Add configuration changes +SELECT eql_v2.add_search_config('users', 'email', 'unique', 'text', migrating => true); + +-- Validate and migrate +SELECT eql_v2.migrate_config(); + +-- After re-encrypting data, activate +SELECT eql_v2.activate_config(); +``` + +### `eql_v2.activate_config()` + +Activate an encrypting configuration. + +```sql +eql_v2.activate_config() RETURNS boolean +``` + +**Description:** +- Moves 'encrypting' configuration to 'active' state +- Marks previous 'active' configuration as 'inactive' +- Should be called after data has been re-encrypted with new index terms + +**Raises exception if:** +- No encrypting configuration exists + +**Example:** +```sql +SELECT eql_v2.activate_config(); +``` + +### `eql_v2.discard()` + +Discard pending configuration without activating. + +```sql +eql_v2.discard() RETURNS boolean +``` + +**Description:** +- Deletes the pending configuration +- Use when you want to abandon configuration changes + +**Raises exception if:** +- No pending configuration exists + +**Example:** +```sql +SELECT eql_v2.discard(); +``` + +### `eql_v2.reload_config()` + +Reload active configuration (no-op for compatibility). + +```sql +eql_v2.reload_config() RETURNS void +``` + +**Description:** +- Placeholder function for configuration reload +- Currently has no effect (configuration is loaded automatically) + +--- + +## Query Functions + +### Operators (Recommended) + +EQL overloads standard PostgreSQL operators to work directly on `eql_v2_encrypted` columns. **Use these whenever possible.** + +#### Equality + +```sql +-- Exact match (uses 'unique' index: hmac_256 or blake3) +SELECT * FROM users WHERE encrypted_email = $1::eql_v2_encrypted; +SELECT * FROM users WHERE encrypted_email = $1::jsonb; + +-- Not equal +SELECT * FROM users WHERE encrypted_email <> $1::eql_v2_encrypted; +``` + +#### Full-Text Match + +```sql +-- Case-sensitive LIKE (uses 'match' index: bloom_filter) +SELECT * FROM docs WHERE encrypted_content ~~ $1::eql_v2_encrypted; +SELECT * FROM docs WHERE encrypted_content LIKE $1::eql_v2_encrypted; + +-- Case-insensitive ILIKE +SELECT * FROM docs WHERE encrypted_content ~~* $1::eql_v2_encrypted; +SELECT * FROM docs WHERE encrypted_content ILIKE $1::eql_v2_encrypted; +``` + +#### Range Comparisons + +```sql +-- Uses 'ore' index: ore_block_u64_8_256 +SELECT * FROM events WHERE encrypted_date < $1::eql_v2_encrypted; +SELECT * FROM events WHERE encrypted_date <= $1::eql_v2_encrypted; +SELECT * FROM events WHERE encrypted_date > $1::eql_v2_encrypted; +SELECT * FROM events WHERE encrypted_date >= $1::eql_v2_encrypted; + +-- Ordering +SELECT * FROM events ORDER BY encrypted_date DESC; +SELECT * FROM events ORDER BY encrypted_date ASC; +``` + +#### JSONB Containment + +```sql +-- Uses 'ste_vec' index +SELECT * FROM users WHERE encrypted_data @> $1::eql_v2_encrypted; +SELECT * FROM users WHERE encrypted_data <@ $1::eql_v2_encrypted; +``` + +#### JSON Path Access + +```sql +-- Extract field by selector hash (returns eql_v2_encrypted) +SELECT encrypted_json->'abc123...' FROM users; +SELECT encrypted_json->encrypted_selector FROM users; + +-- Extract field by array index (returns eql_v2_encrypted) +SELECT encrypted_json->0 FROM users; + +-- Extract field as ciphertext (returns text) +SELECT encrypted_json->>'abc123...' FROM users; +SELECT encrypted_json->>encrypted_selector FROM users; +``` + +### Function Equivalents + +For environments that don't support custom operators (like Supabase), use these function versions: + +#### `eql_v2.eq()` + +Equality comparison. + +```sql +eql_v2.eq(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean +``` + +**Example:** +```sql +SELECT * FROM users WHERE eql_v2.eq(encrypted_email, $1::eql_v2_encrypted); +``` + +#### `eql_v2.neq()` + +Not-equal comparison. + +```sql +eql_v2.neq(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean +``` + +#### `eql_v2.like()` + +Pattern matching (case-sensitive). + +```sql +eql_v2.like(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean +``` + +**Example:** +```sql +SELECT * FROM docs WHERE eql_v2.like(encrypted_content, $1::eql_v2_encrypted); +``` + +#### `eql_v2.ilike()` + +Pattern matching (case-insensitive). + +```sql +eql_v2.ilike(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean +``` + +**Example:** +```sql +SELECT * FROM docs WHERE eql_v2.ilike(encrypted_content, $1::eql_v2_encrypted); +``` + +#### `eql_v2.lt()` + +Less than comparison. + +```sql +eql_v2.lt(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean +``` + +**Example:** +```sql +SELECT * FROM events WHERE eql_v2.lt(encrypted_date, $1::eql_v2_encrypted); +``` + +#### `eql_v2.lte()` + +Less than or equal comparison. + +```sql +eql_v2.lte(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean +``` + +**Example:** +```sql +SELECT * FROM events WHERE eql_v2.lte(encrypted_date, $1::eql_v2_encrypted); +``` + +#### `eql_v2.gt()` + +Greater than comparison. + +```sql +eql_v2.gt(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean +``` + +**Example:** +```sql +SELECT * FROM events WHERE eql_v2.gt(encrypted_date, $1::eql_v2_encrypted); +``` + +#### `eql_v2.gte()` + +Greater than or equal comparison. + +```sql +eql_v2.gte(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean +``` + +**Example:** +```sql +SELECT * FROM events WHERE eql_v2.gte(encrypted_date, $1::eql_v2_encrypted); +``` + +--- + +## Index Term Extraction Functions + +These functions extract specific index terms from encrypted values. Typically used internally by operators, but available for advanced use cases. + +### `eql_v2.hmac_256()` + +Extract HMAC-256 unique index term. + +```sql +eql_v2.hmac_256(val eql_v2_encrypted) RETURNS eql_v2.hmac_256 +eql_v2.hmac_256(val jsonb) RETURNS eql_v2.hmac_256 +``` + +### `eql_v2.blake3()` + +Extract Blake3 unique index term. + +```sql +eql_v2.blake3(val eql_v2_encrypted) RETURNS eql_v2.blake3 +eql_v2.blake3(val jsonb) RETURNS eql_v2.blake3 +``` + +### `eql_v2.bloom_filter()` + +Extract bloom filter match index term. + +```sql +eql_v2.bloom_filter(val eql_v2_encrypted) RETURNS eql_v2.bloom_filter +eql_v2.bloom_filter(val jsonb) RETURNS eql_v2.bloom_filter +``` + +### `eql_v2.ore_block_u64_8_256()` + +Extract ORE (Order-Revealing Encryption) index term. + +```sql +eql_v2.ore_block_u64_8_256(val eql_v2_encrypted) RETURNS eql_v2.ore_block_u64_8_256 +eql_v2.ore_block_u64_8_256(val jsonb) RETURNS eql_v2.ore_block_u64_8_256 +``` + +### `eql_v2.ste_vec()` + +Extract structured encryption vector array. + +```sql +eql_v2.ste_vec(val eql_v2_encrypted) RETURNS eql_v2_encrypted[] +eql_v2.ste_vec(val jsonb) RETURNS eql_v2_encrypted[] +``` + +--- + +## JSONB Path Functions + +Functions for querying encrypted JSONB data using selector hashes. + +### `eql_v2.jsonb_path_query()` + +Returns all encrypted elements matching a selector. + +```sql +eql_v2.jsonb_path_query(val eql_v2_encrypted, selector text) RETURNS SETOF eql_v2_encrypted +eql_v2.jsonb_path_query(val eql_v2_encrypted, selector eql_v2_encrypted) RETURNS SETOF eql_v2_encrypted +eql_v2.jsonb_path_query(val jsonb, selector text) RETURNS SETOF eql_v2_encrypted +``` + +**Example:** +```sql +SELECT eql_v2.jsonb_path_query(encrypted_json, 'abc123...') FROM users; +``` + +### `eql_v2.jsonb_path_query_first()` + +Returns the first encrypted element matching a selector. + +```sql +eql_v2.jsonb_path_query_first(val eql_v2_encrypted, selector text) RETURNS eql_v2_encrypted +eql_v2.jsonb_path_query_first(val eql_v2_encrypted, selector eql_v2_encrypted) RETURNS eql_v2_encrypted +eql_v2.jsonb_path_query_first(val jsonb, selector text) RETURNS eql_v2_encrypted +``` + +### `eql_v2.jsonb_path_exists()` + +Checks if any element matches a selector. + +```sql +eql_v2.jsonb_path_exists(val eql_v2_encrypted, selector text) RETURNS boolean +eql_v2.jsonb_path_exists(val eql_v2_encrypted, selector eql_v2_encrypted) RETURNS boolean +eql_v2.jsonb_path_exists(val jsonb, selector text) RETURNS boolean +``` + +**Example:** +```sql +SELECT * FROM users +WHERE eql_v2.jsonb_path_exists(encrypted_json, 'email_selector'); +``` + +--- + +## Array Functions + +Functions for working with encrypted arrays. + +### `eql_v2.jsonb_array_length()` + +Returns the length of an encrypted array. + +```sql +eql_v2.jsonb_array_length(val eql_v2_encrypted) RETURNS integer +eql_v2.jsonb_array_length(val jsonb) RETURNS integer +``` + +**Example:** +```sql +SELECT eql_v2.jsonb_array_length(encrypted_array) FROM users; +``` + +### `eql_v2.jsonb_array_elements()` + +Returns each array element as an encrypted value. + +```sql +eql_v2.jsonb_array_elements(val eql_v2_encrypted) RETURNS SETOF eql_v2_encrypted +eql_v2.jsonb_array_elements(val jsonb) RETURNS SETOF eql_v2_encrypted +``` + +**Example:** +```sql +SELECT eql_v2.jsonb_array_elements( + eql_v2.jsonb_path_query(encrypted_json, 'array_selector') +) FROM users; +``` + +### `eql_v2.jsonb_array_elements_text()` + +Returns each array element's ciphertext as text. + +```sql +eql_v2.jsonb_array_elements_text(val eql_v2_encrypted) RETURNS SETOF text +eql_v2.jsonb_array_elements_text(val jsonb) RETURNS SETOF text +``` + +--- + +## Helper Functions + +Utility functions for working with encrypted data. + +### `eql_v2.ciphertext()` + +Extract ciphertext from encrypted value. + +```sql +eql_v2.ciphertext(val eql_v2_encrypted) RETURNS text +eql_v2.ciphertext(val jsonb) RETURNS text +``` + +### `eql_v2.meta_data()` + +Extract metadata (table/column identifiers and version). + +```sql +eql_v2.meta_data(val eql_v2_encrypted) RETURNS jsonb +eql_v2.meta_data(val jsonb) RETURNS jsonb +``` + +### `eql_v2.selector()` + +Extract selector hash from encrypted value. + +```sql +eql_v2.selector(val eql_v2_encrypted) RETURNS text +``` + +### `eql_v2.is_ste_vec_array()` + +Check if value represents an encrypted array. + +```sql +eql_v2.is_ste_vec_array(val eql_v2_encrypted) RETURNS boolean +``` + +### `eql_v2.is_ste_vec_value()` + +Check if value is a single ste_vec element. + +```sql +eql_v2.is_ste_vec_value(val eql_v2_encrypted) RETURNS boolean +``` + +### `eql_v2.to_ste_vec_value()` + +Convert ste_vec array with single element to regular encrypted value. + +```sql +eql_v2.to_ste_vec_value(val eql_v2_encrypted) RETURNS eql_v2_encrypted +``` + +### `eql_v2.ste_vec_contains()` + +Check if all ste_vec terms in b exist in a (backs the `@>` operator). + +```sql +eql_v2.ste_vec_contains(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean +``` + +### `eql_v2.has_hmac_256()` + +Check if value contains hmac_256 index term. + +```sql +eql_v2.has_hmac_256(val eql_v2_encrypted) RETURNS boolean +``` + +### `eql_v2.has_blake3()` + +Check if value contains blake3 index term. + +```sql +eql_v2.has_blake3(val eql_v2_encrypted) RETURNS boolean +``` + +### `eql_v2.has_bloom_filter()` + +Check if value contains bloom_filter index term. + +```sql +eql_v2.has_bloom_filter(val eql_v2_encrypted) RETURNS boolean +``` + +### `eql_v2.has_ore_block_u64_8_256()` + +Check if value contains ore index term. + +```sql +eql_v2.has_ore_block_u64_8_256(val eql_v2_encrypted) RETURNS boolean +``` + +--- + +## Aggregate Functions + +### `eql_v2.grouped_value()` + +Aggregate function for grouping encrypted values (returns first non-null value in group). + +```sql +eql_v2.grouped_value(jsonb) RETURNS jsonb +``` + +**Example:** +```sql +SELECT eql_v2.grouped_value( + eql_v2.jsonb_path_query_first(encrypted_json, 'color_selector')::jsonb +) AS color, +COUNT(*) +FROM products +GROUP BY eql_v2.jsonb_path_query_first(encrypted_json, 'color_selector'); +``` + +### `eql_v2.min()` + +Returns the minimum encrypted value in a set (requires `ore` index for ordering). + +```sql +eql_v2.min(eql_v2_encrypted) RETURNS eql_v2_encrypted +``` + +**Example:** +```sql +SELECT eql_v2.min(encrypted_date) FROM events; +SELECT eql_v2.min(encrypted_price) FROM products WHERE category = 'electronics'; +``` + +### `eql_v2.max()` + +Returns the maximum encrypted value in a set (requires `ore` index for ordering). + +```sql +eql_v2.max(eql_v2_encrypted) RETURNS eql_v2_encrypted +``` + +**Example:** +```sql +SELECT eql_v2.max(encrypted_date) FROM events; +SELECT eql_v2.max(encrypted_price) FROM products WHERE category = 'electronics'; +``` + +--- + +## Utility Functions + +### `eql_v2.version()` + +Get the installed EQL version. + +```sql +eql_v2.version() RETURNS text +``` + +**Example:** +```sql +SELECT eql_v2.version(); +-- Returns version string (e.g., '2.1.8') +``` + +### `eql_v2.to_encrypted()` + +Convert jsonb or text to eql_v2_encrypted type. + +```sql +eql_v2.to_encrypted(data jsonb) RETURNS eql_v2_encrypted +eql_v2.to_encrypted(data text) RETURNS eql_v2_encrypted +``` + +**Example:** +```sql +-- Convert jsonb payload to encrypted type +SELECT eql_v2.to_encrypted('{"v":2,"k":"pt","p":"plaintext"}'::jsonb); + +-- Convert text payload to encrypted type +SELECT eql_v2.to_encrypted('{"v":2,"k":"pt","p":"plaintext"}'); +``` + +### `eql_v2.to_jsonb()` + +Convert eql_v2_encrypted to jsonb. + +```sql +eql_v2.to_jsonb(e eql_v2_encrypted) RETURNS jsonb +``` + +**Example:** +```sql +SELECT eql_v2.to_jsonb(encrypted_column) FROM users; +``` + +### `eql_v2.check_encrypted()` + +Validate encrypted payload structure (used in constraints). + +```sql +eql_v2.check_encrypted(val jsonb) RETURNS boolean +eql_v2.check_encrypted(val eql_v2_encrypted) RETURNS boolean +``` + +**Description:** +- Validates that encrypted value has required fields (`v`, `c`, `i`) +- Checks that version is `2` and identifier contains table (`t`) and column (`c`) fields +- Returns true if valid, raises exception if invalid +- Automatically added as constraint when using `eql_v2.add_column()` + +**Example:** +```sql +SELECT eql_v2.check_encrypted('{"v":2,"c":"ciphertext","i":{"t":"users","c":"email"}}'::jsonb); +-- Returns: true + +SELECT eql_v2.check_encrypted('{"invalid":"structure"}'::jsonb); +-- Raises exception: 'Encrypted column missing version (v) field' +``` + +--- + +## See Also + +- [EQL Configuration Guide](../tutorials/proxy-configuration.md) - How to set up encrypted columns +- [Database Indexes](./database-indexes.md) - PostgreSQL B-tree index creation and usage +- [JSON/JSONB Support](./json-support.md) - Working with encrypted JSON data +- [Index Configuration](./index-config.md) - Index types and configuration options +- [Payload Format](./PAYLOAD.md) - EQL data format specification + +--- + +### Didn't find what you wanted? + +[Click here to let us know what was missing from our docs.](https://github.com/cipherstash/encrypt-query-language/issues/new?template=docs-feedback.yml&title=[Docs:]%20Feedback%20on%20eql-functions.md) diff --git a/docs/reference/index-config.md b/docs/reference/index-config.md index b4cf25d..183726e 100644 --- a/docs/reference/index-config.md +++ b/docs/reference/index-config.md @@ -40,6 +40,8 @@ Supported types: - `int` - `small_int` - `big_int` +- `real` +- `double` - `boolean` - `date` - `jsonb` @@ -103,11 +105,16 @@ Try to ensure that the string you search for is at least as long as the `tokenLe An ste_vec index on a encrypted JSONB column enables the use of PostgreSQL's `@>` and `<@` [containment operators](https://www.postgresql.org/docs/16/functions-json.html#FUNCTIONS-JSONB-OP-TABLE). -An ste_vec index requires one piece of configuration: the `context` (a string) which is passed as an info string to a MAC (Message Authenticated Code). -This ensures that all of the encrypted values are unique to that context. -We recommend that you use the table and column name as a the context (e.g. `users/name`). +An ste_vec index requires one piece of configuration: the `prefix` (a string) which is passed as an info string to a MAC (Message Authenticated Code). +This ensures that all of the encrypted values are unique to that prefix. +We recommend that you use the table and column name as the prefix (e.g. `users/name`). -Within a dataset, encrypted columns indexed using an `ste_vec` that use different contexts can't be compared. +**Example:** +```json +{"prefix": "users/encrypted_json"} +``` + +Within a dataset, encrypted columns indexed using an `ste_vec` that use different prefixes can't be compared. Containment queries that manage to mix index terms from multiple columns will never return a positive result. This is by design. @@ -217,30 +224,32 @@ When reduced to a prefix list, it would look like this: Which is then turned into an ste_vec of hashes which can be directly queries against the index. -### Modifying an index (`cs_modify_index`) +### Modifying an index (`eql_v2.modify_search_config`) Modifies an existing index configuration. -Accepts the same parameters as `cs_add_index` +Accepts the same parameters as `eql_v2.add_search_config` ```sql -SELECT cs_modify_index_v2( +SELECT eql_v2.modify_search_config( table_name text, column_name text, index_name text, - cast_as text, - opts jsonb + cast_as text DEFAULT 'text', + opts jsonb DEFAULT '{}', + migrating boolean DEFAULT false ); ``` -### Removing an index (`cs_remove_index`) +### Removing an index (`eql_v2.remove_search_config`) Removes an index configuration from the column. ```sql -SELECT cs_remove_index_v2( +SELECT eql_v2.remove_search_config( table_name text, column_name text, - index_name text + index_name text, + migrating boolean DEFAULT false ); ``` diff --git a/docs/reference/json-support.md b/docs/reference/json-support.md index adee765..be79c7f 100644 --- a/docs/reference/json-support.md +++ b/docs/reference/json-support.md @@ -1,6 +1,6 @@ # EQL with JSON and JSONB -EQL supports encrypting, decrypting, and searching JSON and JSONB objects. +EQL supports encrypting, decrypting, and searching JSON and JSONB objects using structured encryption (ste_vec). ## On this page @@ -8,32 +8,17 @@ EQL supports encrypting, decrypting, and searching JSON and JSONB objects. - [Inserting JSON data](#inserting-json-data) - [Reading JSON data](#reading-json-data) - [Querying JSONB data with EQL](#querying-jsonb-data-with-eql) - - [Containment queries (`cs_ste_vec_v2`)](#containment-queries-cs_ste_vec_v2) - - [Field extraction (`cs_ste_vec_value_v2`)](#field-extraction-cs_ste_vec_value_v2) - - [Field comparison (`cs_ste_vec_term_v2`)](#field-comparison-cs_ste_vec_term_v2) + - [Containment queries (`@>`, `<@`)](#containment-queries---) + - [Field extraction (`jsonb_path_query`)](#field-extraction-jsonb_path_query) + - [JSON path operators (`->`, `->>`)](#json-path-operators---) + - [Array operations](#array-operations) - [Grouping data](#grouping-data) - [EQL functions for JSONB and `ste_vec`](#eql-functions-for-jsonb-and-ste_vec) -- [EJSON paths](#ejson-paths) -- [Native PostgreSQL JSON(B) compared to EQL](#native-postgresql-jsonb-compared-to-eql) - - [`json ->> text` → `text` and `json -> text` → `jsonb`/`json`](#json--text--text-and-json---text--jsonbjson) - - [Decryption Example](#decryption-example) - - [Comparison Example](#comparison-example) - - [`json ->> int` → `text` and `json -> int` → `jsonb`/`json`](#json--int--text-and-json--int--jsonbjson) - - [Decryption Example](#decryption-example-1) - - [Comparison Example](#comparison-example-1) - - [`json #>> text[]` → `text` and `json #> text[]` → `jsonb`/`json`](#json--text--text-and-json---text--jsonbjson-1) - - [Decryption Example](#decryption-example-2) - - [Comparison Example](#comparison-example-2) - - [`@>` and `<@`](#and) - - [`json_array_elements`, `jsonb_array_elements`, `json_array_elements_text`, and `jsonb_array_elements_text`](#json_array_elements-jsonb_array_elements-json_array_elements_text-and-jsonb_array_elements_text) - - [Decryption Example](#decryption-example-3) - - [Comparison Example](#comparison-example-3) - - [`json_array_length` and `jsonb_array_length`](#json_array_length-and-jsonb_array_length) +- [How ste_vec indexing works](#how-ste_vec-indexing-works) ## Configuring the index -Similar to how you configure indexes for text data, you can configure indexes for JSON and JSONB data. -The only difference is that you need to specify the `cast_as` parameter as `json` or `jsonb`. +To enable searchable operations on encrypted JSONB data, configure an `ste_vec` index with the `jsonb` cast type. ```sql SELECT eql_v2.add_search_config( @@ -41,32 +26,17 @@ SELECT eql_v2.add_search_config( 'encrypted_json', 'ste_vec', 'jsonb', - '{"prefix": "users/encrypted_json"}' -- The prefix is in the form of "table/column" + '{"prefix": "users/encrypted_json"}' ); ``` -You can read more about the index configuration options [here](https://github.com/cipherstash/encrypt-query-language/blob/main/docs/reference/INDEX.md). +The `prefix` option is required and should be unique per table/column combination (typically `"table/column"`). -### Inserting JSON data - -When inserting JSON data, this works the same as inserting text data. -You need to wrap the JSON data in the appropriate EQL payload. -CipherStash Proxy will **encrypt** the data automatically. - -**Example:** - -Assuming you want to store the following JSON data: +You can read more about the index configuration options [here](./index-config.md). -```json -{ - "name": "John Doe", - "metadata": { - "age": 42 - } -} -``` +### Inserting JSON data -The EQL payload would be: +When inserting JSON data through CipherStash Proxy or Protect.js, wrap the data in the EQL payload format: ```sql INSERT INTO users (encrypted_json) VALUES ( @@ -74,7 +44,7 @@ INSERT INTO users (encrypted_json) VALUES ( ); ``` -Data is stored in the database as: +Data is stored in the database with encrypted ste_vec indexes: ```json { @@ -84,46 +54,29 @@ Data is stored in the database as: }, "k": "sv", "v": 2, - "sv": [["ciphertext"]] + "sv": [["encrypted_term_1"], ["encrypted_term_2"], ...] } ``` ### Reading JSON data -When querying data, select the encrypted column. CipherStash Proxy will **decrypt** the data automatically. - -**Example:** +When querying through CipherStash Proxy or Protect.js, the encrypted column is automatically decrypted: ```sql SELECT encrypted_json FROM users; ``` -Data is returned as: - -```json -{ - "k": "pt", - "p": "{\"metadata\":{\"age\":42},\"name\":\"John Doe\"}", - "i": { - "t": "users", - "c": "encrypted_json" - }, - "v": 2, - "q": null -} -``` - ## Querying JSONB data with EQL -EQL provides specialized functions to interact with encrypted JSONB data, supporting operations like containment queries, field extraction, and comparisons. +EQL provides specialized functions and operators to work with encrypted JSONB data. -### Containment queries (`cs_ste_vec_v2`) +### Containment queries (`@>`, `<@`) -Retrieve the Structured Encryption Vector for JSONB containment queries. +Use PostgreSQL's containment operators directly on `eql_v2_encrypted` columns to check if one JSONB structure contains another. -**Example: Containment query** +**Example: Check if column contains structure** -Suppose we have the following encrypted JSONB data: +Suppose we have encrypted JSONB data: ```json { @@ -133,21 +86,11 @@ Suppose we have the following encrypted JSONB data: } ``` -We can query records that contain a specific structure. - -**SQL query:** +Query records that contain a specific structure: ```sql SELECT * FROM examples -WHERE cs_ste_vec_v2(encrypted_json) @> cs_ste_vec_v2( - '{ - "v":2, - "k":"pt", - "p":{"top":{"nested":["a"]}}, - "i":{"t":"examples","c":"encrypted_json"}, - "q":"ste_vec" - }' -); +WHERE encrypted_json @> '{"v":2,"k":"pt","p":"{\"top\":{\"nested\":[\"a\"]}}","i":{"t":"examples","c":"encrypted_json"},"q":"ste_vec"}'::eql_v2_encrypted; ``` Equivalent plaintext query: @@ -157,730 +100,197 @@ SELECT * FROM examples WHERE jsonb_column @> '{"top":{"nested":["a"]}}'; ``` -**Note:** The `@>` operator checks if the left JSONB value contains the right JSONB value. +**Note:** The `@>` operator checks if the left value contains the right value. The `<@` operator checks the reverse (if left is contained in right). -**Negative example:** +### Field extraction (`jsonb_path_query`) -If we query for a value that does not exist in the data: +Extract fields from encrypted JSONB using selector hashes. Selectors are generated during encryption and identify specific JSON paths. -**SQL query:** +**Function signature:** ```sql -SELECT * FROM examples -WHERE cs_ste_vec_v2(encrypted_json) @> cs_ste_vec_v2( - '{ - "v":2, - "k":"pt", - "p":{"top":{"nested":["d"]}}, - "i":{"t":"examples","c":"encrypted_json"}, - "q":"ste_vec" - }' -); +eql_v2.jsonb_path_query(val eql_v2_encrypted, selector text) RETURNS SETOF eql_v2_encrypted ``` -This query would return no results, as the value `"d"` is not present in the `"nested"` array. - -### Field extraction (`cs_ste_vec_value_v2`) - -Extract a field from an encrypted JSONB object. - **Example:** -Suppose we have the following encrypted JSONB data: - -```json -{ - "top": { - "nested": ["a", "b", "c"] - } -} -``` - -We can extract the value of the `"top"` key. - -**SQL query:** - ```sql -SELECT cs_ste_vec_value_v2(encrypted_json, - '{ - "v":2, - "k":"pt", - "p":"$.top", - "i":{"t":"examples","c":"encrypted_json"}, - "q":"ejson_path" - }' -) AS value +-- Extract all records where selector 'abc123...' exists +SELECT eql_v2.jsonb_path_query(encrypted_json, 'abc123def456...') FROM examples; -``` - -Equivalent plaintext query: -```sql -SELECT jsonb_column->'top' AS value +-- Get first match only +SELECT eql_v2.jsonb_path_query_first(encrypted_json, 'abc123def456...') FROM examples; -``` -**Result:** - -```json -{ - "nested": ["a", "b", "c"] -} +-- Check if selector exists +SELECT eql_v2.jsonb_path_exists(encrypted_json, 'abc123def456...') +FROM examples; ``` -### Field comparison (`cs_ste_vec_term_v2`) +**Note:** Selectors are hash-based identifiers for JSON paths, not the actual path strings like `$.field`. They are generated during encryption by CipherStash Proxy/Protect.js. -Select rows based on a field value in an encrypted JSONB object. +### JSON path operators (`->`, `->>`) -**Example:** +Use standard PostgreSQL JSON operators on encrypted columns: -Suppose we have encrypted JSONB data with a numeric field: +```sql +-- Extract field by selector (returns eql_v2_encrypted) +SELECT encrypted_json->'selector_hash' FROM examples; -```json -{ - "num": 3 -} +-- Extract field as text (returns ciphertext) +SELECT encrypted_json->>'selector_hash' FROM examples; ``` -We can query records where the `"num"` field is greater than `2`. +### Array operations -**SQL query:** +EQL supports array operations on encrypted JSONB arrays: -```sql -SELECT * FROM examples -WHERE cs_ste_vec_term_v2(encrypted_json, - '{ - "v":2, - "k":"pt", - "p":"$.num", - "i":{"t":"examples","c":"encrypted_json"}, - "q":"ejson_path" - }' -) > cs_ste_vec_term_v2( - '{ - "v":2, - "k":"pt", - "p":"2", - "i":{"t":"examples","c":"encrypted_json"}, - "q":"ste_vec" - }' -); -``` - -Equivalent plaintext query: +**Get array length:** ```sql -SELECT * FROM examples -WHERE (jsonb_column->>'num')::int > 2; +SELECT eql_v2.jsonb_array_length(encrypted_array_field) +FROM examples; ``` -### Grouping data - -Use `cs_ste_vec_term_v2` along with `cs_grouped_value_v2` to group by a field in an encrypted JSONB column. - -**Example:** +**Get array elements:** -Suppose we have records with a `"color"` field: +```sql +-- Returns SETOF eql_v2_encrypted +SELECT eql_v2.jsonb_array_elements(encrypted_array_field) +FROM examples; -```json -{"color": "blue"} -{"color": "blue"} -{"color": "green"} -{"color": "blue"} -{"color": "red"} -{"color": "green"} +-- Returns SETOF text (ciphertext) +SELECT eql_v2.jsonb_array_elements_text(encrypted_array_field) +FROM examples; ``` -We can group the data by the `"color"` field and count occurrences. - -**SQL query:** +**Example with jsonb_path_query:** ```sql -SELECT cs_grouped_value_v2(cs_ste_vec_value_v2(encrypted_json, - '{ - "v":2, - "k":"pt", - "p":"$.color", - "i":{"t":"examples","c":"encrypted_json"}, - "q":"ejson_path" - }' -)) AS color, COUNT(*) -FROM examples -GROUP BY cs_ste_vec_term_v2(encrypted_json, - '{ - "v":2, - "k":"pt", - "p":"$.color", - "i":{"t":"examples","c":"encrypted_json"}, - "q":"ejson_path" - }' -); +-- First query the array field, then get its elements +SELECT eql_v2.jsonb_array_elements( + eql_v2.jsonb_path_query(encrypted_json, 'array_selector_hash') +) +FROM examples; ``` -Equivalent plaintext query: +### Grouping data + +Use `eql_v2.grouped_value()` aggregate function to group encrypted JSONB results: ```sql -SELECT jsonb_column->>'color' AS color, COUNT(*) +SELECT eql_v2.grouped_value( + eql_v2.jsonb_path_query_first(encrypted_json, 'color_selector')::jsonb +) AS color, +COUNT(*) FROM examples -GROUP BY jsonb_column->>'color'; +GROUP BY eql_v2.jsonb_path_query_first(encrypted_json, 'color_selector'); ``` **Result:** | color | count | | ----- | ----- | -| blue | 3 | -| green | 2 | -| red | 1 | +| {"k":"pt","p":"blue",...} | 3 | +| {"k":"pt","p":"green",...} | 2 | +| {"k":"pt","p":"red",...} | 1 | -## EQL Functions for JSONB and `ste_vec` +## EQL functions for JSONB and `ste_vec` -- **Index management** +### Core Functions - - `cs_add_index_v2(table_name text, column_name text, 'ste_vec', 'jsonb', opts jsonb)`: Adds an `ste_vec` index configuration. - - `opts` must include the `"context"` key. +- **`eql_v2.ste_vec(val eql_v2_encrypted) RETURNS eql_v2_encrypted[]`** + - Extracts the ste_vec index array from an encrypted value -- **Query functions** +- **`eql_v2.ste_vec_contains(a eql_v2_encrypted, b eql_v2_encrypted) RETURNS boolean`** + - Returns true if all ste_vec terms in b exist in a + - This is the function backing the `@>` operator - - `cs_ste_vec_v2(val jsonb)`: Retrieves the STE vector for JSONB containment queries. - - `cs_ste_vec_term_v2(val jsonb, epath jsonb)`: Retrieves the encrypted term associated with an encrypted JSON path. - - `cs_ste_vec_value_v2(val jsonb, epath jsonb)`: Retrieves the decrypted value associated with an encrypted JSON path. - - `cs_ste_vec_terms_v2(val jsonb, epath jsonb)`: Retrieves an array of encrypted terms for elements in an array at the given JSON path (used for comparisons). - - `cs_grouped_value_v2(val jsonb)`: Used with `ste_vec` indexes for grouping. +### Path Query Functions -## EJSON paths +- **`eql_v2.jsonb_path_query(val eql_v2_encrypted, selector text) RETURNS SETOF eql_v2_encrypted`** + - Returns all encrypted elements matching the selector -EQL uses an extended JSONPath syntax called EJSONPath for specifying paths in JSONB data. +- **`eql_v2.jsonb_path_query_first(val eql_v2_encrypted, selector text) RETURNS eql_v2_encrypted`** + - Returns the first encrypted element matching the selector -- Root selector: `$` -- Dot notation for keys: `$.key` -- Bracket notation for keys with special characters: `$['key.with.special*chars']` -- Wildcards are supported: `$.some_array_field[*]` -- Array indexing is **not** supported: `$.some_array_field[0]` +- **`eql_v2.jsonb_path_exists(val eql_v2_encrypted, selector text) RETURNS boolean`** + - Returns true if any element matches the selector -**Example paths:** +### Array Functions -- `$.top.nested` selects the `"nested"` key within the `"top"` object. -- `$.array[*]` selects all elements in the `"array"` array. +- **`eql_v2.jsonb_array_length(val eql_v2_encrypted) RETURNS integer`** + - Returns the length of an encrypted array ---- - -## Native PostgreSQL JSON(B) compared to EQL - -EQL supports a subset of functionality supported by the native PostgreSQL JSON(B) functions and operators. -The following examples compare native PostgreSQL JSON(B) functions and operators to the related functionality in EQL. - -### `json ->> text` → `text` and `json -> text` → `jsonb`/`json` - -**Native PostgreSQL JSON(B) example** - -```sql --- `->` (returns JSON(B)) -SELECT plaintext_jsonb->'field_a' FROM examples; - --- `->>` (returns text) -SELECT plaintext_jsonb->>'field_a' FROM examples; -``` - -**EQL example** - -EQL JSONB functions accept an [eJSONPath](#ejson-paths) as an argument (instead of using `->`/`->>`) for lookups. - -#### Decryption example - -`cs_ste_vec_value_v2` returns the plaintext EQL payload to the client. - -```sql -SELECT cs_ste_vec_value_v2(encrypted_json, $1) FROM examples; -``` - -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: -{ - "field_a": 100 -} - -// `$1` is the EQL plaintext payload for the eJSONPath `$.field_a`: -{ - "k": "pt", - "p": "$.field_a", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ejson_path" -} -``` - -#### Comparison example - -`cs_ste_vec_term_v2` returns an ORE term for comparison. - -```sql -SELECT * FROM examples -WHERE cs_ste_vec_term_v2(examples.encrypted_json, $1) > cs_ste_vec_term_v2($2) -``` - -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: -{ - "field_a": 100 -} - -// `$1` is the EQL plaintext payload for the eJSONPath `$.field_a`: -{ - "k": "pt", - "p": "$.field_a", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ejson_path" -} - -// `$2` is the EQL plaintext payload for the ORE term to compare against (in this case, the ORE term for the integer `123`): -{ - "k": "pt", - "p": "123", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ste_vec" -} -``` - -### `json ->> int` → `text` and `json -> int` → `jsonb`/`json` +- **`eql_v2.jsonb_array_elements(val eql_v2_encrypted) RETURNS SETOF eql_v2_encrypted`** + - Returns each array element as an encrypted value -**Native PostgreSQL JSON(B) example** +- **`eql_v2.jsonb_array_elements_text(val eql_v2_encrypted) RETURNS SETOF text`** + - Returns each array element's ciphertext as text -```sql --- `->` (returns JSON(B)) -SELECT plaintext_jsonb->0 FROM examples; +### Helper Functions --- `->>` (returns text) -SELECT plaintext_jsonb->>0 FROM examples; -``` +- **`eql_v2.is_ste_vec_array(val eql_v2_encrypted) RETURNS boolean`** + - Returns true if the value represents an encrypted array -**EQL example** +- **`eql_v2.is_ste_vec_value(val eql_v2_encrypted) RETURNS boolean`** + - Returns true if the value is a single ste_vec element -EQL JSONB functions accept an [eJSONPath](#ejson-paths) as an argument (instead of using `->`/`->>`) for lookups. +- **`eql_v2.to_ste_vec_value(val eql_v2_encrypted) RETURNS eql_v2_encrypted`** + - Converts a ste_vec array with a single element to a regular encrypted value -#### Decryption example +- **`eql_v2.selector(val eql_v2_encrypted) RETURNS text`** + - Extracts the selector hash from an encrypted value -EQL currently doesn't support returning a specific array element for decryption, but `cs_ste_vec_value_v2` can be used to return an array to the client to process. +### Aggregate Functions -The query: +- **`eql_v2.grouped_value(jsonb) RETURNS jsonb`** + - Aggregate function for grouping encrypted values (returns first non-null value in group) -```sql -SELECT cs_ste_vec_value_v2(encrypted_json, $1) AS val FROM examples; -``` +## How ste_vec indexing works -With the params: +Structured Encryption (ste_vec) creates searchable indexes for JSONB by: -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: -{ - "field_a": [1, 2, 3] -} +1. **Flattening the JSON structure** - Each unique path to a leaf value gets a selector (hash) +2. **Creating encrypted terms** - Each path prefix and value is encrypted separately +3. **Storing as array** - All encrypted terms are stored in the `sv` (ste_vec) array -// `$1` is the EQL plaintext payload for the eJSONPath `$.field_a`: -{ - "k": "pt", - "p": "$.field_a", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ejson_path" -} -``` - -Would return the EQL plaintext payload with an array (`[1, 2, 3]` for example): - -```javascript -// Example result for a single row -{ - "k": "pt", - "p": "[1, 2, 3]", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": null -} -``` - -#### Comparison example - -`cs_ste_vec_terms_v2` can be used with the native PostgreSQL array access operator to get a term for comparison by array index. - -The eJSONPath used with `cs_ste_vec_terms_v2` needs to end with `[*]` (`$.some_array_field[*]` for example). - -> [!IMPORTANT] -> Array access with `cs_ste_vec_terms_v2` only works when the given eJSONPath only matches a single array. -> Accessing array elements from `cs_ste_vec_terms_v2` when the eJSONPath matches multiple arrays (for example, when there are nested arrays or multiple arrays at the same depth) can return unexpected results. - -The following query compares the first item in the array at the eJSONPath in `$1` to the value in `$2`. - -```sql -SELECT * FROM examples -WHERE (cs_ste_vec_terms_v2(examples.encrypted_json, $1))[1] > cs_ste_vec_term_v2($2) -``` +**Example document:** -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: -{ - "field_a": [4, 5, 6] -} - -// `$1` is the EQL plaintext payload for the eJSONPath `$.field_a[*]`: -{ - "k": "pt", - "p": "$.field_a[*]", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ejson_path" -} - -// `$2` is the EQL plaintext payload for the ORE term to compare against (in this case, the ORE term for the integer `3`): -{ - "k": "pt", - "p": "3", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ste_vec" -} -``` - -### `json #>> text[]` → `text` and `json #> text[]` → `jsonb`/`json` - -**Native PostgreSQL JSON(B) example** - -```sql --- `#>` (returns JSON(B)) -SELECT plaintext_jsonb#>'{field_a,field_b}' FROM examples; - --- `#>>` (returns text) -SELECT plaintext_jsonb#>>'{field_a,field_b}' FROM examples; -``` - -**EQL example** - -EQL JSONB functions accept an [eJSONPath](#ejson-paths) as an argument (instead of using `#>`/`#>>`) for lookups. - -Note that these are similar to the examples for `->`/`->>`. -The difference in these examples is that the path does a lookup multiple levels deep. - -#### Decryption example - -`cs_ste_vec_value_v2` returns the plaintext EQL payload to the client. - -```sql -SELECT cs_ste_vec_value_v2(encrypted_json, $1) FROM examples; -``` - -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: -{ - "field_a": { - "field_b": 100 - } -} - -// `$1` is the EQL plaintext payload for the eJSONPath `$.field_a.field_b`: -{ - "k": "pt", - "p": "$.field_a.field_b", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ejson_path" -} -``` - -#### Comparison example - -`cs_ste_vec_term_v2` returns an ORE term for comparison. - -```sql -SELECT * FROM examples -WHERE cs_ste_vec_term_v2(examples.encrypted_json, $1) > cs_ste_vec_term_v2($2) -``` - -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: -{ - "field_a": { - "field_b": 100 - } -} - -// `$1` is the EQL plaintext payload for the eJSONPath `$.field_a.field_b`: -{ - "k": "pt", - "p": "$.field_a.field_b", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ejson_path" -} - -// `$2` is the EQL plaintext payload for the ORE term to compare against (in this case, the ORE term for the integer `123`): -{ - "k": "pt", - "p": "123", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ste_vec" -} -``` - -### `@>` and `<@` - -**Native PostgreSQL JSON(B) example** - -```sql --- Checks if the left arg contains the right arg (returns `true` in this example). -SELECT '{"a":1, "b":2}'::jsonb @> '{"b":2}'::jsonb; - --- Checks if the right arg contains the left arg (returns `true` in this example). -SELECT '{"b":2}'::jsonb <@ '{"a":1, "b":2}'::jsonb; -``` - -**EQL example** - -EQL uses the same operators for containment (`@>` and `<@`) queries, but the args need to be wrapped in `cs_ste_vec_v2`. - -Example query: - -```sql --- Checks if the left arg (the `examples.encrypted_json` column) contains the right arg ($1). --- Would return `true` for the example data and param below. -SELECT * WHERE cs_ste_vec_v2(encrypted_json) @> cs_ste_vec_v2($1) FROM examples; - --- Checks if the the right arg ($1) contains left arg (the `examples.encrypted_json` column). --- Would return `false` for the example data and param below. -SELECT * WHERE cs_ste_vec_v2(encrypted_json) <@ cs_ste_vec_v2($1) FROM examples; -``` - -Example params: - -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: +```json { - "field_a": { - "field_b": [1, 2, 3] + "account": { + "email": "alice@example.com", + "roles": ["admin", "owner"] } } - -// `$1` is the EQL plaintext payload for the JSON object `{"field_b": [1, 2, 3]}`: -{ - "k": "pt", - "p": "{\"field_b\": [1, 2, 3]}", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ste_vec" -} -``` - -### `json_array_elements`, `jsonb_array_elements`, `json_array_elements_text`, and `jsonb_array_elements_text` - -**Native PostgreSQL JSON(B) example** - -```sql --- Each returns the results... --- --- Value --- _____ --- a --- b --- --- The only difference is that the input is either json or jsonb (depending --- on the prefix of the function name) and the output is either json, --- jsonb, or text (depending on both the prefix and the suffix). - -SELECT * from json_array_elements('["a", "b"]'); -SELECT * from jsonb_array_elements('["a", "b"]'); -SELECT * from json_array_elements_text('["a", "b"]'); -SELECT * from jsonb_array_elements_text('["a", "b"]'); -``` - -**EQL example** - -#### Decryption example - -EQL currently doesn't support returning a `SETOF` values for decryption (for returning a row per item in an array), but `cs_ste_vec_value_v2` can be used to return an array to the client to process. - -The query: - -```sql -SELECT cs_ste_vec_value_v2(encrypted_json, $1) AS val FROM examples; -``` - -With the params: - -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: -{ - "field_a": [1, 2, 3] -} - -// `$1` is the EQL plaintext payload for the eJSONPath `$.field_a`: -{ - "k": "pt", - "p": "$.field_a", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ejson_path" -} -``` - -Would return the EQL plaintext payload with an array (`[1, 2, 3]` for example): - -```javascript -// Example result for a single row -{ - "k": "pt", - "p": "[1, 2, 3]", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": null -} -``` - -#### Comparison example - -`cs_ste_vec_terms_v2` (note that terms is plural) can be used to return an array of ORE terms for comparison. -The array can be `unnest`ed to work with a `SETOF` ORE terms for comparison. - -The eJSONPath used with `cs_ste_vec_terms_v2` needs to end with `[*]` (`$.some_array_field[*]` for example). - -Example query: - -```sql -SELECT id FROM examples e -WHERE EXISTS ( - SELECT 1 - FROM unnest(cs_ste_vec_terms_v2(e.encrypted_json, $1)) AS term - WHERE term > cs_ste_vec_term_v2($2) -); ``` -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: -{ - "field_a": [1, 2, 3] -} +**Creates selectors for:** +- `$` (root object) +- `$.account` (account object) +- `$.account.email` (email field) +- `$.account.email` with value "alice@example.com" +- `$.account.roles` (roles array) +- `$.account.roles[]` (each role value) -// `$1` is the EQL plaintext payload for the eJSONPath `$.field_a[*]`: -{ - "k": "pt", - "p": "$.field_a[*]", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ejson_path" -} +**Querying:** -// `$2` is the EQL plaintext payload for the ORE term to compare against (in this case, the ORE term for the integer `2`): -{ - "k": "pt", - "p": "2", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ste_vec" -} -``` - -### `json_array_length` and `jsonb_array_length` - -**Native PostgreSQL JSON(B) example** +Containment queries (`@>`) check if all required encrypted terms exist in the target's ste_vec array. This enables queries like: ```sql --- Both of these examples return the int `3`. --- The only difference is the input type. -SELECT json_array_length('[1, 2, 3]'); -SELECT jsonb_array_length('[1, 2, 3]'); -``` - -**EQL example** - -The PostgreSQL `array_length` function can be used with `cs_ste_vec_terms_v2` to find the length of an array. +-- Find records where account.email = "alice@example.com" +WHERE encrypted_data @> ''::eql_v2_encrypted -The eJSONPath used with `cs_ste_vec_terms_v2` needs to end with `[*]` (`$.some_array_field[*]` for example). - -> [!IMPORTANT] -> Determining array length with `cs_ste_vec_terms_v2` only works when the given eJSONPath only matches a single array. -> Attempting to determine array length using `cs_ste_vec_terms_v2` when the eJSONPath matches multiple arrays (for example, when there are nested arrays or multiple arrays at the same depth) can return unexpected results. - -Example query: - -```sql -SELECT COALESCE( -- We `COALESCE` because cs_ste_vec_terms_v2 will return `NULL` for empty arrays. - array_length( -- `cs_ste_vec_terms_v2` returns an array type (not JSON(B)), so we use `array_length`. - cs_ste_vec_terms_v2(encrypted_json, $1), -- Pluck out the array of terms at the path in $1. - 1 -- The array dimension to find the length of (term array are flat, so this should always be 1). - ), - 0 -- Assume a length of `0` when `cs_ste_vec_terms_v2` returns `NULL`. -) AS len FROM examples; +-- Find records where account.roles contains "admin" +WHERE encrypted_data @> ''::eql_v2_encrypted ``` -Example data and params: - -```javascript -// Assume that examples.encrypted_json has JSON objects with the shape: -{ - "val": [1, 2, 3] -} - -// `$1` is the EQL plaintext payload for the eJSONPath `$.val[*]`: -{ - "k": "pt", - "p": "$.val[*]", - "i": { - "t": "examples", - "c": "encrypted_json" - }, - "v": 2, - "q": "ejson_path" -} -``` +The actual encryption and selector generation is handled by CipherStash Proxy or Protect.js, not by EQL directly. --- ### Didn't find what you wanted? -[Click here to let us know what was missing from our docs.](https://github.com/cipherstash/encrypt-query-language/issues/new?template=docs-feedback.yml&title=[Docs:]%20Feedback%20on%20JSON.md) +[Click here to let us know what was missing from our docs.](https://github.com/cipherstash/encrypt-query-language/issues/new?template=docs-feedback.yml&title=[Docs:]%20Feedback%20on%20json-support.md) diff --git a/docs/tutorials/proxy-configuration.md b/docs/tutorials/proxy-configuration.md index a09c647..758829f 100644 --- a/docs/tutorials/proxy-configuration.md +++ b/docs/tutorials/proxy-configuration.md @@ -71,7 +71,7 @@ SELECT eql_v2.reload_config(); Encrypted data is stored as `jsonb` values in the PostgreSQL database, regardless of the original data type. -You can read more about the data format [here](docs/reference/payload.md). +You can read more about the data format [here](../reference/PAYLOAD.md). ### Inserting data @@ -153,7 +153,7 @@ SELECT eql_v2.add_search_config( ); ``` -You can read more about the index configuration options [here](docs/reference/index-config.md). +You can read more about the index configuration options [here](../reference/index-config.md). **Example (Unique index):** @@ -189,7 +189,7 @@ In order to use the specialized functions, you must first configure the correspo ### Equality search -Enable equality search on encrypted data using the `eql_v2.hmac_256` function. +Enable exact equality search on encrypted data using the `unique` index (backed by hmac_256 or blake3). **Index configuration example:** @@ -202,12 +202,20 @@ SELECT eql_v2.add_search_config( ); ``` -**Example:** +**Query using operators (recommended):** ```sql +-- Use the = operator directly on the encrypted column SELECT * FROM users -WHERE eql_v2.hmac_256(encrypted_email) = eql_v2.hmac_256( - '{"v":2,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"},"q":"hmac_256"}' +WHERE encrypted_email = '{"v":2,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"}}'::eql_v2_encrypted; +``` + +**Query using functions (for Supabase or operator-restricted environments):** + +```sql +SELECT * FROM users +WHERE eql_v2.eq(encrypted_email, + '{"v":2,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"}}'::eql_v2_encrypted ); ``` @@ -219,38 +227,50 @@ SELECT * FROM users WHERE email = 'test@example.com'; ### Full-text search -Enables basic full-text search on encrypted data using the `eql_v2.bloom_filter` function. +Enables full-text search on encrypted data using the `match` index (backed by bloom filters). **Index configuration example:** ```sql SELECT eql_v2.add_search_config( 'users', - 'encrypted_email', + 'encrypted_name', 'match', 'text', '{"token_filters": [{"kind": "downcase"}], "tokenizer": { "kind": "ngram", "token_length": 3 }}' ); ``` -**Example:** +**Query using operators (recommended):** ```sql +-- Use the ~~ (LIKE) operator directly on the encrypted column +SELECT * FROM users +WHERE encrypted_name ~~ '{"v":2,"k":"pt","p":"alice","i":{"t":"users","c":"encrypted_name"}}'::eql_v2_encrypted; + +-- Case-insensitive search with ~~* (ILIKE) SELECT * FROM users -WHERE eql_v2.bloom_filter(encrypted_email) @> eql_v2.bloom_filter( - '{"v":2,"k":"pt","p":"test","i":{"t":"users","c":"encrypted_email"},"q":"match"}' +WHERE encrypted_name ~~* '{"v":2,"k":"pt","p":"alice","i":{"t":"users","c":"encrypted_name"}}'::eql_v2_encrypted; +``` + +**Query using functions (for Supabase or operator-restricted environments):** + +```sql +SELECT * FROM users +WHERE eql_v2.like(encrypted_name, + '{"v":2,"k":"pt","p":"alice","i":{"t":"users","c":"encrypted_name"}}'::eql_v2_encrypted ); ``` Equivalent plaintext query: ```sql -SELECT * FROM users WHERE email LIKE '%test%'; +SELECT * FROM users WHERE name LIKE '%alice%'; ``` ### Range queries -Enable range queries on encrypted data using the `eql_v2.ore_block_u64_8_256` function. Supports: +Enable range queries and ordering on encrypted data using the `ore` index (Order-Revealing Encryption). Supports: - `ORDER BY` - `WHERE` with comparison operators (`<`, `<=`, `>`, `>=`, `=`, `<>`) @@ -259,39 +279,34 @@ Enable range queries on encrypted data using the `eql_v2.ore_block_u64_8_256` fu ```sql SELECT eql_v2.add_search_config( - 'users', + 'events', 'encrypted_date', 'ore', 'date' ); ``` -**Example (Filtering):** +**Query using operators (recommended):** ```sql -SELECT * FROM users -WHERE eql_v2.ore_block_u64_8_256(encrypted_date) < eql_v2.ore_block_u64_8_256( - '{"v":2,"k":"pt","p":"2023-10-05","i":{"t":"users","c":"encrypted_date"},"q":"ore"}' -); -``` +-- Range comparison - use comparison operators directly +SELECT * FROM events +WHERE encrypted_date < '{"v":2,"k":"pt","p":"2023-10-05","i":{"t":"events","c":"encrypted_date"}}'::eql_v2_encrypted; -Equivalent plaintext query: +SELECT * FROM events +WHERE encrypted_date >= '{"v":2,"k":"pt","p":"2023-01-01","i":{"t":"events","c":"encrypted_date"}}'::eql_v2_encrypted; -```sql -SELECT * FROM users WHERE date < '2023-10-05'; -``` - -**Example (Ordering):** - -```sql -SELECT id FROM users -ORDER BY eql_v2.ore_block_u64_8_256(encrypted_field) DESC; +-- Ordering - use ORDER BY directly +SELECT * FROM events ORDER BY encrypted_date DESC; +SELECT * FROM events ORDER BY encrypted_date ASC; ``` -Equivalent plaintext query: +Equivalent plaintext queries: ```sql -SELECT id FROM users ORDER BY field DESC; +SELECT * FROM events WHERE date < '2023-10-05'; +SELECT * FROM events WHERE date >= '2023-01-01'; +SELECT * FROM events ORDER BY date DESC; ``` ### Array Operations @@ -353,7 +368,7 @@ WHERE encrypted_name ~~* '{"v":2,"k":"pt","p":"alice%","i":{"t":"users","c":"enc EQL supports encrypting entire JSON and JSONB data sets. This warrants a separate section in the documentation. -You can read more about the JSONB support in the [JSONB reference guide](docs/reference/json-support.md). +You can read more about the JSONB support in the [JSONB reference guide](../reference/json-support.md). ## Frequently Asked Questions diff --git a/eql-build/Cargo.toml b/eql-build/Cargo.toml new file mode 100644 index 0000000..23b1d2a --- /dev/null +++ b/eql-build/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "eql-build" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "eql-build" +path = "src/main.rs" + +[dependencies] +eql-core = { path = "../eql-core" } +eql-postgres = { path = "../eql-postgres" } +anyhow = { workspace = true } diff --git a/eql-build/src/builder.rs b/eql-build/src/builder.rs new file mode 100644 index 0000000..ea950ec --- /dev/null +++ b/eql-build/src/builder.rs @@ -0,0 +1,59 @@ +//! SQL file builder with dependency management + +use anyhow::{Context, Result}; +use std::fs; + +pub struct Builder { + header: String, + files: Vec, +} + +impl Builder { + pub fn new(title: &str) -> Self { + Self { + header: format!("-- {}\n-- Generated by eql-build\n\n", title), + files: Vec::new(), + } + } + + pub fn add_sql_file(&mut self, path: &str) -> Result<()> { + let sql = fs::read_to_string(path) + .with_context(|| format!("Failed to read SQL file: {}", path))?; + + // Remove REQUIRE comments (they're metadata for old build system) + let cleaned = sql + .lines() + .filter(|line| !line.trim_start().starts_with("-- REQUIRE:")) + .collect::>() + .join("\n"); + + self.files.push(cleaned); + Ok(()) + } + + pub fn build(self) -> String { + let mut output = self.header; + + for (i, file) in self.files.iter().enumerate() { + if i > 0 { + output.push_str("\n\n"); + } + output.push_str(file); + } + + output + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_builder_basic() { + let builder = Builder::new("Test"); + let output = builder.build(); + assert!(output.contains("Test")); + assert!(output.contains("Generated by eql-build")); + } +} diff --git a/eql-build/src/main.rs b/eql-build/src/main.rs new file mode 100644 index 0000000..11eae85 --- /dev/null +++ b/eql-build/src/main.rs @@ -0,0 +1,112 @@ +//! Build tool for extracting SQL files in dependency order + +use anyhow::Result; +use std::fs; + +mod builder; + +use builder::Builder; + +fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + + if args.len() < 2 { + eprintln!("Usage: eql-build "); + eprintln!(" database: postgres"); + std::process::exit(1); + } + + let database = &args[1]; + + match database.as_str() { + "postgres" => build_postgres()?, + _ => anyhow::bail!("Unknown database: {}", database), + } + + Ok(()) +} + +fn build_postgres() -> Result<()> { + use eql_postgres::config::AddColumn; + use eql_core::Component; + + println!("Building PostgreSQL installer..."); + + let mut builder = Builder::new("CipherStash EQL for PostgreSQL"); + + // Use automatic dependency resolution + let deps = AddColumn::collect_dependencies(); + println!("Resolved {} dependencies", deps.len()); + + for (i, sql_file) in deps.iter().enumerate() { + println!(" {}. {}", i + 1, sql_file.split('/').last().unwrap_or(sql_file)); + builder.add_sql_file(sql_file)?; + } + + // Write output + fs::create_dir_all("release")?; + let output = builder.build(); + fs::write("release/cipherstash-encrypt-postgres-poc.sql", output)?; + + println!("✓ Generated release/cipherstash-encrypt-postgres-poc.sql"); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_creates_output_file() { + // Clean up any previous output + let _ = std::fs::remove_file("release/cipherstash-encrypt-postgres-poc.sql"); + + // Run build + build_postgres().expect("Build should succeed"); + + // Verify output exists + assert!( + std::path::Path::new("release/cipherstash-encrypt-postgres-poc.sql").exists(), + "Build should create output file" + ); + + // Verify it contains expected SQL + let content = std::fs::read_to_string("release/cipherstash-encrypt-postgres-poc.sql") + .expect("Should be able to read output"); + + assert!(content.contains("eql_v2_configuration_state"), "Should contain config types"); + assert!(content.contains("CREATE FUNCTION eql_v2.add_column"), "Should contain add_column function"); + assert!(content.contains("CREATE FUNCTION eql_v2.config_default"), "Should contain helper functions"); + } + + #[test] + fn test_build_dependency_order() { + build_postgres().expect("Build should succeed"); + + let content = std::fs::read_to_string("release/cipherstash-encrypt-postgres-poc.sql") + .expect("Should be able to read output"); + + // types.sql should come before functions_private.sql + let types_pos = content.find("eql_v2_configuration_state") + .expect("Should contain types"); + let private_pos = content.find("CREATE FUNCTION eql_v2.config_default") + .expect("Should contain private functions"); + + assert!( + types_pos < private_pos, + "Types should be defined before functions that use them" + ); + + // check_encrypted should come before add_encrypted_constraint + let check_pos = content.find("CREATE FUNCTION eql_v2.check_encrypted") + .expect("Should contain check_encrypted"); + let constraint_pos = content.find("CREATE FUNCTION eql_v2.add_encrypted_constraint") + .expect("Should contain add_encrypted_constraint"); + + assert!( + check_pos < constraint_pos, + "check_encrypted should be defined before add_encrypted_constraint" + ); + } +} diff --git a/eql-core/Cargo.toml b/eql-core/Cargo.toml new file mode 100644 index 0000000..2802824 --- /dev/null +++ b/eql-core/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "eql-core" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio-postgres = { workspace = true } +paste = { workspace = true } diff --git a/eql-core/src/component.rs b/eql-core/src/component.rs new file mode 100644 index 0000000..6d7899b --- /dev/null +++ b/eql-core/src/component.rs @@ -0,0 +1,350 @@ +//! Component trait for SQL file dependencies + +/// Declare a SQL component with automatic path inference and boilerplate reduction. +/// +/// This macro generates a component struct and its `Component` trait implementation, +/// automatically inferring the SQL file path from the module and component name. +/// +/// # Syntax +/// +/// ```ignore +/// // Infer path from module::ComponentName (converts PascalCase → snake_case) +/// sql_component!(config::AddColumn); +/// sql_component!(config::AddColumn, deps: [Dep1, Dep2, ...]); +/// +/// // Override path when it doesn't match convention +/// sql_component!(config::ConfigTypes => "types.sql"); +/// sql_component!(config::ConfigTypes => "types.sql", deps: [Dep1]); +/// +/// // Full custom path +/// sql_component!(RemoveColumn => "not_implemented.sql"); +/// ``` +/// +/// # Examples +/// +/// ```ignore +/// // Simple component, infers "config/add_column.sql" +/// sql_component!(config::AddColumn, deps: [ +/// ConfigPrivateFunctions, +/// MigrateActivate, +/// ]); +/// +/// // Override filename (still in config/ directory) +/// sql_component!(config::ConfigTypes => "types.sql"); +/// +/// // Custom path (not following module structure) +/// sql_component!(Placeholder => "not_implemented.sql"); +/// ``` +#[macro_export] +macro_rules! sql_component { + // Pattern 1: module::Component (no deps, infer path) + ($module:ident :: $name:ident) => { + $crate::paste::paste! { + pub struct $name; + + impl $crate::Component for $name { + type Dependencies = (); + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/", + stringify!($module), + "/", + stringify!([<$name:snake>]), + ".sql" + ) + } + } + } + }; + + // Pattern 2: module::Component with single dependency (infer path) + ($module:ident :: $name:ident, deps: [$dep:ty]) => { + $crate::paste::paste! { + pub struct $name; + + impl $crate::Component for $name { + type Dependencies = $dep; + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/", + stringify!($module), + "/", + stringify!([<$name:snake>]), + ".sql" + ) + } + } + } + }; + + // Pattern 3: module::Component with multiple dependencies (infer path) + ($module:ident :: $name:ident, deps: [$dep1:ty, $dep2:ty $(, $deps:ty)* $(,)?]) => { + $crate::paste::paste! { + pub struct $name; + + impl $crate::Component for $name { + type Dependencies = ($dep1, $dep2 $(, $deps)*); + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/", + stringify!($module), + "/", + stringify!([<$name:snake>]), + ".sql" + ) + } + } + } + }; + + // Pattern 4: module::Component => "filename.sql" (override filename, keep module) + ($module:ident :: $name:ident => $filename:literal) => { + pub struct $name; + + impl $crate::Component for $name { + type Dependencies = (); + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/", + stringify!($module), + "/", + $filename + ) + } + } + }; + + // Pattern 5: module::Component => "filename.sql" with single dependency + ($module:ident :: $name:ident => $filename:literal, deps: [$dep:ty]) => { + pub struct $name; + + impl $crate::Component for $name { + type Dependencies = $dep; + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/", + stringify!($module), + "/", + $filename + ) + } + } + }; + + // Pattern 6: module::Component => "filename.sql" with multiple dependencies + ($module:ident :: $name:ident => $filename:literal, deps: [$dep1:ty, $dep2:ty $(, $deps:ty)* $(,)?]) => { + pub struct $name; + + impl $crate::Component for $name { + type Dependencies = ($dep1, $dep2 $(, $deps)*); + + fn sql_file() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/sql/", + stringify!($module), + "/", + $filename + ) + } + } + }; + + // Pattern 7: Component => "full/path.sql" (complete path override, no module) + ($name:ident => $path:literal) => { + pub struct $name; + + impl $crate::Component for $name { + type Dependencies = (); + + fn sql_file() -> &'static str { + concat!(env!("CARGO_MANIFEST_DIR"), "/src/sql/", $path) + } + } + }; + + // Pattern 8: Component => "full/path.sql" with single dependency + ($name:ident => $path:literal, deps: [$dep:ty]) => { + pub struct $name; + + impl $crate::Component for $name { + type Dependencies = $dep; + + fn sql_file() -> &'static str { + concat!(env!("CARGO_MANIFEST_DIR"), "/src/sql/", $path) + } + } + }; + + // Pattern 9: Component => "full/path.sql" with multiple dependencies + ($name:ident => $path:literal, deps: [$dep1:ty, $dep2:ty $(, $deps:ty)* $(,)?]) => { + pub struct $name; + + impl $crate::Component for $name { + type Dependencies = ($dep1, $dep2 $(, $deps)*); + + fn sql_file() -> &'static str { + concat!(env!("CARGO_MANIFEST_DIR"), "/src/sql/", $path) + } + } + }; +} + +/// Marker trait for dependency specifications +pub trait Dependencies { + /// Collect all dependency SQL files in dependency order (dependencies first) + fn collect_sql_files(files: &mut Vec<&'static str>); +} + +/// Unit type represents no dependencies +impl Dependencies for () { + fn collect_sql_files(_files: &mut Vec<&'static str>) { + // No dependencies + } +} + +/// Single dependency +impl Dependencies for T { + fn collect_sql_files(files: &mut Vec<&'static str>) { + // First collect transitive dependencies + T::Dependencies::collect_sql_files(files); + // Then add this dependency + if !files.contains(&T::sql_file()) { + files.push(T::sql_file()); + } + } +} + +/// Two dependencies +impl Dependencies for (A, B) { + fn collect_sql_files(files: &mut Vec<&'static str>) { + A::Dependencies::collect_sql_files(files); + if !files.contains(&A::sql_file()) { + files.push(A::sql_file()); + } + B::Dependencies::collect_sql_files(files); + if !files.contains(&B::sql_file()) { + files.push(B::sql_file()); + } + } +} + +/// Three dependencies +impl Dependencies for (A, B, C) { + fn collect_sql_files(files: &mut Vec<&'static str>) { + A::Dependencies::collect_sql_files(files); + if !files.contains(&A::sql_file()) { + files.push(A::sql_file()); + } + B::Dependencies::collect_sql_files(files); + if !files.contains(&B::sql_file()) { + files.push(B::sql_file()); + } + C::Dependencies::collect_sql_files(files); + if !files.contains(&C::sql_file()) { + files.push(C::sql_file()); + } + } +} + +/// Four dependencies +impl Dependencies for (A, B, C, D) { + fn collect_sql_files(files: &mut Vec<&'static str>) { + A::Dependencies::collect_sql_files(files); + if !files.contains(&A::sql_file()) { + files.push(A::sql_file()); + } + B::Dependencies::collect_sql_files(files); + if !files.contains(&B::sql_file()) { + files.push(B::sql_file()); + } + C::Dependencies::collect_sql_files(files); + if !files.contains(&C::sql_file()) { + files.push(C::sql_file()); + } + D::Dependencies::collect_sql_files(files); + if !files.contains(&D::sql_file()) { + files.push(D::sql_file()); + } + } +} + +/// A component represents a single SQL file with its dependencies +pub trait Component { + /// Type specifying what this component depends on + type Dependencies: Dependencies; + + /// Path to the SQL file containing this component's implementation + fn sql_file() -> &'static str; + + /// Collect this component and all its dependencies in load order + fn collect_dependencies() -> Vec<&'static str> { + let mut files = Vec::new(); + // First collect all transitive dependencies + Self::Dependencies::collect_sql_files(&mut files); + // Then add self + if !files.contains(&Self::sql_file()) { + files.push(Self::sql_file()); + } + files + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct A; + impl Component for A { + type Dependencies = (); + fn sql_file() -> &'static str { "a.sql" } + } + + struct B; + impl Component for B { + type Dependencies = A; + fn sql_file() -> &'static str { "b.sql" } + } + + struct C; + impl Component for C { + type Dependencies = (A, B); + fn sql_file() -> &'static str { "c.sql" } + } + + #[test] + fn test_no_dependencies() { + let deps = A::collect_dependencies(); + assert_eq!(deps, vec!["a.sql"]); + } + + #[test] + fn test_single_dependency() { + let deps = B::collect_dependencies(); + assert_eq!(deps, vec!["a.sql", "b.sql"]); + } + + #[test] + fn test_multiple_dependencies() { + let deps = C::collect_dependencies(); + assert_eq!(deps, vec!["a.sql", "b.sql", "c.sql"]); + } + + #[test] + fn test_deduplication() { + // C depends on both A and B, but A should only appear once + let deps = C::collect_dependencies(); + let a_count = deps.iter().filter(|&&f| f == "a.sql").count(); + assert_eq!(a_count, 1, "a.sql should only appear once"); + } +} diff --git a/eql-core/src/config.rs b/eql-core/src/config.rs new file mode 100644 index 0000000..a722619 --- /dev/null +++ b/eql-core/src/config.rs @@ -0,0 +1,63 @@ +//! Configuration management trait + +use crate::Component; + +/// Configuration management functions for encrypted columns +pub trait Config { + type AddColumnComponent: Component; + type RemoveColumnComponent: Component; + type AddSearchConfigComponent: Component; + + /// Add a column for encryption/decryption. + /// + /// Initializes a column to work with CipherStash encryption. The column + /// must be of type `eql_v2_encrypted`. + /// + /// # Parameters + /// + /// - `table_name` - Name of the table containing the column + /// - `column_name` - Name of the column to configure + /// - `cast_as` - PostgreSQL type for decrypted data (default: 'text') + /// - `migrating` - Whether this is part of a migration (default: false) + /// + /// # Returns + /// + /// JSONB containing the updated configuration. + /// + /// # Examples + /// + /// ```sql + /// -- Configure a text column for encryption + /// SELECT eql_v2.add_column('users', 'encrypted_email', 'text'); + /// + /// -- Configure a JSONB column + /// SELECT eql_v2.add_column('users', 'encrypted_data', 'jsonb'); + /// ``` + fn add_column() -> &'static Self::AddColumnComponent; + + /// Remove column configuration completely. + /// + /// # Examples + /// + /// ```sql + /// SELECT eql_v2.remove_column('users', 'encrypted_email'); + /// ``` + fn remove_column() -> &'static Self::RemoveColumnComponent; + + /// Add a searchable index to an encrypted column. + /// + /// # Supported index types + /// + /// - `unique` - Exact equality (uses hmac_256 or blake3) + /// - `match` - Full-text search (uses bloom_filter) + /// - `ore` - Range queries and ordering (uses ore_block_u64_8_256) + /// - `ste_vec` - JSONB containment queries (uses structured encryption) + /// + /// # Examples + /// + /// ```sql + /// SELECT eql_v2.add_search_config('users', 'encrypted_email', 'unique', 'text'); + /// SELECT eql_v2.add_search_config('docs', 'encrypted_content', 'match', 'text'); + /// ``` + fn add_search_config() -> &'static Self::AddSearchConfigComponent; +} diff --git a/eql-core/src/error.rs b/eql-core/src/error.rs new file mode 100644 index 0000000..052458d --- /dev/null +++ b/eql-core/src/error.rs @@ -0,0 +1,59 @@ +//! Error types for EQL operations + +use thiserror::Error; + +/// Top-level error type for all EQL operations +#[derive(Error, Debug)] +pub enum EqlError { + #[error("Component error: {0}")] + Component(#[from] ComponentError), + + #[error("Database error: {0}")] + Database(#[from] DatabaseError), +} + +/// Errors related to SQL components and dependencies +#[derive(Error, Debug)] +pub enum ComponentError { + #[error("SQL file not found: {path}")] + SqlFileNotFound { path: String }, + + #[error("Dependency cycle detected: {cycle}")] + DependencyCycle { cycle: String }, + + #[error("IO error reading SQL file {path}: {source}")] + IoError { + path: String, + #[source] + source: std::io::Error, + }, + + #[error("Missing dependency: {component} requires {missing}")] + MissingDependency { + component: String, + missing: String, + }, +} + +/// Errors related to database operations +#[derive(Error, Debug)] +pub enum DatabaseError { + #[error("Connection failed: {0}")] + Connection(#[source] tokio_postgres::Error), + + #[error("Transaction failed: {0}")] + Transaction(String), + + #[error("Query failed: {query}: {source}")] + Query { + query: String, + #[source] + source: tokio_postgres::Error, + }, + + #[error("Expected JSONB value to have key '{key}', got: {actual}")] + MissingJsonbKey { + key: String, + actual: serde_json::Value, + }, +} diff --git a/eql-core/src/lib.rs b/eql-core/src/lib.rs new file mode 100644 index 0000000..3f4069c --- /dev/null +++ b/eql-core/src/lib.rs @@ -0,0 +1,57 @@ +//! EQL Core - Trait definitions for multi-database SQL extension API + +pub mod component; +pub mod config; +pub mod error; + +pub use component::{Component, Dependencies}; +pub use config::Config; +pub use error::{ComponentError, DatabaseError, EqlError}; + +// Re-export paste for use in sql_component! macro +#[doc(hidden)] +pub use paste; + +// Note: sql_component! macro is automatically available at crate root via #[macro_export] + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_component_trait_compiles() { + // This test verifies the trait definition compiles + // Actual implementations will be in eql-postgres + struct TestComponent; + + impl Component for TestComponent { + type Dependencies = (); + + fn sql_file() -> &'static str { + "test.sql" + } + } + + assert_eq!(TestComponent::sql_file(), "test.sql"); + } + + #[test] + fn test_error_types_display() { + let err = ComponentError::SqlFileNotFound { + path: "test.sql".to_string(), + }; + assert!(err.to_string().contains("SQL file not found")); + assert!(err.to_string().contains("test.sql")); + } + + #[test] + fn test_database_error_context() { + let err = DatabaseError::MissingJsonbKey { + key: "tables".to_string(), + actual: serde_json::json!({"wrong": "value"}), + }; + let err_string = err.to_string(); + assert!(err_string.contains("tables")); + assert!(err_string.contains("wrong")); + } +} diff --git a/eql-postgres/Cargo.toml b/eql-postgres/Cargo.toml new file mode 100644 index 0000000..1fbb053 --- /dev/null +++ b/eql-postgres/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "eql-postgres" +version = "0.1.0" +edition = "2021" + +[dependencies] +eql-core = { path = "../eql-core" } +serde = { workspace = true } +serde_json = { workspace = true } + +[dev-dependencies] +eql-test = { path = "../eql-test" } +tokio = { workspace = true } diff --git a/eql-postgres/src/config.rs b/eql-postgres/src/config.rs new file mode 100644 index 0000000..240e933 --- /dev/null +++ b/eql-postgres/src/config.rs @@ -0,0 +1,51 @@ +//! PostgreSQL implementation of Config trait + +use eql_core::{sql_component, Config}; + +// ============================================ +// SQL Component Declarations +// ============================================ + +// Configuration components +sql_component!(config::ConfigTypes => "types.sql"); +sql_component!(config::ConfigTables => "tables.sql", deps: [ConfigTypes]); +sql_component!(config::ConfigIndexes => "indexes.sql", deps: [ConfigTables]); +sql_component!(config::ConfigPrivateFunctions => "functions_private.sql", deps: [ConfigTypes]); +sql_component!(config::MigrateActivate, deps: [ConfigIndexes]); + +// Encrypted components +sql_component!(encrypted::CheckEncrypted); +sql_component!(encrypted::AddEncryptedConstraint, deps: [CheckEncrypted]); + +// Main configuration function +sql_component!(config::AddColumn, deps: [ + ConfigPrivateFunctions, + MigrateActivate, + AddEncryptedConstraint, + ConfigTypes, +]); + +// Placeholder components for POC +sql_component!(RemoveColumn => "not_implemented.sql"); +sql_component!(AddSearchConfig => "not_implemented.sql"); + +// PostgreSQL implementation of Config trait +pub struct PostgresEQL; + +impl Config for PostgresEQL { + type AddColumnComponent = AddColumn; + type RemoveColumnComponent = RemoveColumn; + type AddSearchConfigComponent = AddSearchConfig; + + fn add_column() -> &'static Self::AddColumnComponent { + &AddColumn + } + + fn remove_column() -> &'static Self::RemoveColumnComponent { + &RemoveColumn + } + + fn add_search_config() -> &'static Self::AddSearchConfigComponent { + &AddSearchConfig + } +} diff --git a/eql-postgres/src/lib.rs b/eql-postgres/src/lib.rs new file mode 100644 index 0000000..20d0421 --- /dev/null +++ b/eql-postgres/src/lib.rs @@ -0,0 +1,43 @@ +//! PostgreSQL implementation of EQL + +pub mod config; + +pub use config::PostgresEQL; + +#[cfg(test)] +mod tests { + use super::*; + use eql_core::Component; + + #[test] + fn test_component_sql_files_exist() { + use config::AddColumn; + let path = AddColumn::sql_file(); + assert!( + std::path::Path::new(path).exists(), + "add_column SQL file should exist at {}", + path + ); + } + + #[test] + fn test_add_column_dependencies_collected() { + use config::AddColumn; + + let deps = AddColumn::collect_dependencies(); + + // Should include all dependencies in order + assert!(deps.len() > 1, "AddColumn should have dependencies"); + + // Dependencies should come before AddColumn itself + let add_column_path = AddColumn::sql_file(); + let add_column_pos = deps.iter().position(|&f| f == add_column_path); + assert!(add_column_pos.is_some(), "Should include AddColumn itself"); + + // Verify no duplicates + let mut seen = std::collections::HashSet::new(); + for file in &deps { + assert!(seen.insert(file), "Dependency {} appears twice", file); + } + } +} diff --git a/eql-postgres/src/sql/config/add_column.sql b/eql-postgres/src/sql/config/add_column.sql new file mode 100644 index 0000000..8823fa3 --- /dev/null +++ b/eql-postgres/src/sql/config/add_column.sql @@ -0,0 +1,49 @@ +-- Add a column for encryption/decryption +-- +-- This function initializes a column to work with CipherStash encryption. +-- The column must be of type eql_v2_encrypted. +-- +-- Depends on: config/types.sql, config/functions_private.sql, +-- config/migrate_activate.sql, encrypted/add_encrypted_constraint.sql + +CREATE FUNCTION eql_v2.add_column(table_name text, column_name text, cast_as text DEFAULT 'text', migrating boolean DEFAULT false) + RETURNS jsonb +AS $$ + DECLARE + _config jsonb; + BEGIN + -- set the active config + SELECT data INTO _config FROM public.eql_v2_configuration WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + + -- set default config + SELECT eql_v2.config_default(_config) INTO _config; + + -- if index exists + IF _config #> array['tables', table_name] ? column_name THEN + RAISE EXCEPTION 'Config exists for column: % %', table_name, column_name; + END IF; + + SELECT eql_v2.config_add_table(table_name, _config) INTO _config; + + SELECT eql_v2.config_add_column(table_name, column_name, _config) INTO _config; + + SELECT eql_v2.config_add_cast(table_name, column_name, cast_as, _config) INTO _config; + + -- create a new pending record if we don't have one + INSERT INTO public.eql_v2_configuration (state, data) VALUES ('pending', _config) + ON CONFLICT (state) + WHERE state = 'pending' + DO UPDATE + SET data = _config; + + IF NOT migrating THEN + PERFORM eql_v2.migrate_config(); + PERFORM eql_v2.activate_config(); + END IF; + + PERFORM eql_v2.add_encrypted_constraint(table_name, column_name); + + -- exeunt + RETURN _config; + END; +$$ LANGUAGE plpgsql; diff --git a/eql-postgres/src/sql/config/functions_private.sql b/eql-postgres/src/sql/config/functions_private.sql new file mode 100644 index 0000000..3e81074 --- /dev/null +++ b/eql-postgres/src/sql/config/functions_private.sql @@ -0,0 +1,95 @@ +-- REQUIRE: src/config/types.sql +-- +-- Private configuration functions +-- Internal implemention details that customers should not need to worry about. +-- +-- + +CREATE FUNCTION eql_v2.config_default(config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + BEGIN + IF config IS NULL THEN + SELECT jsonb_build_object('v', 1, 'tables', jsonb_build_object()) INTO config; + END IF; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.config_add_table(table_name text, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + DECLARE + tbl jsonb; + BEGIN + IF NOT config #> array['tables'] ? table_name THEN + SELECT jsonb_insert(config, array['tables', table_name], jsonb_build_object()) INTO config; + END IF; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- Add the column if it doesn't exist + +CREATE FUNCTION eql_v2.config_add_column(table_name text, column_name text, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + DECLARE + col jsonb; + BEGIN + IF NOT config #> array['tables', table_name] ? column_name THEN + SELECT jsonb_build_object('indexes', jsonb_build_object()) into col; + SELECT jsonb_set(config, array['tables', table_name, column_name], col) INTO config; + END IF; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- Set the cast + +CREATE FUNCTION eql_v2.config_add_cast(table_name text, column_name text, cast_as text, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + BEGIN + SELECT jsonb_set(config, array['tables', table_name, column_name, 'cast_as'], to_jsonb(cast_as)) INTO config; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- Add the column if it doesn't exist + +CREATE FUNCTION eql_v2.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + BEGIN + SELECT jsonb_insert(config, array['tables', table_name, column_name, 'indexes', index_name], opts) INTO config; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- +-- Default options for match index +-- + +CREATE FUNCTION eql_v2.config_match_default() + RETURNS jsonb +LANGUAGE sql STRICT PARALLEL SAFE +BEGIN ATOMIC + SELECT jsonb_build_object( + 'k', 6, + 'bf', 2048, + 'include_original', true, + 'tokenizer', json_build_object('kind', 'ngram', 'token_length', 3), + 'token_filters', json_build_array(json_build_object('kind', 'downcase'))); +END; diff --git a/eql-postgres/src/sql/config/indexes.sql b/eql-postgres/src/sql/config/indexes.sql new file mode 100644 index 0000000..570a729 --- /dev/null +++ b/eql-postgres/src/sql/config/indexes.sql @@ -0,0 +1,11 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/config/tables.sql + + +-- +-- Define partial indexes to ensure that there is only one active, pending and encrypting config at a time +-- +CREATE UNIQUE INDEX ON public.eql_v2_configuration (state) WHERE state = 'active'; +CREATE UNIQUE INDEX ON public.eql_v2_configuration (state) WHERE state = 'pending'; +CREATE UNIQUE INDEX ON public.eql_v2_configuration (state) WHERE state = 'encrypting'; + diff --git a/eql-postgres/src/sql/config/migrate_activate.sql b/eql-postgres/src/sql/config/migrate_activate.sql new file mode 100644 index 0000000..37547b8 --- /dev/null +++ b/eql-postgres/src/sql/config/migrate_activate.sql @@ -0,0 +1,51 @@ +-- Configuration migration and activation functions +-- +-- Depends on: config/types.sql (for eql_v2_configuration table) + +-- Stub for ready_for_encryption (POC only) +CREATE FUNCTION eql_v2.ready_for_encryption() + RETURNS boolean +AS $$ +BEGIN + -- POC: Always return true + -- Real implementation would validate all configured columns exist + RETURN true; +END; +$$ LANGUAGE plpgsql; + +-- Marks the currently pending configuration as encrypting +CREATE FUNCTION eql_v2.migrate_config() + RETURNS boolean +AS $$ +BEGIN + IF EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'encrypting') THEN + RAISE EXCEPTION 'An encryption is already in progress'; + END IF; + + IF NOT EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'pending') THEN + RAISE EXCEPTION 'No pending configuration exists to encrypt'; + END IF; + + IF NOT eql_v2.ready_for_encryption() THEN + RAISE EXCEPTION 'Some pending columns do not have an encrypted target'; + END IF; + + UPDATE public.eql_v2_configuration SET state = 'encrypting' WHERE state = 'pending'; + RETURN true; +END; +$$ LANGUAGE plpgsql; + +-- Activates the currently encrypting configuration +CREATE FUNCTION eql_v2.activate_config() + RETURNS boolean +AS $$ +BEGIN + IF EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'encrypting') THEN + UPDATE public.eql_v2_configuration SET state = 'inactive' WHERE state = 'active'; + UPDATE public.eql_v2_configuration SET state = 'active' WHERE state = 'encrypting'; + RETURN true; + ELSE + RAISE EXCEPTION 'No encrypting configuration exists to activate'; + END IF; +END; +$$ LANGUAGE plpgsql; diff --git a/eql-postgres/src/sql/config/tables.sql b/eql-postgres/src/sql/config/tables.sql new file mode 100644 index 0000000..8fded8c --- /dev/null +++ b/eql-postgres/src/sql/config/tables.sql @@ -0,0 +1,15 @@ +-- REQUIRE: src/config/types.sql + +-- +-- +-- CREATE the eql_v2_configuration TABLE +-- +CREATE TABLE IF NOT EXISTS public.eql_v2_configuration +( + id bigint GENERATED ALWAYS AS IDENTITY, + state eql_v2_configuration_state NOT NULL DEFAULT 'pending', + data jsonb, + created_at timestamptz not null default current_timestamp, + PRIMARY KEY(id) +); + diff --git a/eql-postgres/src/sql/config/types.sql b/eql-postgres/src/sql/config/types.sql new file mode 100644 index 0000000..a0d5cc4 --- /dev/null +++ b/eql-postgres/src/sql/config/types.sql @@ -0,0 +1,26 @@ +-- +-- cs_configuration_data_v2 is a jsonb column that stores the actual configuration +-- +-- For some reason CREATE DOMAIN and CREATE TYPE do not support IF NOT EXISTS +-- Types cannot be dropped if used by a table, and we never drop the configuration table +-- DOMAIN constraints are added separately and not tied to DOMAIN creation +-- +-- DO $$ +-- BEGIN +-- IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'configuration_data') THEN +-- CREATE DOMAIN eql_v2.configuration_data AS JSONB; +-- END IF; +-- END +-- $$; + +-- +-- cs_configuration_state_v2 is an ENUM that defines the valid configuration states +-- -- +DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'eql_v2_configuration_state') THEN + CREATE TYPE public.eql_v2_configuration_state AS ENUM ('active', 'inactive', 'encrypting', 'pending'); + END IF; + END +$$; + diff --git a/eql-postgres/src/sql/encrypted/add_encrypted_constraint.sql b/eql-postgres/src/sql/encrypted/add_encrypted_constraint.sql new file mode 100644 index 0000000..1c1159a --- /dev/null +++ b/eql-postgres/src/sql/encrypted/add_encrypted_constraint.sql @@ -0,0 +1,16 @@ +-- Add constraint to verify encrypted column structure +-- +-- Depends on: check_encrypted function + +CREATE FUNCTION eql_v2.add_encrypted_constraint(table_name TEXT, column_name TEXT) + RETURNS void +AS $$ +BEGIN + EXECUTE format( + 'ALTER TABLE %I ADD CONSTRAINT eql_v2_encrypted_check_%I CHECK (eql_v2.check_encrypted(%I))', + table_name, + column_name, + column_name + ); +END; +$$ LANGUAGE plpgsql; diff --git a/eql-postgres/src/sql/encrypted/check_encrypted.sql b/eql-postgres/src/sql/encrypted/check_encrypted.sql new file mode 100644 index 0000000..948061e --- /dev/null +++ b/eql-postgres/src/sql/encrypted/check_encrypted.sql @@ -0,0 +1,12 @@ +-- Stub for check_encrypted function (minimal implementation for POC) +-- Full implementation would validate encrypted data structure + +CREATE FUNCTION eql_v2.check_encrypted(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ +BEGIN + -- For POC: Just check that the data field is JSONB + RETURN (val).data IS NOT NULL; +END; +$$ LANGUAGE plpgsql; diff --git a/eql-postgres/tests/config_test.rs b/eql-postgres/tests/config_test.rs new file mode 100644 index 0000000..c89ebad --- /dev/null +++ b/eql-postgres/tests/config_test.rs @@ -0,0 +1,103 @@ +use eql_postgres::config::AddColumn; +use eql_core::Component; +use eql_test::TestDb; + +#[tokio::test] +async fn test_add_column_creates_config() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Create schema + db.execute("CREATE SCHEMA IF NOT EXISTS eql_v2;") + .await.expect("Failed to create schema"); + + // Create minimal encrypted type stub for POC + db.execute( + "CREATE TYPE eql_v2_encrypted AS (data jsonb);" + ).await.expect("Failed to create encrypted type"); + + // Load all dependencies in order + let deps = AddColumn::collect_dependencies(); + for sql_file in deps { + let sql = std::fs::read_to_string(sql_file) + .unwrap_or_else(|e| panic!("Failed to read {}: {}", sql_file, e)); + + db.batch_execute(&sql) + .await + .unwrap_or_else(|e| panic!("Failed to load {}: {}", sql_file, e)); + } + + // Setup: Create test table with encrypted column + db.execute( + "CREATE TABLE users ( + id int, + email eql_v2_encrypted + )" + ).await.expect("Failed to create table"); + + // Execute: Call add_column + let result = db.query_one( + "SELECT eql_v2.add_column('users', 'email', 'text')" + ) + .await + .expect("Failed to call add_column"); + + // Assert: Result has expected structure + db.assert_jsonb_has_key(&result, 0, "tables") + .expect("Expected 'tables' key in config"); + + db.assert_jsonb_has_key(&result, 0, "v") + .expect("Expected 'v' (version) key in config"); + + // Assert: Configuration was stored + let config_row = db.query_one( + "SELECT data FROM public.eql_v2_configuration WHERE state = 'active'" + ) + .await + .expect("Should have active config"); + + db.assert_jsonb_has_key(&config_row, 0, "tables") + .expect("Stored config should have 'tables' key"); + + // Assert: Constraint was added + let constraint_exists = db.query_one( + "SELECT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'eql_v2_encrypted_check_email' + )" + ).await.expect("Failed to check constraint"); + + let exists: bool = constraint_exists.get(0); + assert!(exists, "Encrypted constraint should exist"); +} + +#[tokio::test] +async fn test_add_column_rejects_duplicate() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Load schema and dependencies (same as above) + db.execute("CREATE SCHEMA IF NOT EXISTS eql_v2;") + .await.expect("Failed to create schema"); + + db.execute("CREATE TYPE eql_v2_encrypted AS (data jsonb);") + .await.expect("Failed to create encrypted type"); + + let deps = AddColumn::collect_dependencies(); + for sql_file in deps { + let sql = std::fs::read_to_string(sql_file).unwrap(); + db.batch_execute(&sql).await.unwrap(); + } + + db.execute("CREATE TABLE users (id int, email eql_v2_encrypted)") + .await.expect("Failed to create table"); + + // First call succeeds + db.query_one("SELECT eql_v2.add_column('users', 'email', 'text')") + .await + .expect("First add_column should succeed"); + + // Second call should fail + let result = db.query_one("SELECT eql_v2.add_column('users', 'email', 'text')").await; + assert!(result.is_err(), "Duplicate add_column should fail"); + + // Test passes - duplicate prevention working (error raised by SQL function) +} diff --git a/eql-test/Cargo.toml b/eql-test/Cargo.toml new file mode 100644 index 0000000..f1244b4 --- /dev/null +++ b/eql-test/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "eql-test" +version = "0.1.0" +edition = "2021" + +[dependencies] +eql-core = { path = "../eql-core" } +tokio = { workspace = true } +tokio-postgres = { workspace = true } +serde_json = { workspace = true } diff --git a/eql-test/src/lib.rs b/eql-test/src/lib.rs new file mode 100644 index 0000000..431c843 --- /dev/null +++ b/eql-test/src/lib.rs @@ -0,0 +1,141 @@ +//! Test harness providing transaction isolation for SQL tests + +use eql_core::error::DatabaseError; +use tokio_postgres::{Client, NoTls, Row}; + +pub struct TestDb { + client: Client, + in_transaction: bool, +} + +impl TestDb { + /// Create new test database with transaction isolation + pub async fn new() -> Result { + let (client, connection) = tokio_postgres::connect( + &Self::connection_string(), + NoTls, + ) + .await + .map_err(DatabaseError::Connection)?; + + // Spawn connection handler + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("Connection error: {}", e); + } + }); + + // Begin transaction for isolation + client.execute("BEGIN", &[]) + .await + .map_err(|e| DatabaseError::Query { + query: "BEGIN".to_string(), + source: e, + })?; + + Ok(Self { + client, + in_transaction: true, + }) + } + + fn connection_string() -> String { + std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| "host=localhost port=7432 user=cipherstash password=password dbname=postgres".to_string()) + } + + /// Execute SQL (for setup/implementation loading) + pub async fn execute(&self, sql: &str) -> Result { + self.client.execute(sql, &[]) + .await + .map_err(|e| DatabaseError::Query { + query: sql.to_string(), + source: e, + }) + } + + /// Execute batch of SQL statements (for loading multi-statement SQL files) + pub async fn batch_execute(&self, sql: &str) -> Result<(), DatabaseError> { + self.client.batch_execute(sql) + .await + .map_err(|e| DatabaseError::Query { + query: sql.to_string(), + source: e, + }) + } + + /// Query with single result + pub async fn query_one(&self, sql: &str) -> Result { + self.client.query_one(sql, &[]) + .await + .map_err(|e| DatabaseError::Query { + query: sql.to_string(), + source: e, + }) + } + + /// Assert JSONB result has key + pub fn assert_jsonb_has_key(&self, result: &Row, column_index: usize, key: &str) -> Result<(), DatabaseError> { + let json: serde_json::Value = result.get(column_index); + if json.get(key).is_none() { + return Err(DatabaseError::MissingJsonbKey { + key: key.to_string(), + actual: json, + }); + } + Ok(()) + } +} + +impl Drop for TestDb { + fn drop(&mut self) { + if self.in_transaction { + // Auto-rollback on drop + // Note: Can't use async in Drop, but connection will rollback anyway + // when client drops + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_testdb_transaction_isolation() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + // Create a temporary table + db.execute("CREATE TEMPORARY TABLE test_table (id int, value text)") + .await + .expect("Failed to create table"); + + // Insert data + db.execute("INSERT INTO test_table VALUES (1, 'test')") + .await + .expect("Failed to insert"); + + // Query data + let row = db.query_one("SELECT value FROM test_table WHERE id = 1") + .await + .expect("Failed to query"); + + let value: String = row.get(0); + assert_eq!(value, "test"); + + // Transaction will rollback on drop - table won't exist in next test + } + + #[tokio::test] + async fn test_database_error_includes_query() { + let db = TestDb::new().await.expect("Failed to create TestDb"); + + let result = db.execute("INVALID SQL SYNTAX").await; + assert!(result.is_err()); + + let err = result.unwrap_err(); + let err_string = err.to_string(); + assert!(err_string.contains("Query failed")); + assert!(err_string.contains("INVALID SQL SYNTAX")); + } +}