From af1f22e970290e701b00d7d233b71e7082597910 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Thu, 19 Mar 2026 03:26:41 +0530 Subject: [PATCH 01/42] feat: request time encryption --- Cargo.lock | 289 ++++++++++++----- Cargo.toml | 6 + api/src/v1/config.rs | 61 ++++ common/Cargo.toml | 5 + common/src/encryption.rs | 546 ++++++++++++++++++++++++++++++++ common/src/lib.rs | 1 + common/src/types/config.rs | 31 ++ lite/src/backend/core.rs | 3 + lite/src/backend/error.rs | 8 + lite/src/backend/read.rs | 4 + lite/src/backend/streams.rs | 35 +- lite/src/handlers/v1/error.rs | 11 + lite/src/handlers/v1/records.rs | 199 ++++++++++-- lite/src/handlers/v1/streams.rs | 4 +- lite/src/init.rs | 1 + sdk/src/types.rs | 2 + 16 files changed, 1094 insertions(+), 112 deletions(-) create mode 100644 common/src/encryption.rs diff --git a/Cargo.lock b/Cargo.lock index 2c8a744c..bdc85fc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,51 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aegis" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae1572243695de9c6c8d16c7889899abac907d14c148f1939d837122bbeca79" +dependencies = [ + "cc", + "softaes", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -77,9 +122,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.14" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" @@ -118,9 +163,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arc-swap" -version = "1.9.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" dependencies = [ "rustversion", ] @@ -343,9 +388,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -354,9 +399,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" +checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e" dependencies = [ "cc", "cmake", @@ -391,9 +436,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.97.0" +version = "1.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" +checksum = "f64a6eded248c6b453966e915d32aeddb48ea63ad17932682774eb026fbef5b1" dependencies = [ "aws-credential-types", "aws-runtime", @@ -415,9 +460,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.99.0" +version = "1.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" +checksum = "db96d720d3c622fcbe08bae1c4b04a72ce6257d8b0584cb5418da00ae20a344f" dependencies = [ "aws-credential-types", "aws-runtime", @@ -439,9 +484,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.101.0" +version = "1.100.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a" +checksum = "fafbdda43b93f57f699c5dfe8328db590b967b8a820a13ccdd6687355dfcc7ca" dependencies = [ "aws-credential-types", "aws-runtime", @@ -612,9 +657,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.4.7" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c" +checksum = "d2b1117b3b2bbe166d11199b540ceed0d0f7676e36e7b962b5a437a9971eac75" dependencies = [ "base64-simd", "bytes", @@ -964,9 +1009,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.57" +version = "1.2.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ "find-msvc-tools", "jobserver", @@ -1011,6 +1056,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.6.0" @@ -1047,9 +1102,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" [[package]] name = "cmake" @@ -1089,9 +1144,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.5" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "colored" @@ -1148,9 +1203,9 @@ dependencies = [ [[package]] name = "config" -version = "0.15.22" +version = "0.15.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68cfe19cd7d23ffde002c24ffa5cda73931913ef394d5eaaa32037dc940c0c" +checksum = "b30fa8254caad766fc03cb0ccae691e14bf3bd72bfff27f72802ce729551b3d6" dependencies = [ "async-trait", "convert_case 0.6.0", @@ -1161,19 +1216,20 @@ dependencies = [ "serde-untagged", "serde_core", "serde_json", - "toml 1.0.7+spec-1.1.0", - "winnow 1.0.0", + "toml 0.9.12+spec-1.1.0", + "winnow 0.7.15", "yaml-rust2", ] [[package]] name = "console" -version = "0.16.3" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +checksum = "03e45a4a8926227e4197636ba97a9fc9b00477e9f4bd711395687c5f0734bec4" dependencies = [ "encode_unicode", "libc", + "once_cell", "unicode-width 0.2.2", "windows-sys 0.61.2", ] @@ -1330,6 +1386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1343,6 +1400,15 @@ dependencies = [ "phf", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "darling" version = "0.14.4" @@ -1724,9 +1790,9 @@ dependencies = [ [[package]] name = "euclid" -version = "0.22.14" +version = "0.22.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +checksum = "df61bf483e837f88d5c2291dcf55c67be7e676b3a51acc48db3a7b163b91ed63" dependencies = [ "num-traits", ] @@ -2185,6 +2251,16 @@ dependencies = [ "wasip3", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gimli" version = "0.32.3" @@ -2621,11 +2697,20 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "instability" -version = "0.3.12" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" +checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d" dependencies = [ "darling 0.23.0", "indoc", @@ -2673,9 +2758,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.18" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jobserver" @@ -2720,9 +2805,9 @@ dependencies = [ [[package]] name = "kasuari" -version = "0.4.12" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde5057d6143cc94e861d90f591b9303d6716c6b9602309150bd068853c10899" +checksum = "8fe90c1150662e858c7d5f945089b7517b0a80d8bf7ba4b1b5ffc984e7230a5b" dependencies = [ "hashbrown 0.16.1", "portable-atomic", @@ -2749,9 +2834,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" [[package]] name = "libmimalloc-sys" @@ -3265,9 +3350,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.4" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "once_cell_polyfill" @@ -3275,6 +3360,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -3592,6 +3683,18 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -3668,7 +3771,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.5+spec-1.1.0", + "toml_edit 0.25.4+spec-1.1.0", ] [[package]] @@ -3902,9 +4005,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", "getrandom 0.3.4", @@ -4502,9 +4605,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ "aws-lc-rs", "ring", @@ -4610,7 +4713,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-stream", - "toml 1.0.7+spec-1.1.0", + "toml 1.0.4+spec-1.1.0", "tower-http", "tracing", "tracing-subscriber", @@ -4622,6 +4725,8 @@ dependencies = [ name = "s2-common" version = "0.30.0" dependencies = [ + "aegis", + "aes-gcm", "axum", "blake3", "bytes", @@ -4629,10 +4734,13 @@ dependencies = [ "compact_str", "enum-ordinalize", "enumset", + "hex", "http 1.4.0", "proptest", + "rand 0.10.0", "rkyv", "rstest", + "secrecy", "serde", "serde_json", "strum 0.28.0", @@ -4750,9 +4858,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.29" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ "windows-sys 0.61.2", ] @@ -5152,14 +5260,20 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] +[[package]] +name = "softaes" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fef461faaeb36c340b6c887167a9054a034f6acfc50a014ead26a02b4356b3de" + [[package]] name = "spin" version = "0.9.8" @@ -5436,9 +5550,9 @@ dependencies = [ [[package]] name = "test-context" -version = "0.5.5" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3c85fe78076e2c55ed2dbeffd7a093effc1f906f8f55d17eb07736c2babcc95" +checksum = "7d94db16dc1c321805ce55f286c4023fa58a2c9c742568f95c5cfe2e95d250d7" dependencies = [ "futures", "test-context-macros", @@ -5446,9 +5560,9 @@ dependencies = [ [[package]] name = "test-context-macros" -version = "0.5.5" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0279b838142e2c96bf194427ec2af755fb6a86ff2ae0fa99ac7da287d0066f9" +checksum = "aabcca9d2cad192cfe258cd3562b7584516191a5c9b6a0002a6bb8b75ee7d21d" dependencies = [ "proc-macro2", "quote", @@ -5577,9 +5691,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" dependencies = [ "tinyvec_macros", ] @@ -5715,17 +5829,17 @@ dependencies = [ [[package]] name = "toml" -version = "1.0.7+spec-1.1.0" +version = "1.0.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd28d57d8a6f6e458bc0b8784f8fdcc4b99a437936056fa122cb234f18656a96" +checksum = "c94c3321114413476740df133f0d8862c61d87c8d26f04c6841e033c8c80db47" dependencies = [ "indexmap", "serde_core", "serde_spanned 1.0.4", - "toml_datetime 1.0.1+spec-1.1.0", + "toml_datetime 1.0.0+spec-1.1.0", "toml_parser", "toml_writer", - "winnow 1.0.0", + "winnow 0.7.15", ] [[package]] @@ -5748,9 +5862,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.0.1+spec-1.1.0" +version = "1.0.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b320e741db58cac564e26c607d3cc1fdc4a88fd36c879568c07856ed83ff3e9" +checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e" dependencies = [ "serde_core", ] @@ -5771,23 +5885,23 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.5+spec-1.1.0" +version = "0.25.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca1a40644a28bce036923f6a431df0b34236949d111cc07cb6dca830c9ef2e1" +checksum = "7193cbd0ce53dc966037f54351dbbcf0d5a642c7f0038c382ef9e677ce8c13f2" dependencies = [ "indexmap", - "toml_datetime 1.0.1+spec-1.1.0", + "toml_datetime 1.0.0+spec-1.1.0", "toml_parser", - "winnow 1.0.0", + "winnow 0.7.15", ] [[package]] name = "toml_parser" -version = "1.0.10+spec-1.1.0" +version = "1.0.9+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" +checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" dependencies = [ - "winnow 1.0.0", + "winnow 0.7.15", ] [[package]] @@ -5798,9 +5912,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "toml_writer" -version = "1.0.7+spec-1.1.0" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17aaa1c6e3dc22b1da4b6bba97d066e354c7945cac2f7852d4e4e7ca7a6b56d" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" [[package]] name = "tower" @@ -5900,9 +6014,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.23" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" dependencies = [ "matchers", "nu-ansi-term", @@ -6028,6 +6142,16 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -6750,15 +6874,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winnow" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" -dependencies = [ - "memchr", -] - [[package]] name = "wit-bindgen" version = "0.51.0" @@ -6934,18 +7049,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.47" +version = "0.8.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.47" +version = "0.8.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index b83e5be7..55e34399 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,6 +74,12 @@ uuid = "1.22" xxhash-rust = "0.8" zstd = "0.13" +# Encryption +aegis = "0.9" +aes-gcm = "0.10" +hex = "0.4" +secrecy = "0.10.3" + [patch.crates-io] utoipa = { git = "https://github.com/infiniteregrets/utoipa", rev = "9cd181d40ac0a50551ebe1995a4d73e8685890d6" } diff --git a/api/src/v1/config.rs b/api/src/v1/config.rs index 8a9ec38c..992c89d0 100644 --- a/api/src/v1/config.rs +++ b/api/src/v1/config.rs @@ -269,6 +269,10 @@ pub struct StreamConfig { /// Delete-on-empty configuration. #[serde(default)] pub delete_on_empty: Option, + /// Encryption algorithm. `"aegis-256"` | `"aes-256-gcm"` | absent (plaintext). + /// Immutable after stream creation. + #[serde(skip_serializing_if = "Option::is_none")] + pub encryption: Option, } impl StreamConfig { @@ -278,6 +282,7 @@ impl StreamConfig { retention_policy, timestamping, delete_on_empty, + encryption, } = config; let config = StreamConfig { @@ -285,6 +290,11 @@ impl StreamConfig { retention_policy: retention_policy.map(Into::into), timestamping: TimestampingConfig::to_opt(timestamping), delete_on_empty: DeleteOnEmptyConfig::to_opt(delete_on_empty), + encryption: encryption.and_then(|alg| match alg { + types::config::EncryptionAlgorithm::None => None, + types::config::EncryptionAlgorithm::Aegis256 => Some("aegis-256".to_owned()), + types::config::EncryptionAlgorithm::Aes256Gcm => Some("aes-256-gcm".to_owned()), + }), }; if config == Self::default() { None @@ -301,6 +311,7 @@ impl From for StreamConfig { retention_policy, timestamping, delete_on_empty, + encryption, } = value; Self { @@ -308,6 +319,11 @@ impl From for StreamConfig { retention_policy: Some(retention_policy.into()), timestamping: Some(timestamping.into()), delete_on_empty: Some(delete_on_empty.into()), + encryption: encryption.map(|alg| match alg { + types::config::EncryptionAlgorithm::None => "none".to_owned(), + types::config::EncryptionAlgorithm::Aegis256 => "aegis-256".to_owned(), + types::config::EncryptionAlgorithm::Aes256Gcm => "aes-256-gcm".to_owned(), + }), } } } @@ -321,6 +337,7 @@ impl TryFrom for types::config::OptionalStreamConfig { retention_policy, timestamping, delete_on_empty, + encryption, } = value; let retention_policy = match retention_policy { @@ -328,11 +345,23 @@ impl TryFrom for types::config::OptionalStreamConfig { Some(policy) => Some(policy.try_into()?), }; + let encryption = match encryption.as_deref() { + None => None, + Some("aegis-256") => Some(types::config::EncryptionAlgorithm::Aegis256), + Some("aes-256-gcm") => Some(types::config::EncryptionAlgorithm::Aes256Gcm), + Some(other) => { + return Err(types::ValidationError(format!( + "unknown encryption algorithm: {other:?}" + ))); + } + }; + Ok(Self { storage_class: storage_class.map(Into::into), retention_policy, timestamping: timestamping.map(Into::into).unwrap_or_default(), delete_on_empty: delete_on_empty.map(Into::into).unwrap_or_default(), + encryption, }) } } @@ -412,6 +441,11 @@ pub struct BasinConfig { #[serde(default)] #[cfg_attr(feature = "utoipa", schema(default = false))] pub create_stream_on_read: bool, + /// Allowlist of encryption algorithms permitted for streams in this basin. + /// `"none"` = plaintext allowed; `"aegis-256"` | `"aes-256-gcm"` = algorithm allowed. + /// Empty = all allowed (including plaintext). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub allowed_encryption: Vec, } impl TryFrom for types::config::BasinConfig { @@ -422,8 +456,24 @@ impl TryFrom for types::config::BasinConfig { default_stream_config, create_stream_on_append, create_stream_on_read, + allowed_encryption, } = value; + let mut parsed_allowed_encryption = Vec::with_capacity(allowed_encryption.len()); + for s in &allowed_encryption { + let alg = match s.as_str() { + "none" => types::config::EncryptionAlgorithm::None, + "aegis-256" => types::config::EncryptionAlgorithm::Aegis256, + "aes-256-gcm" => types::config::EncryptionAlgorithm::Aes256Gcm, + other => { + return Err(types::ValidationError(format!( + "unknown encryption algorithm in allowed_encryption: {other:?}" + ))); + } + }; + parsed_allowed_encryption.push(alg); + } + Ok(Self { default_stream_config: match default_stream_config { Some(config) => config.try_into()?, @@ -431,6 +481,7 @@ impl TryFrom for types::config::BasinConfig { }, create_stream_on_append, create_stream_on_read, + allowed_encryption: parsed_allowed_encryption, }) } } @@ -441,12 +492,21 @@ impl From for BasinConfig { default_stream_config, create_stream_on_append, create_stream_on_read, + allowed_encryption, } = value; Self { default_stream_config: StreamConfig::to_opt(default_stream_config), create_stream_on_append, create_stream_on_read, + allowed_encryption: allowed_encryption + .into_iter() + .map(|alg| match alg { + types::config::EncryptionAlgorithm::None => "none".to_owned(), + types::config::EncryptionAlgorithm::Aegis256 => "aegis-256".to_owned(), + types::config::EncryptionAlgorithm::Aes256Gcm => "aes-256-gcm".to_owned(), + }) + .collect(), } } } @@ -569,6 +629,7 @@ mod tests { default_stream_config, create_stream_on_append, create_stream_on_read, + allowed_encryption: vec![], } }, ) diff --git a/common/Cargo.toml b/common/Cargo.toml index 9a5df828..70bbf098 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -15,6 +15,8 @@ rkyv = ["dep:rkyv", "compact_str/rkyv"] utoipa = ["dep:utoipa"] [dependencies] +aegis = { workspace = true } +aes-gcm = { workspace = true } axum = { workspace = true, optional = true } blake3 = { workspace = true } bytes = { workspace = true } @@ -22,8 +24,11 @@ clap = { workspace = true, optional = true, features = ["derive"] } compact_str = { workspace = true, features = ["serde"] } enum-ordinalize = { workspace = true } enumset = { workspace = true } +hex = { workspace = true } http = { workspace = true } +rand = { workspace = true } rkyv = { workspace = true, optional = true } +secrecy = { workspace = true } serde = { workspace = true, features = ["derive"] } strum = { workspace = true, features = ["derive"] } thiserror = { workspace = true } diff --git a/common/src/encryption.rs b/common/src/encryption.rs new file mode 100644 index 00000000..56da07ab --- /dev/null +++ b/common/src/encryption.rs @@ -0,0 +1,546 @@ +//! Server-side record encryption for HIPAA/BAA compliance. +//! +//! The `S2-Encryption` header carries the algorithm and key for each request. +//! The server encrypts on append and decrypts on read; the key never persists +//! beyond the request lifetime. +//! +//! ## Wire format (body of stored EnvelopeRecord) +//! +//! ```text +//! [alg_id: 1 byte] [nonce] [ciphertext] [tag] +//! ``` +//! +//! | alg_id | Algorithm | Nonce | Tag | +//! |--------|-------------|--------|------| +//! | 0x01 | AEGIS-256 | 32 B | 32 B | +//! | 0x02 | AES-256-GCM | 12 B | 16 B | + +use aegis::aegis256::Aegis256; +use aes_gcm::{ + Aes256Gcm, KeyInit, + aead::{Aead, Payload}, +}; +use bytes::{BufMut, Bytes, BytesMut}; +use http::HeaderMap; +use rand::random; +use secrecy::{CloneableSecret, ExposeSecret, SecretBox}; + +use crate::record::{Encodable as _, EnvelopeRecord, Header}; +pub use crate::types::config::EncryptionAlgorithm; + +pub const S2_ENCRYPTION_HEADER: &str = "s2-encryption"; + +const ALG_ID_AEGIS256: u8 = 0x01; +const ALG_ID_AES256GCM: u8 = 0x02; + +const NONCE_BYTES_AEGIS256: usize = 32; +const TAG_BYTES_AEGIS256: usize = 32; + +const NONCE_BYTES_AES256GCM: usize = 12; +const TAG_BYTES_AES256GCM: usize = 16; + +/// Newtype for a 32-byte encryption key that allows cloning and zeroizes on drop. +#[derive(Clone)] +pub struct KeyBytes(pub [u8; 32]); + +impl secrecy::zeroize::Zeroize for KeyBytes { + fn zeroize(&mut self) { + self.0.iter_mut().for_each(|b| *b = 0); + } +} + +impl CloneableSecret for KeyBytes {} + +/// A cloneable, debug-redacted wrapper around a 32-byte key. +pub type EncryptionKey = SecretBox; + +fn make_key(bytes: [u8; 32]) -> EncryptionKey { + SecretBox::new(Box::new(KeyBytes(bytes))) +} + +/// Parsed `S2-Encryption` header directive. +#[derive(Clone, Debug)] +pub enum EncryptionDirective { + /// Client provides the key; server encrypts/decrypts. + Key { + alg: EncryptionAlgorithm, + key: EncryptionKey, + }, + /// Client handles encryption itself; server passes bytes through. + Attest, +} + +#[derive(Debug, Clone, thiserror::Error)] +pub enum EncryptionError { + #[error("Malformed S2-Encryption header: {0}")] + MalformedHeader(String), + #[error("Algorithm mismatch: stream requires {expected:?}, got {got:?}")] + AlgorithmMismatch { + expected: EncryptionAlgorithm, + got: EncryptionAlgorithm, + }, + #[error("Encryption required: stream has encryption={0:?} but no key was provided")] + EncryptionRequired(EncryptionAlgorithm), + #[error("Decryption failed")] + DecryptionFailed, + #[error("Record encoding error: {0}")] + EncodingFailed(String), +} + +/// Parse the `S2-Encryption` header value. +/// +/// Returns: +/// - `Ok(None)` if the header is absent. +/// - `Ok(Some(directive))` on a valid header. +/// - `Err` if the header is present but malformed. +pub fn parse_s2_encryption_header( + headers: &HeaderMap, +) -> Result, EncryptionError> { + let value = match headers.get(S2_ENCRYPTION_HEADER) { + Some(v) => v, + None => return Ok(None), + }; + + let s = value + .to_str() + .map_err(|_| EncryptionError::MalformedHeader("header is not valid UTF-8".to_owned()))?; + + // "attest" → client-side encryption, server passes through. + if s.trim() == "attest" { + return Ok(Some(EncryptionDirective::Attest)); + } + + // "alg=; key=<64 hex chars>" + let (alg_part, key_part) = s.split_once(';').ok_or_else(|| { + EncryptionError::MalformedHeader(format!("expected 'alg=...; key=...', got {s:?}")) + })?; + + let alg_str = alg_part + .trim() + .strip_prefix("alg=") + .ok_or_else(|| { + EncryptionError::MalformedHeader(format!("expected 'alg=...', got {alg_part:?}")) + })? + .trim(); + + let key_hex = key_part + .trim() + .strip_prefix("key=") + .ok_or_else(|| { + EncryptionError::MalformedHeader(format!("expected 'key=...', got {key_part:?}")) + })? + .trim(); + + let alg = match alg_str { + "aegis-256" => EncryptionAlgorithm::Aegis256, + "aes-256-gcm" => EncryptionAlgorithm::Aes256Gcm, + other => { + return Err(EncryptionError::MalformedHeader(format!( + "unknown algorithm {other:?}; expected 'aegis-256' or 'aes-256-gcm'" + ))); + } + }; + + if key_hex.len() != 64 { + return Err(EncryptionError::MalformedHeader(format!( + "key must be 64 hex characters (32 bytes), got {} characters", + key_hex.len() + ))); + } + + let key_bytes: Vec = hex::decode(key_hex) + .map_err(|e| EncryptionError::MalformedHeader(format!("key is not valid hex: {e}")))?; + + let key_array: [u8; 32] = key_bytes + .try_into() + .map_err(|_| EncryptionError::MalformedHeader("key must be exactly 32 bytes".to_owned()))?; + + Ok(Some(EncryptionDirective::Key { + alg, + key: make_key(key_array), + })) +} + +/// Validate the directive against the stream's configured encryption algorithm. +/// +/// Returns: +/// - `Ok(None)` if the stream has no encryption configured (pass-through). +/// - `Ok(Some(directive))` if encryption should proceed. +/// - `Err(EncryptionRequired)` if the stream requires encryption but none was provided. +/// - `Err(AlgorithmMismatch)` if the algorithms differ. +pub fn check_encryption_directive<'a>( + stream_alg: Option, + directive: Option<&'a EncryptionDirective>, +) -> Result, EncryptionError> { + let Some(required_alg) = stream_alg else { + // Stream has no encryption: ignore any directive. + return Ok(None); + }; + + match directive { + None => Err(EncryptionError::EncryptionRequired(required_alg)), + Some(EncryptionDirective::Attest) => Ok(directive), + Some(EncryptionDirective::Key { alg, .. }) => { + if *alg != required_alg { + return Err(EncryptionError::AlgorithmMismatch { + expected: required_alg, + got: *alg, + }); + } + Ok(directive) + } + } +} + +/// Encode headers + body as `EnvelopeRecord` bytes (the plaintext input to encryption). +pub fn encode_record_plaintext( + headers: Vec
, + body: Bytes, +) -> Result { + EnvelopeRecord::try_from_parts(headers, body) + .map(|r| r.to_bytes()) + .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) +} + +/// Decode `EnvelopeRecord` bytes back to `(headers, body)` after decryption. +pub fn decode_record_plaintext(bytes: Bytes) -> Result<(Vec
, Bytes), EncryptionError> { + EnvelopeRecord::try_from(bytes) + .map(|r| r.into_parts()) + .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) +} + +/// Encrypt a record. +/// +/// Output layout: `[alg_id][random_nonce][ciphertext][tag]` as contiguous `Bytes`. +pub fn encrypt_record( + plaintext: &[u8], + alg: EncryptionAlgorithm, + key: &EncryptionKey, + aad: &[u8], +) -> Result { + match alg { + EncryptionAlgorithm::Aegis256 => { + let nonce: [u8; NONCE_BYTES_AEGIS256] = random(); + let (ciphertext, tag) = + Aegis256::::new(&key.expose_secret().0, &nonce) + .encrypt(plaintext, aad); + + let mut out = BytesMut::with_capacity( + 1 + NONCE_BYTES_AEGIS256 + ciphertext.len() + TAG_BYTES_AEGIS256, + ); + out.put_u8(ALG_ID_AEGIS256); + out.put_slice(&nonce); + out.put_slice(&ciphertext); + out.put_slice(&tag); + Ok(out.freeze()) + } + EncryptionAlgorithm::Aes256Gcm => { + let nonce: [u8; NONCE_BYTES_AES256GCM] = random(); + let cipher = Aes256Gcm::new_from_slice(&key.expose_secret().0).map_err(|_| { + EncryptionError::EncodingFailed("invalid AES key length".to_owned()) + })?; + let nonce_generic = aes_gcm::Nonce::from_slice(&nonce); + // aes-gcm appends the 16-byte tag to the ciphertext automatically. + let ciphertext_with_tag = cipher + .encrypt( + nonce_generic, + Payload { + msg: plaintext, + aad, + }, + ) + .map_err(|_| EncryptionError::DecryptionFailed)?; + + let mut out = + BytesMut::with_capacity(1 + NONCE_BYTES_AES256GCM + ciphertext_with_tag.len()); + out.put_u8(ALG_ID_AES256GCM); + out.put_slice(&nonce); + out.put_slice(&ciphertext_with_tag); + Ok(out.freeze()) + } + EncryptionAlgorithm::None => Err(EncryptionError::EncodingFailed( + "cannot encrypt with None algorithm".to_owned(), + )), + } +} + +/// Decrypt a record body. +/// +/// Returns: +/// - `Ok(Some(plaintext))` on success. +/// - `Ok(None)` if the first byte is not a known `alg_id` (unencrypted pass-through). +/// - `Err` on auth tag failure, truncation, or other error. +pub fn decrypt_record( + body: &[u8], + key: &EncryptionKey, + aad: &[u8], +) -> Result, EncryptionError> { + let alg_id = match body.first() { + Some(&b) => b, + None => return Ok(None), + }; + + match alg_id { + ALG_ID_AEGIS256 => { + // Layout after alg_id: [nonce:32][ciphertext:n][tag:32] + let rest = &body[1..]; + if rest.len() < NONCE_BYTES_AEGIS256 + TAG_BYTES_AEGIS256 { + return Err(EncryptionError::DecryptionFailed); + } + let nonce: &[u8; NONCE_BYTES_AEGIS256] = + rest[..NONCE_BYTES_AEGIS256].try_into().unwrap(); + let after_nonce = &rest[NONCE_BYTES_AEGIS256..]; + if after_nonce.len() < TAG_BYTES_AEGIS256 { + return Err(EncryptionError::DecryptionFailed); + } + let tag_offset = after_nonce.len() - TAG_BYTES_AEGIS256; + let ciphertext = &after_nonce[..tag_offset]; + let tag: &[u8; TAG_BYTES_AEGIS256] = after_nonce[tag_offset..].try_into().unwrap(); + + let plaintext = Aegis256::::new(&key.expose_secret().0, nonce) + .decrypt(ciphertext, tag, aad) + .map_err(|_| EncryptionError::DecryptionFailed)?; + Ok(Some(Bytes::from(plaintext))) + } + ALG_ID_AES256GCM => { + // Layout after alg_id: [nonce:12][ciphertext+tag:n+16] + let rest = &body[1..]; + if rest.len() < NONCE_BYTES_AES256GCM + TAG_BYTES_AES256GCM { + return Err(EncryptionError::DecryptionFailed); + } + let (nonce_bytes, ciphertext_with_tag) = rest.split_at(NONCE_BYTES_AES256GCM); + let cipher = Aes256Gcm::new_from_slice(&key.expose_secret().0).map_err(|_| { + EncryptionError::EncodingFailed("invalid AES key length".to_owned()) + })?; + let nonce_generic = aes_gcm::Nonce::from_slice(nonce_bytes); + let plaintext = cipher + .decrypt( + nonce_generic, + Payload { + msg: ciphertext_with_tag, + aad, + }, + ) + .map_err(|_| EncryptionError::DecryptionFailed)?; + Ok(Some(Bytes::from(plaintext))) + } + // Unknown first byte: not encrypted by S2, pass through. + _ => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_key_fn() -> EncryptionKey { + make_key([0x42u8; 32]) + } + + fn make_wrong_key_fn() -> EncryptionKey { + make_key([0x99u8; 32]) + } + + const AAD: &[u8] = b"test-basin/test-stream"; + + fn roundtrip(alg: EncryptionAlgorithm) { + let headers = vec![Header { + name: Bytes::from_static(b"x-test"), + value: Bytes::from_static(b"hello"), + }]; + let body = Bytes::from_static(b"secret payload"); + + let plaintext = encode_record_plaintext(headers.clone(), body.clone()).unwrap(); + let key = make_key_fn(); + let ciphertext = encrypt_record(&plaintext, alg, &key, AAD).unwrap(); + let decrypted = decrypt_record(&ciphertext, &key, AAD).unwrap().unwrap(); + let (out_headers, out_body) = decode_record_plaintext(decrypted).unwrap(); + + assert_eq!(out_headers, headers); + assert_eq!(out_body, body); + } + + #[test] + fn roundtrip_aegis256() { + roundtrip(EncryptionAlgorithm::Aegis256); + } + + #[test] + fn roundtrip_aes256gcm() { + roundtrip(EncryptionAlgorithm::Aes256Gcm); + } + + #[test] + fn wrong_key_fails_aegis256() { + let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); + let key = make_key_fn(); + let ciphertext = + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD).unwrap(); + let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), AAD); + assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); + } + + #[test] + fn wrong_key_fails_aes256gcm() { + let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); + let key = make_key_fn(); + let ciphertext = + encrypt_record(&plaintext, EncryptionAlgorithm::Aes256Gcm, &key, AAD).unwrap(); + let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), AAD); + assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); + } + + #[test] + fn truncated_ciphertext_fails_no_panic() { + let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); + let key = make_key_fn(); + let ciphertext = + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD).unwrap(); + // Truncate to 3 bytes — too short to contain nonce+tag. + let truncated = &ciphertext[..3]; + let result = decrypt_record(truncated, &key, AAD); + assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); + } + + #[test] + fn unknown_first_byte_passthrough() { + // First byte 0x00 is not a known alg_id. + let body = b"\x00some opaque bytes"; + let key = make_key_fn(); + let result = decrypt_record(body, &key, AAD).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn empty_body_passthrough() { + let key = make_key_fn(); + let result = decrypt_record(b"", &key, AAD).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn parse_header_valid_aegis() { + let mut headers = HeaderMap::new(); + headers.insert( + S2_ENCRYPTION_HEADER, + http::HeaderValue::from_static( + "alg=aegis-256; key=0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", + ), + ); + let directive = parse_s2_encryption_header(&headers).unwrap().unwrap(); + assert!(matches!( + directive, + EncryptionDirective::Key { + alg: EncryptionAlgorithm::Aegis256, + .. + } + )); + } + + #[test] + fn parse_header_valid_aes() { + let mut headers = HeaderMap::new(); + headers.insert( + S2_ENCRYPTION_HEADER, + http::HeaderValue::from_static( + "alg=aes-256-gcm; key=0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", + ), + ); + let directive = parse_s2_encryption_header(&headers).unwrap().unwrap(); + assert!(matches!( + directive, + EncryptionDirective::Key { + alg: EncryptionAlgorithm::Aes256Gcm, + .. + } + )); + } + + #[test] + fn parse_header_attest() { + let mut headers = HeaderMap::new(); + headers.insert( + S2_ENCRYPTION_HEADER, + http::HeaderValue::from_static("attest"), + ); + let directive = parse_s2_encryption_header(&headers).unwrap().unwrap(); + assert!(matches!(directive, EncryptionDirective::Attest)); + } + + #[test] + fn parse_header_absent() { + let headers = HeaderMap::new(); + let result = parse_s2_encryption_header(&headers).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn parse_header_malformed_no_semicolon() { + let mut headers = HeaderMap::new(); + headers.insert( + S2_ENCRYPTION_HEADER, + http::HeaderValue::from_static("alg=aegis-256"), + ); + let result = parse_s2_encryption_header(&headers); + assert!(matches!(result, Err(EncryptionError::MalformedHeader(_)))); + } + + #[test] + fn parse_header_wrong_key_length() { + let mut headers = HeaderMap::new(); + headers.insert( + S2_ENCRYPTION_HEADER, + http::HeaderValue::from_static("alg=aegis-256; key=deadbeef"), + ); + let result = parse_s2_encryption_header(&headers); + assert!(matches!(result, Err(EncryptionError::MalformedHeader(_)))); + } + + #[test] + fn parse_header_invalid_hex() { + let mut headers = HeaderMap::new(); + headers.insert( + S2_ENCRYPTION_HEADER, + http::HeaderValue::from_static( + "alg=aegis-256; key=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + ), + ); + let result = parse_s2_encryption_header(&headers); + assert!(matches!(result, Err(EncryptionError::MalformedHeader(_)))); + } + + #[test] + fn check_directive_alg_mismatch() { + let key = make_key_fn(); + let directive = EncryptionDirective::Key { + alg: EncryptionAlgorithm::Aes256Gcm, + key, + }; + let result = + check_encryption_directive(Some(EncryptionAlgorithm::Aegis256), Some(&directive)); + assert!(matches!( + result, + Err(EncryptionError::AlgorithmMismatch { .. }) + )); + } + + #[test] + fn check_directive_required_but_absent() { + let result = check_encryption_directive(Some(EncryptionAlgorithm::Aegis256), None); + assert!(matches!( + result, + Err(EncryptionError::EncryptionRequired(_)) + )); + } + + #[test] + fn check_directive_no_stream_encryption() { + let key = make_key_fn(); + let directive = EncryptionDirective::Key { + alg: EncryptionAlgorithm::Aegis256, + key, + }; + let result = check_encryption_directive(None, Some(&directive)); + assert!(result.unwrap().is_none()); + } +} diff --git a/common/src/lib.rs b/common/src/lib.rs index 6685a7f4..2451ed4b 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -3,6 +3,7 @@ pub mod bash; pub mod caps; pub mod deep_size; +pub mod encryption; pub mod http; pub mod maybe; pub mod read_extent; diff --git a/common/src/types/config.rs b/common/src/types/config.rs index bd60be11..69eb0ac0 100644 --- a/common/src/types/config.rs +++ b/common/src/types/config.rs @@ -33,6 +33,19 @@ use enum_ordinalize::Ordinalize; use crate::maybe::Maybe; +/// Encryption algorithm for stream records. +/// +/// When used in `StreamConfig.encryption`, `None` is represented by Rust's `Option::None` +/// (meaning plaintext). When used in `BasinConfig.allowed_encryption`, the `None` variant +/// is a sentinel that explicitly permits plaintext streams. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncryptionAlgorithm { + /// Sentinel: plaintext is explicitly allowed (used only in `allowed_encryption` lists). + None, + Aegis256, + Aes256Gcm, +} + #[derive( Debug, Default, @@ -106,6 +119,8 @@ pub struct StreamConfig { pub retention_policy: RetentionPolicy, pub timestamping: TimestampingConfig, pub delete_on_empty: DeleteOnEmptyConfig, + /// Encryption algorithm for this stream. `None` = plaintext. Immutable after creation. + pub encryption: Option, } #[derive(Debug, Clone, Default)] @@ -230,6 +245,8 @@ pub struct OptionalStreamConfig { pub retention_policy: Option, pub timestamping: OptionalTimestampingConfig, pub delete_on_empty: OptionalDeleteOnEmptyConfig, + /// Encryption algorithm. `None` = not specified (falls back to basin default or plaintext). + pub encryption: Option, } impl OptionalStreamConfig { @@ -274,11 +291,14 @@ impl OptionalStreamConfig { let delete_on_empty = self.delete_on_empty.merge(basin_defaults.delete_on_empty); + let encryption = self.encryption.or(basin_defaults.encryption); + StreamConfig { storage_class, retention_policy, timestamping, delete_on_empty, + encryption, } } } @@ -290,6 +310,7 @@ impl From for StreamReconfiguration { retention_policy, timestamping, delete_on_empty, + encryption: _, // encryption is immutable; not represented in StreamReconfiguration } = value; Self { @@ -308,6 +329,7 @@ impl From for StreamConfig { retention_policy, timestamping, delete_on_empty, + encryption, } = value; Self { @@ -315,6 +337,7 @@ impl From for StreamConfig { retention_policy: retention_policy.unwrap_or_default(), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), + encryption, } } } @@ -326,6 +349,7 @@ impl From for OptionalStreamConfig { retention_policy, timestamping, delete_on_empty, + encryption, } = value; Self { @@ -333,6 +357,7 @@ impl From for OptionalStreamConfig { retention_policy: Some(retention_policy), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), + encryption, } } } @@ -342,6 +367,10 @@ pub struct BasinConfig { pub default_stream_config: OptionalStreamConfig, pub create_stream_on_append: bool, pub create_stream_on_read: bool, + /// Allowlist of encryption algorithms for streams in this basin. + /// Empty = all allowed (including plaintext). Use `EncryptionAlgorithm::None` to + /// explicitly permit plaintext when the list is non-empty. + pub allowed_encryption: Vec, } impl BasinConfig { @@ -366,6 +395,7 @@ impl BasinConfig { self.create_stream_on_read = create_stream_on_read; } + // allowed_encryption is not reconfigurable via BasinReconfiguration (S2-ops managed). self } } @@ -376,6 +406,7 @@ impl From for BasinReconfiguration { default_stream_config, create_stream_on_append, create_stream_on_read, + allowed_encryption: _, // not reconfigurable via BasinReconfiguration } = value; Self { diff --git a/lite/src/backend/core.rs b/lite/src/backend/core.rs index 16286445..feedcaa1 100644 --- a/lite/src/backend/core.rs +++ b/lite/src/backend/core.rs @@ -301,6 +301,7 @@ impl Backend { basin.clone(), stream.clone(), OptionalStreamConfig::default(), + None, CreateMode::CreateOnly(None), ) .await @@ -311,6 +312,7 @@ impl Backend { CreateStreamError::BasinDeletionPending(e) => Err(e)?, CreateStreamError::StreamDeletionPending(e) => Err(e)?, CreateStreamError::BasinNotFound(e) => Err(e)?, + CreateStreamError::InvalidConfig(_) => {} CreateStreamError::StreamAlreadyExists(_) => {} } } @@ -450,6 +452,7 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), + None, CreateMode::CreateOnly(None), ) .await diff --git a/lite/src/backend/error.rs b/lite/src/backend/error.rs index 5633d659..07578c7c 100644 --- a/lite/src/backend/error.rs +++ b/lite/src/backend/error.rs @@ -154,6 +154,8 @@ pub enum AppendError { ConditionFailed(#[from] AppendConditionFailedError), #[error(transparent)] TimestampMissing(#[from] AppendTimestampRequiredError), + #[error(transparent)] + Encryption(#[from] s2_common::encryption::EncryptionError), } impl From for AppendError { @@ -225,6 +227,8 @@ pub enum ReadError { StreamDeletionPending(#[from] StreamDeletionPendingError), #[error(transparent)] Unwritten(#[from] UnwrittenError), + #[error(transparent)] + Encryption(#[from] s2_common::encryption::EncryptionError), } impl From for ReadError { @@ -281,6 +285,8 @@ pub enum CreateStreamError { StreamAlreadyExists(#[from] StreamAlreadyExistsError), #[error(transparent)] StreamDeletionPending(#[from] StreamDeletionPendingError), + #[error("{0}")] + InvalidConfig(String), } impl From for CreateStreamError { @@ -420,6 +426,8 @@ pub enum ReconfigureStreamError { StreamNotFound(#[from] StreamNotFoundError), #[error(transparent)] StreamDeletionPending(#[from] StreamDeletionPendingError), + #[error("immutable field: {0}")] + ImmutableField(&'static str), } impl From for ReconfigureStreamError { diff --git a/lite/src/backend/read.rs b/lite/src/backend/read.rs index 5578a9f8..425ed71a 100644 --- a/lite/src/backend/read.rs +++ b/lite/src/backend/read.rs @@ -468,6 +468,7 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), + None, CreateMode::CreateOnly(None), ) .await @@ -545,6 +546,7 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), + None, CreateMode::CreateOnly(None), ) .await @@ -600,6 +602,7 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), + None, CreateMode::CreateOnly(None), ) .await @@ -698,6 +701,7 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), + None, CreateMode::CreateOnly(None), ) .await diff --git a/lite/src/backend/streams.rs b/lite/src/backend/streams.rs index dc10007c..156e10ee 100644 --- a/lite/src/backend/streams.rs +++ b/lite/src/backend/streams.rs @@ -3,7 +3,7 @@ use s2_common::{ record::StreamPosition, types::{ basin::BasinName, - config::{OptionalStreamConfig, StreamReconfiguration}, + config::{EncryptionAlgorithm, OptionalStreamConfig, StreamReconfiguration}, resources::{CreateMode, ListItemsRequestParts, Page, RequestToken}, stream::{ListStreamsRequest, StreamInfo, StreamName}, }, @@ -82,6 +82,7 @@ impl Backend { basin: BasinName, stream: StreamName, config: impl Into, + encryption: Option, mode: CreateMode, ) -> Result, CreateStreamError> { let config = config.into(); @@ -150,15 +151,33 @@ impl Backend { let is_reconfigure = existing_meta_opt.is_some(); let (resolved, created_at) = match existing_meta_opt { Some(existing) => (existing.config.reconfigure(config), existing.created_at), - None => ( - OptionalStreamConfig::default().reconfigure(config), - OffsetDateTime::now_utc(), - ), + None => { + let mut cfg = OptionalStreamConfig::default().reconfigure(config); + cfg.encryption = encryption; + (cfg, OffsetDateTime::now_utc()) + } }; let resolved: OptionalStreamConfig = resolved .merge(basin_meta.config.default_stream_config) .into(); + // Enforce basin's allowed_encryption list on creation. + if !is_reconfigure && !basin_meta.config.allowed_encryption.is_empty() { + let is_allowed = match resolved.encryption { + None => basin_meta + .config + .allowed_encryption + .contains(&EncryptionAlgorithm::None), + Some(alg) => basin_meta.config.allowed_encryption.contains(&alg), + }; + if !is_allowed { + return Err(CreateStreamError::InvalidConfig(format!( + "encryption algorithm {:?} not permitted for this basin", + resolved.encryption + ))); + } + } + let meta = kv::stream_meta::StreamMeta { config: resolved.clone(), created_at, @@ -289,8 +308,14 @@ impl Backend { .min_age .filter(|age| !age.is_zero()); + let original_encryption = meta.config.encryption; meta.config = meta.config.reconfigure(reconfig); + // Encryption is immutable after creation. + if meta.config.encryption != original_encryption { + return Err(ReconfigureStreamError::ImmutableField("encryption")); + } + txn.put(&meta_key, kv::stream_meta::ser_value(&meta))?; let stream_id = StreamId::new(&basin, &stream); diff --git a/lite/src/handlers/v1/error.rs b/lite/src/handlers/v1/error.rs index d983d546..4ca98760 100644 --- a/lite/src/handlers/v1/error.rs +++ b/lite/src/handlers/v1/error.rs @@ -16,6 +16,7 @@ use crate::backend::error::{ AppendConditionFailedError, AppendError, CheckTailError, CreateBasinError, CreateStreamError, DeleteBasinError, DeleteStreamError, GetBasinConfigError, GetStreamConfigError, ListBasinsError, ListStreamsError, ReadError, ReconfigureBasinError, ReconfigureStreamError, + ReconfigureStreamError::ImmutableField, }; #[derive(Debug, thiserror::Error)] @@ -64,6 +65,12 @@ pub enum ServiceError { NotImplemented, } +impl From for ServiceError { + fn from(never: std::convert::Infallible) -> Self { + match never {} + } +} + impl From for ServiceError { fn from(value: AppendRequestRejection) -> Self { match value { @@ -148,6 +155,7 @@ impl ServiceError { CreateStreamError::StreamDeletionPending(e) => { standard(ErrorCode::StreamDeletionPending, e.to_string()) } + CreateStreamError::InvalidConfig(e) => standard(ErrorCode::Invalid, e.to_string()), }, ServiceError::GetStreamConfig(e) => match e { GetStreamConfigError::Storage(e) => standard(ErrorCode::Storage, e.to_string()), @@ -183,6 +191,7 @@ impl ServiceError { ReconfigureStreamError::StreamDeletionPending(e) => { standard(ErrorCode::StreamDeletionPending, e.to_string()) } + ImmutableField(_) => standard(ErrorCode::Invalid, e.to_string()), }, ServiceError::CheckTail(e) => match e { CheckTailError::Storage(e) => standard(ErrorCode::Storage, e.to_string()), @@ -237,6 +246,7 @@ impl ServiceError { } => v1t::stream::AppendConditionFailed::SeqNumMismatch(*assigned_seq_num), }), AppendError::TimestampMissing(e) => standard(ErrorCode::Invalid, e.to_string()), + AppendError::Encryption(e) => standard(ErrorCode::Invalid, e.to_string()), }, ServiceError::Read(e) => match e { ReadError::Storage(e) => standard(ErrorCode::Storage, e.to_string()), @@ -257,6 +267,7 @@ impl ServiceError { ReadError::Unwritten(tail) => ErrorResponse::Unwritten(v1t::stream::TailResponse { tail: tail.0.into(), }), + ReadError::Encryption(e) => standard(ErrorCode::Invalid, e.to_string()), }, ServiceError::NotImplemented => { standard(ErrorCode::PermissionDenied, "Not implemented".to_string()) diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index e9d1c258..b6a81e20 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -14,13 +14,21 @@ use s2_api::{ }; use s2_common::{ caps::RECORD_BATCH_MAX, + encryption::{ + EncryptionDirective, EncryptionError, check_encryption_directive, decode_record_plaintext, + decrypt_record, encode_record_plaintext, encrypt_record, parse_s2_encryption_header, + }, http::extract::Header, read_extent::{CountOrBytes, ReadLimit}, - record::{Metered, MeteredSize as _}, + record::{Metered, MeteredSize as _, Record, SequencedRecord}, types::{ ValidationError, basin::BasinName, - stream::{ReadBatch, ReadEnd, ReadFrom, ReadSessionOutput, ReadStart, StreamName}, + config::EncryptionAlgorithm, + stream::{ + AppendInput, AppendRecordBatch, AppendRecordParts, ReadBatch, ReadEnd, ReadFrom, + ReadSessionOutput, ReadStart, StreamName, + }, }, }; @@ -127,6 +135,7 @@ pub async fn check_tail( #[derive(FromRequest)] #[from_request(rejection(ServiceError))] pub struct ReadArgs { + headers: http::HeaderMap, #[from_request(via(Header))] basin: BasinName, #[from_request(via(Path))] @@ -172,6 +181,7 @@ pub struct ReadArgs { pub async fn read( State(backend): State, ReadArgs { + headers, basin, stream, start, @@ -179,6 +189,21 @@ pub async fn read( request, }: ReadArgs, ) -> Result { + let directive = parse_s2_encryption_header(&headers) + .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; + + let config = backend + .get_stream_config(basin.clone(), stream.clone()) + .await?; + + // On reads, EncryptionRequired is OK — return opaque bytes without decryption. + let decrypt_directive = match check_encryption_directive(config.encryption, directive.as_ref()) + { + Ok(checked) => checked.cloned(), + Err(EncryptionError::EncryptionRequired(_)) => None, + Err(e) => return Err(ServiceError::Validation(ValidationError(e.to_string()))), + }; + let start: ReadStart = start.try_into()?; match request { v1t::stream::ReadRequest::Unary { @@ -186,8 +211,12 @@ pub async fn read( response_mime, } => { let (start, end) = prepare_read(start, end, ReadMode::Unary)?; - let session = backend.read(basin, stream, start, end).await?; + let session = backend + .read(basin.clone(), stream.clone(), start, end) + .await?; let batch = merge_read_session(session, end.wait).await?; + let batch = decrypt_read_batch(batch, decrypt_directive.as_ref(), &basin, &stream) + .map_err(ReadError::Encryption)?; match response_mime { JsonOrProto::Json => Ok(Json(v1t::stream::json::serialize_read_batch( format, &batch, @@ -205,7 +234,9 @@ pub async fn read( } => { let (start, end) = apply_last_event_id(start, end, last_event_id); let (start, end) = prepare_read(start, end, ReadMode::Streaming)?; - let session = backend.read(basin, stream, start, end).await?; + let session = backend + .read(basin.clone(), stream.clone(), start, end) + .await?; let events = async_stream::stream! { let mut processed = CountOrBytes::ZERO; tokio::pin!(session); @@ -216,6 +247,15 @@ pub async fn read( yield v1t::stream::sse::ping_event(); }, Ok(ReadSessionOutput::Batch(batch)) => { + let batch = match decrypt_read_batch(batch, decrypt_directive.as_ref(), &basin, &stream) { + Ok(batch) => batch, + Err(err) => { + let (_, body) = ServiceError::from(ReadError::Encryption(err)).to_response().to_parts(); + yield v1t::stream::sse::error_event(body); + errored = true; + break; + } + }; let Some(last_record) = batch.records.last() else { continue; }; @@ -246,19 +286,22 @@ pub async fn read( response_compression, } => { let (start, end) = prepare_read(start, end, ReadMode::Streaming)?; - let s2s_stream = - backend - .read(basin, stream, start, end) - .await? - .map_ok(|msg| match msg { - ReadSessionOutput::Heartbeat(tail) => v1t::stream::proto::ReadBatch { - records: vec![], - tail: Some(tail.into()), - }, - ReadSessionOutput::Batch(batch) => { - v1t::stream::proto::ReadBatch::from(batch) - } - }); + let s2s_stream = backend + .read(basin.clone(), stream.clone(), start, end) + .await? + .map(move |msg| match msg { + Ok(ReadSessionOutput::Heartbeat(tail)) => Ok(v1t::stream::proto::ReadBatch { + records: vec![], + tail: Some(tail.into()), + }), + Ok(ReadSessionOutput::Batch(batch)) => { + let batch = + decrypt_read_batch(batch, decrypt_directive.as_ref(), &basin, &stream) + .map_err(ReadError::Encryption)?; + Ok(v1t::stream::proto::ReadBatch::from(batch)) + } + Err(e) => Err(e), + }); let response_stream = s2s::FramedMessageStream::<_>::new( response_compression, Box::pin(s2s_stream.map_err(ServiceError::from)), @@ -313,9 +356,48 @@ async fn merge_read_session( Ok(acc) } +fn decrypt_read_batch( + batch: ReadBatch, + directive: Option<&EncryptionDirective>, + basin: &BasinName, + stream: &StreamName, +) -> Result { + let Some(EncryptionDirective::Key { key, .. }) = directive else { + return Ok(batch); + }; + let aad = format!("{basin}/{stream}"); + let records: Vec = batch + .records + .into_inner() + .into_iter() + .map(|sr| { + let Record::Envelope(ref env) = sr.record else { + return Ok(sr); + }; + match decrypt_record(env.body(), key, aad.as_bytes())? { + None => Ok(sr), + Some(plaintext) => { + let (headers, body) = decode_record_plaintext(plaintext)?; + let record = Record::try_from_parts(headers, body) + .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; + Ok(SequencedRecord { + position: sr.position, + record, + }) + } + } + }) + .collect::>()?; + Ok(ReadBatch { + records: Metered::from(records), + tail: batch.tail, + }) +} + #[derive(FromRequest)] #[from_request(rejection(ServiceError))] pub struct AppendArgs { + headers: http::HeaderMap, #[from_request(via(Header))] basin: BasinName, #[from_request(via(Path))] @@ -350,16 +432,35 @@ pub struct AppendArgs { pub async fn append( State(backend): State, AppendArgs { + headers, basin, stream, request, }: AppendArgs, ) -> Result { + let directive = parse_s2_encryption_header(&headers) + .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; + + let config = backend + .get_stream_config(basin.clone(), stream.clone()) + .await?; + + check_encryption_directive(config.encryption, directive.as_ref()) + .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; + match request { v1t::stream::AppendRequest::Unary { input, response_mime, } => { + let input = encrypt_append_input( + input, + config.encryption, + directive.as_ref(), + &basin, + &stream, + ) + .map_err(crate::backend::error::AppendError::Encryption)?; let ack = backend.append(basin, stream, input).await?; match response_mime { JsonOrProto::Json => { @@ -378,15 +479,28 @@ pub async fn append( } => { let (err_tx, err_rx) = tokio::sync::oneshot::channel(); + let stream_alg = config.encryption; + let enc_basin = basin.clone(); + let enc_stream = stream.clone(); let inputs = async_stream::stream! { tokio::pin!(inputs); let mut err_tx = Some(err_tx); while let Some(input) = inputs.next().await { match input { - Ok(input) => yield input, + Ok(input) => { + match encrypt_append_input(input, stream_alg, directive.as_ref(), &enc_basin, &enc_stream) { + Ok(encrypted) => yield encrypted, + Err(e) => { + if let Some(tx) = err_tx.take() { + let _ = tx.send(ServiceError::Append(e.into())); + } + break; + } + } + } Err(e) => { if let Some(tx) = err_tx.take() { - let _ = tx.send(e); + let _ = tx.send(e.into()); } break; } @@ -422,3 +536,50 @@ pub async fn append( } } } + +fn encrypt_append_input( + input: AppendInput, + stream_alg: Option, + directive: Option<&EncryptionDirective>, + basin: &BasinName, + stream: &StreamName, +) -> Result { + let Some(EncryptionDirective::Key { alg, key }) = directive else { + return Ok(input); + }; + if stream_alg.is_none() { + return Ok(input); + } + let aad = format!("{basin}/{stream}"); + let mut encrypted_records = Vec::with_capacity(input.records.len()); + for record in input.records.into_iter() { + let AppendRecordParts { timestamp, record } = record.into(); + let encrypted = match &*record { + Record::Envelope(env) => { + let plaintext = + encode_record_plaintext(env.headers().to_vec(), env.body().clone())?; + let enc_body = encrypt_record(&plaintext, *alg, key, aad.as_bytes())?; + let enc_record = Record::try_from_parts(vec![], enc_body) + .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; + Metered::from(enc_record) + } + Record::Command(_) => record, + }; + encrypted_records.push( + AppendRecordParts { + timestamp, + record: encrypted, + } + .try_into() + .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_string()))?, + ); + } + let records: AppendRecordBatch = encrypted_records + .try_into() + .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_string()))?; + Ok(AppendInput { + records, + match_seq_num: input.match_seq_num, + fencing_token: input.fencing_token, + }) +} diff --git a/lite/src/handlers/v1/streams.rs b/lite/src/handlers/v1/streams.rs index 13bee62c..561cbd94 100644 --- a/lite/src/handlers/v1/streams.rs +++ b/lite/src/handlers/v1/streams.rs @@ -121,11 +121,13 @@ pub async fn create_stream( .map(TryInto::try_into) .transpose()? .unwrap_or_default(); + let encryption = config.encryption; let info = backend .create_stream( basin, request.stream, config, + encryption, CreateMode::CreateOnly(request_token), ) .await?; @@ -220,7 +222,7 @@ pub async fn create_or_reconfigure_stream( .transpose()? .unwrap_or_default(); let info = backend - .create_stream(basin, stream, config, CreateMode::CreateOrReconfigure) + .create_stream(basin, stream, config, None, CreateMode::CreateOrReconfigure) .await?; let status = if info.is_created() { StatusCode::CREATED diff --git a/lite/src/init.rs b/lite/src/init.rs index c7a1a76b..68fa49e9 100644 --- a/lite/src/init.rs +++ b/lite/src/init.rs @@ -388,6 +388,7 @@ pub async fn apply(backend: &Backend, spec: ResourcesSpec) -> eyre::Result<()> { basin.clone(), stream.clone(), reconfiguration, + None, CreateMode::CreateOrReconfigure, ) .await diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 2b361e4c..cfe3fefc 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -764,6 +764,7 @@ impl From for api::config::StreamConfig { retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), + encryption: None, } } } @@ -834,6 +835,7 @@ impl From for api::config::BasinConfig { default_stream_config: value.default_stream_config.map(Into::into), create_stream_on_append: value.create_stream_on_append, create_stream_on_read: value.create_stream_on_read, + allowed_encryption: vec![], } } } From 0798e1672049fada9dd92ec7320ce38bef9183f2 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Thu, 19 Mar 2026 04:08:31 +0530 Subject: [PATCH 02/42] . --- api/src/v1/config.rs | 40 ++++++++++++++++++--------------- common/src/types/config.rs | 37 +++++++++++++++++------------- lite/src/backend/streams.rs | 25 ++++++++++++--------- lite/src/handlers/v1/records.rs | 18 +++++++-------- lite/src/handlers/v1/streams.rs | 2 +- sdk/src/types.rs | 4 ++-- 6 files changed, 70 insertions(+), 56 deletions(-) diff --git a/api/src/v1/config.rs b/api/src/v1/config.rs index 992c89d0..772f4b89 100644 --- a/api/src/v1/config.rs +++ b/api/src/v1/config.rs @@ -272,7 +272,7 @@ pub struct StreamConfig { /// Encryption algorithm. `"aegis-256"` | `"aes-256-gcm"` | absent (plaintext). /// Immutable after stream creation. #[serde(skip_serializing_if = "Option::is_none")] - pub encryption: Option, + pub encryption_algorithm: Option, } impl StreamConfig { @@ -282,7 +282,7 @@ impl StreamConfig { retention_policy, timestamping, delete_on_empty, - encryption, + encryption_algorithm, } = config; let config = StreamConfig { @@ -290,7 +290,7 @@ impl StreamConfig { retention_policy: retention_policy.map(Into::into), timestamping: TimestampingConfig::to_opt(timestamping), delete_on_empty: DeleteOnEmptyConfig::to_opt(delete_on_empty), - encryption: encryption.and_then(|alg| match alg { + encryption_algorithm: encryption_algorithm.and_then(|alg| match alg { types::config::EncryptionAlgorithm::None => None, types::config::EncryptionAlgorithm::Aegis256 => Some("aegis-256".to_owned()), types::config::EncryptionAlgorithm::Aes256Gcm => Some("aes-256-gcm".to_owned()), @@ -311,7 +311,7 @@ impl From for StreamConfig { retention_policy, timestamping, delete_on_empty, - encryption, + encryption_algorithm, } = value; Self { @@ -319,7 +319,7 @@ impl From for StreamConfig { retention_policy: Some(retention_policy.into()), timestamping: Some(timestamping.into()), delete_on_empty: Some(delete_on_empty.into()), - encryption: encryption.map(|alg| match alg { + encryption_algorithm: encryption_algorithm.map(|alg| match alg { types::config::EncryptionAlgorithm::None => "none".to_owned(), types::config::EncryptionAlgorithm::Aegis256 => "aegis-256".to_owned(), types::config::EncryptionAlgorithm::Aes256Gcm => "aes-256-gcm".to_owned(), @@ -337,7 +337,7 @@ impl TryFrom for types::config::OptionalStreamConfig { retention_policy, timestamping, delete_on_empty, - encryption, + encryption_algorithm, } = value; let retention_policy = match retention_policy { @@ -345,7 +345,7 @@ impl TryFrom for types::config::OptionalStreamConfig { Some(policy) => Some(policy.try_into()?), }; - let encryption = match encryption.as_deref() { + let encryption_algorithm = match encryption_algorithm.as_deref() { None => None, Some("aegis-256") => Some(types::config::EncryptionAlgorithm::Aegis256), Some("aes-256-gcm") => Some(types::config::EncryptionAlgorithm::Aes256Gcm), @@ -361,7 +361,7 @@ impl TryFrom for types::config::OptionalStreamConfig { retention_policy, timestamping: timestamping.map(Into::into).unwrap_or_default(), delete_on_empty: delete_on_empty.map(Into::into).unwrap_or_default(), - encryption, + encryption_algorithm, }) } } @@ -445,7 +445,7 @@ pub struct BasinConfig { /// `"none"` = plaintext allowed; `"aegis-256"` | `"aes-256-gcm"` = algorithm allowed. /// Empty = all allowed (including plaintext). #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub allowed_encryption: Vec, + pub allowed_encryption_algorithms: Vec, } impl TryFrom for types::config::BasinConfig { @@ -456,22 +456,23 @@ impl TryFrom for types::config::BasinConfig { default_stream_config, create_stream_on_append, create_stream_on_read, - allowed_encryption, + allowed_encryption_algorithms, } = value; - let mut parsed_allowed_encryption = Vec::with_capacity(allowed_encryption.len()); - for s in &allowed_encryption { + let mut parsed_allowed_encryption_algorithms = + Vec::with_capacity(allowed_encryption_algorithms.len()); + for s in &allowed_encryption_algorithms { let alg = match s.as_str() { "none" => types::config::EncryptionAlgorithm::None, "aegis-256" => types::config::EncryptionAlgorithm::Aegis256, "aes-256-gcm" => types::config::EncryptionAlgorithm::Aes256Gcm, other => { return Err(types::ValidationError(format!( - "unknown encryption algorithm in allowed_encryption: {other:?}" + "unknown encryption algorithm in allowed_encryption_algorithms: {other:?}" ))); } }; - parsed_allowed_encryption.push(alg); + parsed_allowed_encryption_algorithms.push(alg); } Ok(Self { @@ -481,7 +482,7 @@ impl TryFrom for types::config::BasinConfig { }, create_stream_on_append, create_stream_on_read, - allowed_encryption: parsed_allowed_encryption, + allowed_encryption_algorithms: parsed_allowed_encryption_algorithms, }) } } @@ -492,14 +493,14 @@ impl From for BasinConfig { default_stream_config, create_stream_on_append, create_stream_on_read, - allowed_encryption, + allowed_encryption_algorithms, } = value; Self { default_stream_config: StreamConfig::to_opt(default_stream_config), create_stream_on_append, create_stream_on_read, - allowed_encryption: allowed_encryption + allowed_encryption_algorithms: allowed_encryption_algorithms .into_iter() .map(|alg| match alg { types::config::EncryptionAlgorithm::None => "none".to_owned(), @@ -613,6 +614,7 @@ mod tests { retention_policy, timestamping, delete_on_empty, + encryption_algorithm: None, }, ) } @@ -629,7 +631,7 @@ mod tests { default_stream_config, create_stream_on_append, create_stream_on_read, - allowed_encryption: vec![], + allowed_encryption_algorithms: vec![], } }, ) @@ -723,6 +725,7 @@ mod tests { delete_on_empty: types::config::OptionalDeleteOnEmptyConfig { min_age: doe.map(Duration::from_secs), }, + encryption_algorithm: None, } }) } @@ -910,6 +913,7 @@ mod tests { }, create_stream_on_append: base_on_append, create_stream_on_read: base_on_read, + allowed_encryption_algorithms: vec![], }; let reconfig = types::config::BasinReconfiguration::default(); diff --git a/common/src/types/config.rs b/common/src/types/config.rs index 69eb0ac0..7f9ac11b 100644 --- a/common/src/types/config.rs +++ b/common/src/types/config.rs @@ -35,12 +35,13 @@ use crate::maybe::Maybe; /// Encryption algorithm for stream records. /// -/// When used in `StreamConfig.encryption`, `None` is represented by Rust's `Option::None` -/// (meaning plaintext). When used in `BasinConfig.allowed_encryption`, the `None` variant -/// is a sentinel that explicitly permits plaintext streams. +/// When used in `StreamConfig.encryption_algorithm`, `None` is represented by Rust's `Option::None` +/// (meaning plaintext). When used in `BasinConfig.allowed_encryption_algorithms`, the `None` +/// variant is a sentinel that explicitly permits plaintext streams. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EncryptionAlgorithm { - /// Sentinel: plaintext is explicitly allowed (used only in `allowed_encryption` lists). + /// Sentinel: plaintext is explicitly allowed (used only in `allowed_encryption_algorithms` + /// lists). None, Aegis256, Aes256Gcm, @@ -120,7 +121,7 @@ pub struct StreamConfig { pub timestamping: TimestampingConfig, pub delete_on_empty: DeleteOnEmptyConfig, /// Encryption algorithm for this stream. `None` = plaintext. Immutable after creation. - pub encryption: Option, + pub encryption_algorithm: Option, } #[derive(Debug, Clone, Default)] @@ -246,7 +247,7 @@ pub struct OptionalStreamConfig { pub timestamping: OptionalTimestampingConfig, pub delete_on_empty: OptionalDeleteOnEmptyConfig, /// Encryption algorithm. `None` = not specified (falls back to basin default or plaintext). - pub encryption: Option, + pub encryption_algorithm: Option, } impl OptionalStreamConfig { @@ -291,14 +292,16 @@ impl OptionalStreamConfig { let delete_on_empty = self.delete_on_empty.merge(basin_defaults.delete_on_empty); - let encryption = self.encryption.or(basin_defaults.encryption); + let encryption_algorithm = self + .encryption_algorithm + .or(basin_defaults.encryption_algorithm); StreamConfig { storage_class, retention_policy, timestamping, delete_on_empty, - encryption, + encryption_algorithm, } } } @@ -310,7 +313,8 @@ impl From for StreamReconfiguration { retention_policy, timestamping, delete_on_empty, - encryption: _, // encryption is immutable; not represented in StreamReconfiguration + encryption_algorithm: _, /* encryption_algorithm is immutable; not represented in + * StreamReconfiguration */ } = value; Self { @@ -329,7 +333,7 @@ impl From for StreamConfig { retention_policy, timestamping, delete_on_empty, - encryption, + encryption_algorithm, } = value; Self { @@ -337,7 +341,7 @@ impl From for StreamConfig { retention_policy: retention_policy.unwrap_or_default(), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), - encryption, + encryption_algorithm, } } } @@ -349,7 +353,7 @@ impl From for OptionalStreamConfig { retention_policy, timestamping, delete_on_empty, - encryption, + encryption_algorithm, } = value; Self { @@ -357,7 +361,7 @@ impl From for OptionalStreamConfig { retention_policy: Some(retention_policy), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), - encryption, + encryption_algorithm, } } } @@ -370,7 +374,7 @@ pub struct BasinConfig { /// Allowlist of encryption algorithms for streams in this basin. /// Empty = all allowed (including plaintext). Use `EncryptionAlgorithm::None` to /// explicitly permit plaintext when the list is non-empty. - pub allowed_encryption: Vec, + pub allowed_encryption_algorithms: Vec, } impl BasinConfig { @@ -395,7 +399,8 @@ impl BasinConfig { self.create_stream_on_read = create_stream_on_read; } - // allowed_encryption is not reconfigurable via BasinReconfiguration (S2-ops managed). + // allowed_encryption_algorithms is not reconfigurable via BasinReconfiguration (S2-ops + // managed). self } } @@ -406,7 +411,7 @@ impl From for BasinReconfiguration { default_stream_config, create_stream_on_append, create_stream_on_read, - allowed_encryption: _, // not reconfigurable via BasinReconfiguration + allowed_encryption_algorithms: _, // not reconfigurable via BasinReconfiguration } = value; Self { diff --git a/lite/src/backend/streams.rs b/lite/src/backend/streams.rs index 156e10ee..6fdea9ae 100644 --- a/lite/src/backend/streams.rs +++ b/lite/src/backend/streams.rs @@ -153,7 +153,7 @@ impl Backend { Some(existing) => (existing.config.reconfigure(config), existing.created_at), None => { let mut cfg = OptionalStreamConfig::default().reconfigure(config); - cfg.encryption = encryption; + cfg.encryption_algorithm = encryption; (cfg, OffsetDateTime::now_utc()) } }; @@ -161,19 +161,22 @@ impl Backend { .merge(basin_meta.config.default_stream_config) .into(); - // Enforce basin's allowed_encryption list on creation. - if !is_reconfigure && !basin_meta.config.allowed_encryption.is_empty() { - let is_allowed = match resolved.encryption { + // Enforce basin's allowed_encryption_algorithms list on creation. + if !is_reconfigure && !basin_meta.config.allowed_encryption_algorithms.is_empty() { + let is_allowed = match resolved.encryption_algorithm { None => basin_meta .config - .allowed_encryption + .allowed_encryption_algorithms .contains(&EncryptionAlgorithm::None), - Some(alg) => basin_meta.config.allowed_encryption.contains(&alg), + Some(alg) => basin_meta + .config + .allowed_encryption_algorithms + .contains(&alg), }; if !is_allowed { return Err(CreateStreamError::InvalidConfig(format!( "encryption algorithm {:?} not permitted for this basin", - resolved.encryption + resolved.encryption_algorithm ))); } } @@ -308,12 +311,14 @@ impl Backend { .min_age .filter(|age| !age.is_zero()); - let original_encryption = meta.config.encryption; + let original_encryption = meta.config.encryption_algorithm; meta.config = meta.config.reconfigure(reconfig); // Encryption is immutable after creation. - if meta.config.encryption != original_encryption { - return Err(ReconfigureStreamError::ImmutableField("encryption")); + if meta.config.encryption_algorithm != original_encryption { + return Err(ReconfigureStreamError::ImmutableField( + "encryption_algorithm", + )); } txn.put(&meta_key, kv::stream_meta::ser_value(&meta))?; diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index b6a81e20..8b0a3778 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -197,12 +197,12 @@ pub async fn read( .await?; // On reads, EncryptionRequired is OK — return opaque bytes without decryption. - let decrypt_directive = match check_encryption_directive(config.encryption, directive.as_ref()) - { - Ok(checked) => checked.cloned(), - Err(EncryptionError::EncryptionRequired(_)) => None, - Err(e) => return Err(ServiceError::Validation(ValidationError(e.to_string()))), - }; + let decrypt_directive = + match check_encryption_directive(config.encryption_algorithm, directive.as_ref()) { + Ok(checked) => checked.cloned(), + Err(EncryptionError::EncryptionRequired(_)) => None, + Err(e) => return Err(ServiceError::Validation(ValidationError(e.to_string()))), + }; let start: ReadStart = start.try_into()?; match request { @@ -445,7 +445,7 @@ pub async fn append( .get_stream_config(basin.clone(), stream.clone()) .await?; - check_encryption_directive(config.encryption, directive.as_ref()) + check_encryption_directive(config.encryption_algorithm, directive.as_ref()) .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; match request { @@ -455,7 +455,7 @@ pub async fn append( } => { let input = encrypt_append_input( input, - config.encryption, + config.encryption_algorithm, directive.as_ref(), &basin, &stream, @@ -479,7 +479,7 @@ pub async fn append( } => { let (err_tx, err_rx) = tokio::sync::oneshot::channel(); - let stream_alg = config.encryption; + let stream_alg = config.encryption_algorithm; let enc_basin = basin.clone(); let enc_stream = stream.clone(); let inputs = async_stream::stream! { diff --git a/lite/src/handlers/v1/streams.rs b/lite/src/handlers/v1/streams.rs index 561cbd94..61a81c43 100644 --- a/lite/src/handlers/v1/streams.rs +++ b/lite/src/handlers/v1/streams.rs @@ -121,7 +121,7 @@ pub async fn create_stream( .map(TryInto::try_into) .transpose()? .unwrap_or_default(); - let encryption = config.encryption; + let encryption = config.encryption_algorithm; let info = backend .create_stream( basin, diff --git a/sdk/src/types.rs b/sdk/src/types.rs index cfe3fefc..c0228ce6 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -764,7 +764,7 @@ impl From for api::config::StreamConfig { retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), - encryption: None, + encryption_algorithm: None, } } } @@ -835,7 +835,7 @@ impl From for api::config::BasinConfig { default_stream_config: value.default_stream_config.map(Into::into), create_stream_on_append: value.create_stream_on_append, create_stream_on_read: value.create_stream_on_read, - allowed_encryption: vec![], + allowed_encryption_algorithms: vec![], } } } From f44c37b06cc418578b059231fc5172ca7b2950c3 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Thu, 19 Mar 2026 04:32:13 +0530 Subject: [PATCH 03/42] . --- common/src/encryption.rs | 114 ++++++++++++---- lite/src/backend/streams.rs | 2 - lite/src/handlers/v1/records.rs | 225 ++++++++++++-------------------- 3 files changed, 169 insertions(+), 172 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index 56da07ab..3a7a3240 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -87,12 +87,6 @@ pub enum EncryptionError { EncodingFailed(String), } -/// Parse the `S2-Encryption` header value. -/// -/// Returns: -/// - `Ok(None)` if the header is absent. -/// - `Ok(Some(directive))` on a valid header. -/// - `Err` if the header is present but malformed. pub fn parse_s2_encryption_header( headers: &HeaderMap, ) -> Result, EncryptionError> { @@ -105,12 +99,10 @@ pub fn parse_s2_encryption_header( .to_str() .map_err(|_| EncryptionError::MalformedHeader("header is not valid UTF-8".to_owned()))?; - // "attest" → client-side encryption, server passes through. if s.trim() == "attest" { return Ok(Some(EncryptionDirective::Attest)); } - // "alg=; key=<64 hex chars>" let (alg_part, key_part) = s.split_once(';').ok_or_else(|| { EncryptionError::MalformedHeader(format!("expected 'alg=...; key=...', got {s:?}")) })?; @@ -161,19 +153,11 @@ pub fn parse_s2_encryption_header( })) } -/// Validate the directive against the stream's configured encryption algorithm. -/// -/// Returns: -/// - `Ok(None)` if the stream has no encryption configured (pass-through). -/// - `Ok(Some(directive))` if encryption should proceed. -/// - `Err(EncryptionRequired)` if the stream requires encryption but none was provided. -/// - `Err(AlgorithmMismatch)` if the algorithms differ. pub fn check_encryption_directive<'a>( stream_alg: Option, directive: Option<&'a EncryptionDirective>, ) -> Result, EncryptionError> { let Some(required_alg) = stream_alg else { - // Stream has no encryption: ignore any directive. return Ok(None); }; @@ -192,7 +176,6 @@ pub fn check_encryption_directive<'a>( } } -/// Encode headers + body as `EnvelopeRecord` bytes (the plaintext input to encryption). pub fn encode_record_plaintext( headers: Vec
, body: Bytes, @@ -202,16 +185,12 @@ pub fn encode_record_plaintext( .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) } -/// Decode `EnvelopeRecord` bytes back to `(headers, body)` after decryption. pub fn decode_record_plaintext(bytes: Bytes) -> Result<(Vec
, Bytes), EncryptionError> { EnvelopeRecord::try_from(bytes) .map(|r| r.into_parts()) .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) } -/// Encrypt a record. -/// -/// Output layout: `[alg_id][random_nonce][ciphertext][tag]` as contiguous `Bytes`. pub fn encrypt_record( plaintext: &[u8], alg: EncryptionAlgorithm, @@ -240,7 +219,6 @@ pub fn encrypt_record( EncryptionError::EncodingFailed("invalid AES key length".to_owned()) })?; let nonce_generic = aes_gcm::Nonce::from_slice(&nonce); - // aes-gcm appends the 16-byte tag to the ciphertext automatically. let ciphertext_with_tag = cipher .encrypt( nonce_generic, @@ -264,12 +242,6 @@ pub fn encrypt_record( } } -/// Decrypt a record body. -/// -/// Returns: -/// - `Ok(Some(plaintext))` on success. -/// - `Ok(None)` if the first byte is not a known `alg_id` (unencrypted pass-through). -/// - `Err` on auth tag failure, truncation, or other error. pub fn decrypt_record( body: &[u8], key: &EncryptionKey, @@ -324,11 +296,95 @@ pub fn decrypt_record( .map_err(|_| EncryptionError::DecryptionFailed)?; Ok(Some(Bytes::from(plaintext))) } - // Unknown first byte: not encrypted by S2, pass through. _ => Ok(None), } } +pub fn encrypt_append_input( + input: crate::types::stream::AppendInput, + stream_alg: Option, + directive: Option<&EncryptionDirective>, + aad: &[u8], +) -> Result { + let Some(EncryptionDirective::Key { alg, key }) = directive else { + return Ok(input); + }; + if stream_alg.is_none() { + return Ok(input); + } + let mut encrypted_records = Vec::with_capacity(input.records.len()); + for record in input.records.into_iter() { + let crate::types::stream::AppendRecordParts { timestamp, record } = record.into(); + let encrypted = match &*record { + crate::record::Record::Envelope(env) => { + let plaintext = + encode_record_plaintext(env.headers().to_vec(), env.body().clone())?; + let enc_body = encrypt_record(&plaintext, *alg, key, aad)?; + let enc_record = crate::record::Record::try_from_parts(vec![], enc_body) + .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; + crate::record::Metered::from(enc_record) + } + crate::record::Record::Command(_) => record, + }; + encrypted_records.push( + crate::types::stream::AppendRecordParts { + timestamp, + record: encrypted, + } + .try_into() + .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_string()))?, + ); + } + let records: crate::types::stream::AppendRecordBatch = encrypted_records + .try_into() + .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_string()))?; + Ok(crate::types::stream::AppendInput { + records, + match_seq_num: input.match_seq_num, + fencing_token: input.fencing_token, + }) +} + +pub fn decrypt_read_batch( + batch: crate::types::stream::ReadBatch, + directive: Option<&EncryptionDirective>, + aad: &[u8], +) -> Result { + let Some(EncryptionDirective::Key { key, .. }) = directive else { + return Ok(batch); + }; + let records: Vec = batch + .records + .into_inner() + .into_iter() + .map(|sr| { + let crate::record::Record::Envelope(ref env) = sr.record else { + return Ok(sr); + }; + match decrypt_record(env.body(), key, aad)? { + None => Ok(sr), + Some(plaintext) => { + let (headers, body) = decode_record_plaintext(plaintext)?; + let record = crate::record::Record::try_from_parts(headers, body) + .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; + Ok(crate::record::SequencedRecord { + position: sr.position, + record, + }) + } + } + }) + .collect::>()?; + Ok(crate::types::stream::ReadBatch { + records: crate::record::Metered::from(records), + tail: batch.tail, + }) +} + +pub fn stream_aad(basin: &impl std::fmt::Display, stream: &impl std::fmt::Display) -> String { + format!("{basin}/{stream}") +} + #[cfg(test)] mod tests { use super::*; diff --git a/lite/src/backend/streams.rs b/lite/src/backend/streams.rs index 6fdea9ae..ba48862f 100644 --- a/lite/src/backend/streams.rs +++ b/lite/src/backend/streams.rs @@ -161,7 +161,6 @@ impl Backend { .merge(basin_meta.config.default_stream_config) .into(); - // Enforce basin's allowed_encryption_algorithms list on creation. if !is_reconfigure && !basin_meta.config.allowed_encryption_algorithms.is_empty() { let is_allowed = match resolved.encryption_algorithm { None => basin_meta @@ -314,7 +313,6 @@ impl Backend { let original_encryption = meta.config.encryption_algorithm; meta.config = meta.config.reconfigure(reconfig); - // Encryption is immutable after creation. if meta.config.encryption_algorithm != original_encryption { return Err(ReconfigureStreamError::ImmutableField( "encryption_algorithm", diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index 8b0a3778..dfd01732 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -15,20 +15,17 @@ use s2_api::{ use s2_common::{ caps::RECORD_BATCH_MAX, encryption::{ - EncryptionDirective, EncryptionError, check_encryption_directive, decode_record_plaintext, - decrypt_record, encode_record_plaintext, encrypt_record, parse_s2_encryption_header, + self, EncryptionDirective, EncryptionError, check_encryption_directive, + parse_s2_encryption_header, stream_aad, }, http::extract::Header, read_extent::{CountOrBytes, ReadLimit}, - record::{Metered, MeteredSize as _, Record, SequencedRecord}, + record::{Metered, MeteredSize as _}, types::{ ValidationError, basin::BasinName, config::EncryptionAlgorithm, - stream::{ - AppendInput, AppendRecordBatch, AppendRecordParts, ReadBatch, ReadEnd, ReadFrom, - ReadSessionOutput, ReadStart, StreamName, - }, + stream::{ReadBatch, ReadEnd, ReadFrom, ReadSessionOutput, ReadStart, StreamName}, }, }; @@ -45,6 +42,71 @@ pub fn router() -> axum::Router { .route(super::paths::streams::records::APPEND, post(append)) } +struct EncryptionContext { + aad: Vec, + stream_alg: Option, + directive: Option, +} + +impl EncryptionContext { + async fn resolve_for_append( + backend: &Backend, + headers: &http::HeaderMap, + basin: &BasinName, + stream: &StreamName, + ) -> Result { + let directive = parse_s2_encryption_header(headers) + .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; + let config = backend + .get_stream_config(basin.clone(), stream.clone()) + .await?; + check_encryption_directive(config.encryption_algorithm, directive.as_ref()) + .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; + Ok(Self { + aad: stream_aad(basin, stream).into_bytes(), + stream_alg: config.encryption_algorithm, + directive, + }) + } + + async fn resolve_for_read( + backend: &Backend, + headers: &http::HeaderMap, + basin: &BasinName, + stream: &StreamName, + ) -> Result { + let directive = parse_s2_encryption_header(headers) + .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; + let config = backend + .get_stream_config(basin.clone(), stream.clone()) + .await?; + let checked = + match check_encryption_directive(config.encryption_algorithm, directive.as_ref()) { + Ok(d) => d.cloned(), + Err(EncryptionError::EncryptionRequired(_)) => None, + Err(e) => { + return Err(ServiceError::Validation(ValidationError(e.to_string()))); + } + }; + Ok(Self { + aad: stream_aad(basin, stream).into_bytes(), + stream_alg: config.encryption_algorithm, + directive: checked, + }) + } + + fn encrypt_input( + &self, + input: s2_common::types::stream::AppendInput, + ) -> Result { + encryption::encrypt_append_input(input, self.stream_alg, self.directive.as_ref(), &self.aad) + } + + fn decrypt_batch(&self, batch: ReadBatch) -> Result { + encryption::decrypt_read_batch(batch, self.directive.as_ref(), &self.aad) + } +} + fn validate_read_until(start: ReadStart, end: ReadEnd) -> Result<(), ServiceError> { if let ReadFrom::Timestamp(ts) = start.from && end.until.deny(ts) @@ -189,20 +251,7 @@ pub async fn read( request, }: ReadArgs, ) -> Result { - let directive = parse_s2_encryption_header(&headers) - .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; - - let config = backend - .get_stream_config(basin.clone(), stream.clone()) - .await?; - - // On reads, EncryptionRequired is OK — return opaque bytes without decryption. - let decrypt_directive = - match check_encryption_directive(config.encryption_algorithm, directive.as_ref()) { - Ok(checked) => checked.cloned(), - Err(EncryptionError::EncryptionRequired(_)) => None, - Err(e) => return Err(ServiceError::Validation(ValidationError(e.to_string()))), - }; + let enc = EncryptionContext::resolve_for_read(&backend, &headers, &basin, &stream).await?; let start: ReadStart = start.try_into()?; match request { @@ -215,8 +264,7 @@ pub async fn read( .read(basin.clone(), stream.clone(), start, end) .await?; let batch = merge_read_session(session, end.wait).await?; - let batch = decrypt_read_batch(batch, decrypt_directive.as_ref(), &basin, &stream) - .map_err(ReadError::Encryption)?; + let batch = enc.decrypt_batch(batch).map_err(ReadError::Encryption)?; match response_mime { JsonOrProto::Json => Ok(Json(v1t::stream::json::serialize_read_batch( format, &batch, @@ -247,7 +295,7 @@ pub async fn read( yield v1t::stream::sse::ping_event(); }, Ok(ReadSessionOutput::Batch(batch)) => { - let batch = match decrypt_read_batch(batch, decrypt_directive.as_ref(), &basin, &stream) { + let batch = match enc.decrypt_batch(batch) { Ok(batch) => batch, Err(err) => { let (_, body) = ServiceError::from(ReadError::Encryption(err)).to_response().to_parts(); @@ -295,9 +343,7 @@ pub async fn read( tail: Some(tail.into()), }), Ok(ReadSessionOutput::Batch(batch)) => { - let batch = - decrypt_read_batch(batch, decrypt_directive.as_ref(), &basin, &stream) - .map_err(ReadError::Encryption)?; + let batch = enc.decrypt_batch(batch).map_err(ReadError::Encryption)?; Ok(v1t::stream::proto::ReadBatch::from(batch)) } Err(e) => Err(e), @@ -356,44 +402,6 @@ async fn merge_read_session( Ok(acc) } -fn decrypt_read_batch( - batch: ReadBatch, - directive: Option<&EncryptionDirective>, - basin: &BasinName, - stream: &StreamName, -) -> Result { - let Some(EncryptionDirective::Key { key, .. }) = directive else { - return Ok(batch); - }; - let aad = format!("{basin}/{stream}"); - let records: Vec = batch - .records - .into_inner() - .into_iter() - .map(|sr| { - let Record::Envelope(ref env) = sr.record else { - return Ok(sr); - }; - match decrypt_record(env.body(), key, aad.as_bytes())? { - None => Ok(sr), - Some(plaintext) => { - let (headers, body) = decode_record_plaintext(plaintext)?; - let record = Record::try_from_parts(headers, body) - .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; - Ok(SequencedRecord { - position: sr.position, - record, - }) - } - } - }) - .collect::>()?; - Ok(ReadBatch { - records: Metered::from(records), - tail: batch.tail, - }) -} - #[derive(FromRequest)] #[from_request(rejection(ServiceError))] pub struct AppendArgs { @@ -438,29 +446,16 @@ pub async fn append( request, }: AppendArgs, ) -> Result { - let directive = parse_s2_encryption_header(&headers) - .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; - - let config = backend - .get_stream_config(basin.clone(), stream.clone()) - .await?; - - check_encryption_directive(config.encryption_algorithm, directive.as_ref()) - .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; + let enc = EncryptionContext::resolve_for_append(&backend, &headers, &basin, &stream).await?; match request { v1t::stream::AppendRequest::Unary { input, response_mime, } => { - let input = encrypt_append_input( - input, - config.encryption_algorithm, - directive.as_ref(), - &basin, - &stream, - ) - .map_err(crate::backend::error::AppendError::Encryption)?; + let input = enc + .encrypt_input(input) + .map_err(crate::backend::error::AppendError::Encryption)?; let ack = backend.append(basin, stream, input).await?; match response_mime { JsonOrProto::Json => { @@ -479,25 +474,20 @@ pub async fn append( } => { let (err_tx, err_rx) = tokio::sync::oneshot::channel(); - let stream_alg = config.encryption_algorithm; - let enc_basin = basin.clone(); - let enc_stream = stream.clone(); let inputs = async_stream::stream! { tokio::pin!(inputs); let mut err_tx = Some(err_tx); while let Some(input) = inputs.next().await { match input { - Ok(input) => { - match encrypt_append_input(input, stream_alg, directive.as_ref(), &enc_basin, &enc_stream) { - Ok(encrypted) => yield encrypted, - Err(e) => { - if let Some(tx) = err_tx.take() { - let _ = tx.send(ServiceError::Append(e.into())); - } - break; + Ok(input) => match enc.encrypt_input(input) { + Ok(encrypted) => yield encrypted, + Err(e) => { + if let Some(tx) = err_tx.take() { + let _ = tx.send(ServiceError::Append(e.into())); } + break; } - } + }, Err(e) => { if let Some(tx) = err_tx.take() { let _ = tx.send(e.into()); @@ -536,50 +526,3 @@ pub async fn append( } } } - -fn encrypt_append_input( - input: AppendInput, - stream_alg: Option, - directive: Option<&EncryptionDirective>, - basin: &BasinName, - stream: &StreamName, -) -> Result { - let Some(EncryptionDirective::Key { alg, key }) = directive else { - return Ok(input); - }; - if stream_alg.is_none() { - return Ok(input); - } - let aad = format!("{basin}/{stream}"); - let mut encrypted_records = Vec::with_capacity(input.records.len()); - for record in input.records.into_iter() { - let AppendRecordParts { timestamp, record } = record.into(); - let encrypted = match &*record { - Record::Envelope(env) => { - let plaintext = - encode_record_plaintext(env.headers().to_vec(), env.body().clone())?; - let enc_body = encrypt_record(&plaintext, *alg, key, aad.as_bytes())?; - let enc_record = Record::try_from_parts(vec![], enc_body) - .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; - Metered::from(enc_record) - } - Record::Command(_) => record, - }; - encrypted_records.push( - AppendRecordParts { - timestamp, - record: encrypted, - } - .try_into() - .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_string()))?, - ); - } - let records: AppendRecordBatch = encrypted_records - .try_into() - .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_string()))?; - Ok(AppendInput { - records, - match_seq_num: input.match_seq_num, - fencing_token: input.fencing_token, - }) -} From d6cb24bf8d3420634564dbe1e6343ae439f05439 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 20 Mar 2026 14:17:37 +0530 Subject: [PATCH 04/42] . --- api/src/v1/config.rs | 88 ++++++++++++++++++--------------- common/src/encryption.rs | 9 ++-- common/src/types/config.rs | 21 ++++++++ lite/src/handlers/v1/streams.rs | 19 +++++-- sdk/src/types.rs | 67 ++++++++++++++++++++++++- 5 files changed, 152 insertions(+), 52 deletions(-) diff --git a/api/src/v1/config.rs b/api/src/v1/config.rs index 772f4b89..8d891cd7 100644 --- a/api/src/v1/config.rs +++ b/api/src/v1/config.rs @@ -290,11 +290,9 @@ impl StreamConfig { retention_policy: retention_policy.map(Into::into), timestamping: TimestampingConfig::to_opt(timestamping), delete_on_empty: DeleteOnEmptyConfig::to_opt(delete_on_empty), - encryption_algorithm: encryption_algorithm.and_then(|alg| match alg { - types::config::EncryptionAlgorithm::None => None, - types::config::EncryptionAlgorithm::Aegis256 => Some("aegis-256".to_owned()), - types::config::EncryptionAlgorithm::Aes256Gcm => Some("aes-256-gcm".to_owned()), - }), + encryption_algorithm: encryption_algorithm + .filter(|alg| *alg != types::config::EncryptionAlgorithm::None) + .map(|alg| alg.as_api_str().to_owned()), }; if config == Self::default() { None @@ -319,11 +317,7 @@ impl From for StreamConfig { retention_policy: Some(retention_policy.into()), timestamping: Some(timestamping.into()), delete_on_empty: Some(delete_on_empty.into()), - encryption_algorithm: encryption_algorithm.map(|alg| match alg { - types::config::EncryptionAlgorithm::None => "none".to_owned(), - types::config::EncryptionAlgorithm::Aegis256 => "aegis-256".to_owned(), - types::config::EncryptionAlgorithm::Aes256Gcm => "aes-256-gcm".to_owned(), - }), + encryption_algorithm: encryption_algorithm.map(|alg| alg.as_api_str().to_owned()), } } } @@ -345,16 +339,14 @@ impl TryFrom for types::config::OptionalStreamConfig { Some(policy) => Some(policy.try_into()?), }; - let encryption_algorithm = match encryption_algorithm.as_deref() { - None => None, - Some("aegis-256") => Some(types::config::EncryptionAlgorithm::Aegis256), - Some("aes-256-gcm") => Some(types::config::EncryptionAlgorithm::Aes256Gcm), - Some(other) => { - return Err(types::ValidationError(format!( - "unknown encryption algorithm: {other:?}" - ))); - } - }; + let encryption_algorithm = encryption_algorithm + .as_deref() + .map(|s| { + types::config::EncryptionAlgorithm::parse_api_str(s).ok_or_else(|| { + types::ValidationError(format!("unknown encryption algorithm: {s:?}")) + }) + }) + .transpose()?; Ok(Self { storage_class: storage_class.map(Into::into), @@ -387,6 +379,28 @@ pub struct StreamReconfiguration { #[serde(default, skip_serializing_if = "Maybe::is_unspecified")] #[cfg_attr(feature = "utoipa", schema(value_type = Option))] pub delete_on_empty: Maybe>, + /// Encryption algorithm for the stream. Only used during stream creation; ignored on + /// reconfigure (encryption is immutable after creation). + /// `"aegis-256"` | `"aes-256-gcm"` | absent (plaintext). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encryption_algorithm: Option, +} + +impl StreamReconfiguration { + /// Extract and validate the encryption algorithm, consuming the field. + pub fn take_encryption_algorithm( + &mut self, + ) -> Result, types::ValidationError> { + self.encryption_algorithm + .take() + .as_deref() + .map(|s| { + types::config::EncryptionAlgorithm::parse_api_str(s).ok_or_else(|| { + types::ValidationError(format!("unknown encryption algorithm: {s:?}")) + }) + }) + .transpose() + } } impl TryFrom for types::config::StreamReconfiguration { @@ -398,6 +412,7 @@ impl TryFrom for types::config::StreamReconfiguration { retention_policy, timestamping, delete_on_empty, + encryption_algorithm: _, // immutable; not part of StreamReconfiguration } = value; Ok(Self { @@ -423,6 +438,7 @@ impl From for StreamReconfiguration { retention_policy: retention_policy.map_opt(Into::into), timestamping: timestamping.map_opt(Into::into), delete_on_empty: delete_on_empty.map_opt(Into::into), + encryption_algorithm: None, } } } @@ -459,21 +475,16 @@ impl TryFrom for types::config::BasinConfig { allowed_encryption_algorithms, } = value; - let mut parsed_allowed_encryption_algorithms = - Vec::with_capacity(allowed_encryption_algorithms.len()); - for s in &allowed_encryption_algorithms { - let alg = match s.as_str() { - "none" => types::config::EncryptionAlgorithm::None, - "aegis-256" => types::config::EncryptionAlgorithm::Aegis256, - "aes-256-gcm" => types::config::EncryptionAlgorithm::Aes256Gcm, - other => { - return Err(types::ValidationError(format!( - "unknown encryption algorithm in allowed_encryption_algorithms: {other:?}" - ))); - } - }; - parsed_allowed_encryption_algorithms.push(alg); - } + let parsed_allowed_encryption_algorithms = allowed_encryption_algorithms + .iter() + .map(|s| { + types::config::EncryptionAlgorithm::parse_api_str(s).ok_or_else(|| { + types::ValidationError(format!( + "unknown encryption algorithm in allowed_encryption_algorithms: {s:?}" + )) + }) + }) + .collect::, _>>()?; Ok(Self { default_stream_config: match default_stream_config { @@ -502,11 +513,7 @@ impl From for BasinConfig { create_stream_on_read, allowed_encryption_algorithms: allowed_encryption_algorithms .into_iter() - .map(|alg| match alg { - types::config::EncryptionAlgorithm::None => "none".to_owned(), - types::config::EncryptionAlgorithm::Aegis256 => "aegis-256".to_owned(), - types::config::EncryptionAlgorithm::Aes256Gcm => "aes-256-gcm".to_owned(), - }) + .map(|alg| alg.as_api_str().to_owned()) .collect(), } } @@ -661,6 +668,7 @@ mod tests { retention_policy, timestamping, delete_on_empty, + encryption_algorithm: None, } }, ) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index 3a7a3240..a317d027 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -123,14 +123,13 @@ pub fn parse_s2_encryption_header( })? .trim(); - let alg = match alg_str { - "aegis-256" => EncryptionAlgorithm::Aegis256, - "aes-256-gcm" => EncryptionAlgorithm::Aes256Gcm, - other => { + let alg = match EncryptionAlgorithm::parse_api_str(alg_str) { + Some(EncryptionAlgorithm::None) | None => { return Err(EncryptionError::MalformedHeader(format!( - "unknown algorithm {other:?}; expected 'aegis-256' or 'aes-256-gcm'" + "unknown algorithm {alg_str:?}; expected 'aegis-256' or 'aes-256-gcm'" ))); } + Some(alg) => alg, }; if key_hex.len() != 64 { diff --git a/common/src/types/config.rs b/common/src/types/config.rs index 7f9ac11b..107639c3 100644 --- a/common/src/types/config.rs +++ b/common/src/types/config.rs @@ -47,6 +47,27 @@ pub enum EncryptionAlgorithm { Aes256Gcm, } +impl EncryptionAlgorithm { + /// Wire-format string used in JSON API and `S2-Encryption` header. + pub fn as_api_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Aegis256 => "aegis-256", + Self::Aes256Gcm => "aes-256-gcm", + } + } + + /// Parse from wire-format string. Returns `None` for unrecognised values. + pub fn parse_api_str(s: &str) -> Option { + match s { + "none" => Some(Self::None), + "aegis-256" => Some(Self::Aegis256), + "aes-256-gcm" => Some(Self::Aes256Gcm), + _ => Option::None, + } + } +} + #[derive( Debug, Default, diff --git a/lite/src/handlers/v1/streams.rs b/lite/src/handlers/v1/streams.rs index 61a81c43..1a9e1207 100644 --- a/lite/src/handlers/v1/streams.rs +++ b/lite/src/handlers/v1/streams.rs @@ -217,12 +217,21 @@ pub async fn create_or_reconfigure_stream( config: JsonOpt(config), }: CreateOrReconfigureArgs, ) -> Result<(StatusCode, Json), ServiceError> { - let config: StreamReconfiguration = config - .map(TryInto::try_into) - .transpose()? - .unwrap_or_default(); + let (encryption, config): (_, StreamReconfiguration) = match config { + Some(mut api_config) => { + let enc = api_config.take_encryption_algorithm()?; + (enc, api_config.try_into()?) + } + None => (None, StreamReconfiguration::default()), + }; let info = backend - .create_stream(basin, stream, config, None, CreateMode::CreateOrReconfigure) + .create_stream( + basin, + stream, + config, + encryption, + CreateMode::CreateOrReconfigure, + ) .await?; let status = if info.is_created() { StatusCode::CREATED diff --git a/sdk/src/types.rs b/sdk/src/types.rs index c0228ce6..85bdc1dd 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -533,6 +533,15 @@ impl From for api::config::StorageClass { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Encryption algorithm for stream records. +pub enum EncryptionAlgorithm { + /// AEGIS-256 authenticated encryption. + Aegis256, + /// AES-256-GCM authenticated encryption (NIST-compliant). + Aes256Gcm, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] /// Retention policy for records in a stream. pub enum RetentionPolicy { @@ -705,6 +714,10 @@ pub struct StreamConfig { /// /// See [`DeleteOnEmptyConfig`] for defaults. pub delete_on_empty: Option, + /// Encryption algorithm for this stream. Immutable after creation. + /// + /// Defaults to `None` (plaintext). + pub encryption_algorithm: Option, } impl StreamConfig { @@ -744,6 +757,14 @@ impl StreamConfig { ..self } } + + /// Set the encryption algorithm for the stream. Immutable after creation. + pub fn with_encryption_algorithm(self, alg: EncryptionAlgorithm) -> Self { + Self { + encryption_algorithm: Some(alg), + ..self + } + } } impl From for StreamConfig { @@ -753,18 +774,34 @@ impl From for StreamConfig { retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), + encryption_algorithm: value.encryption_algorithm.as_deref().and_then(|s| { + use s2_common::types::config::EncryptionAlgorithm as E; + match E::parse_api_str(s)? { + E::Aegis256 => Some(EncryptionAlgorithm::Aegis256), + E::Aes256Gcm => Some(EncryptionAlgorithm::Aes256Gcm), + E::None => Option::None, + } + }), } } } impl From for api::config::StreamConfig { fn from(value: StreamConfig) -> Self { + use s2_common::types::config::EncryptionAlgorithm as E; Self { storage_class: value.storage_class.map(Into::into), retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), - encryption_algorithm: None, + encryption_algorithm: value.encryption_algorithm.map(|a| { + match a { + EncryptionAlgorithm::Aegis256 => E::Aegis256, + EncryptionAlgorithm::Aes256Gcm => E::Aes256Gcm, + } + .as_api_str() + .to_owned() + }), } } } @@ -785,6 +822,9 @@ pub struct BasinConfig { /// /// Defaults to `false`. pub create_stream_on_read: bool, + /// Allowlist of encryption algorithms permitted for streams in this basin. + /// Empty means all algorithms (including plaintext) are allowed. + pub allowed_encryption_algorithms: Vec, } impl BasinConfig { @@ -821,21 +861,43 @@ impl BasinConfig { impl From for BasinConfig { fn from(value: api::config::BasinConfig) -> Self { + use s2_common::types::config::EncryptionAlgorithm as E; Self { default_stream_config: value.default_stream_config.map(Into::into), create_stream_on_append: value.create_stream_on_append, create_stream_on_read: value.create_stream_on_read, + allowed_encryption_algorithms: value + .allowed_encryption_algorithms + .iter() + .filter_map(|s| match E::parse_api_str(s)? { + E::Aegis256 => Some(EncryptionAlgorithm::Aegis256), + E::Aes256Gcm => Some(EncryptionAlgorithm::Aes256Gcm), + E::None => Option::None, + }) + .collect(), } } } impl From for api::config::BasinConfig { fn from(value: BasinConfig) -> Self { + use s2_common::types::config::EncryptionAlgorithm as E; Self { default_stream_config: value.default_stream_config.map(Into::into), create_stream_on_append: value.create_stream_on_append, create_stream_on_read: value.create_stream_on_read, - allowed_encryption_algorithms: vec![], + allowed_encryption_algorithms: value + .allowed_encryption_algorithms + .into_iter() + .map(|a| { + match a { + EncryptionAlgorithm::Aegis256 => E::Aegis256, + EncryptionAlgorithm::Aes256Gcm => E::Aes256Gcm, + } + .as_api_str() + .to_owned() + }) + .collect(), } } } @@ -1310,6 +1372,7 @@ impl From for api::config::StreamReconfiguration { retention_policy: value.retention_policy.map(|m| m.map(Into::into)), timestamping: value.timestamping.map(|m| m.map(Into::into)), delete_on_empty: value.delete_on_empty.map(|m| m.map(Into::into)), + encryption_algorithm: None, } } } From e11c23fbdf1c357e7668d52f22be25ab2b7caed4 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 20 Mar 2026 20:51:01 +0530 Subject: [PATCH 05/42] . --- common/src/encryption.rs | 209 +++++++++++-------- lite/src/backend/append.rs | 12 +- lite/src/backend/error.rs | 3 + lite/src/backend/mod.rs | 2 + lite/src/backend/read.rs | 11 +- lite/src/backend/streamer.rs | 48 ++++- lite/src/handlers/v1/records.rs | 36 +--- lite/tests/backend/common.rs | 2 +- lite/tests/backend/control_plane/stream.rs | 6 +- lite/tests/backend/data_plane/append.rs | 53 +++-- lite/tests/backend/data_plane/auto_create.rs | 6 +- lite/tests/backend/data_plane/mixed.rs | 4 +- lite/tests/backend/data_plane/read.rs | 24 +-- lite/tests/backend/data_plane/read_follow.rs | 6 +- 14 files changed, 252 insertions(+), 170 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index a317d027..bf74aa45 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -81,6 +81,8 @@ pub enum EncryptionError { }, #[error("Encryption required: stream has encryption={0:?} but no key was provided")] EncryptionRequired(EncryptionAlgorithm), + #[error("Encryption key provided but stream is plaintext")] + EncryptionNotExpected, #[error("Decryption failed")] DecryptionFailed, #[error("Record encoding error: {0}")] @@ -157,6 +159,9 @@ pub fn check_encryption_directive<'a>( directive: Option<&'a EncryptionDirective>, ) -> Result, EncryptionError> { let Some(required_alg) = stream_alg else { + if matches!(directive, Some(EncryptionDirective::Key { .. })) { + return Err(EncryptionError::EncryptionNotExpected); + } return Ok(None); }; @@ -190,18 +195,34 @@ pub fn decode_record_plaintext(bytes: Bytes) -> Result<(Vec
, Bytes), Enc .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) } +/// Build the effective AAD by appending the algorithm ID byte and seq_num to the +/// caller-provided base AAD. This binds the algorithm tag and stream position into +/// the AEAD authentication, so flipping the stored alg_id byte or reordering records +/// within a stream is detected as an authentication failure. +/// +/// Layout: `[base_aad | alg_id: 1 byte | seq_num: 8 bytes LE]` +fn effective_aad(aad: &[u8], alg_id: u8, seq_num: crate::record::SeqNum) -> Vec { + let mut buf = Vec::with_capacity(aad.len() + 1 + 8); + buf.extend_from_slice(aad); + buf.push(alg_id); + buf.extend_from_slice(&seq_num.to_le_bytes()); + buf +} + pub fn encrypt_record( plaintext: &[u8], alg: EncryptionAlgorithm, key: &EncryptionKey, aad: &[u8], + seq_num: crate::record::SeqNum, ) -> Result { match alg { EncryptionAlgorithm::Aegis256 => { + let full_aad = effective_aad(aad, ALG_ID_AEGIS256, seq_num); let nonce: [u8; NONCE_BYTES_AEGIS256] = random(); let (ciphertext, tag) = Aegis256::::new(&key.expose_secret().0, &nonce) - .encrypt(plaintext, aad); + .encrypt(plaintext, &full_aad); let mut out = BytesMut::with_capacity( 1 + NONCE_BYTES_AEGIS256 + ciphertext.len() + TAG_BYTES_AEGIS256, @@ -213,6 +234,7 @@ pub fn encrypt_record( Ok(out.freeze()) } EncryptionAlgorithm::Aes256Gcm => { + let full_aad = effective_aad(aad, ALG_ID_AES256GCM, seq_num); let nonce: [u8; NONCE_BYTES_AES256GCM] = random(); let cipher = Aes256Gcm::new_from_slice(&key.expose_secret().0).map_err(|_| { EncryptionError::EncodingFailed("invalid AES key length".to_owned()) @@ -223,7 +245,7 @@ pub fn encrypt_record( nonce_generic, Payload { msg: plaintext, - aad, + aad: &full_aad, }, ) .map_err(|_| EncryptionError::DecryptionFailed)?; @@ -245,37 +267,34 @@ pub fn decrypt_record( body: &[u8], key: &EncryptionKey, aad: &[u8], -) -> Result, EncryptionError> { - let alg_id = match body.first() { - Some(&b) => b, - None => return Ok(None), - }; + seq_num: crate::record::SeqNum, +) -> Result { + let (&alg_id, rest) = body + .split_first() + .ok_or(EncryptionError::DecryptionFailed)?; + + let full_aad = effective_aad(aad, alg_id, seq_num); match alg_id { ALG_ID_AEGIS256 => { // Layout after alg_id: [nonce:32][ciphertext:n][tag:32] - let rest = &body[1..]; if rest.len() < NONCE_BYTES_AEGIS256 + TAG_BYTES_AEGIS256 { return Err(EncryptionError::DecryptionFailed); } let nonce: &[u8; NONCE_BYTES_AEGIS256] = rest[..NONCE_BYTES_AEGIS256].try_into().unwrap(); let after_nonce = &rest[NONCE_BYTES_AEGIS256..]; - if after_nonce.len() < TAG_BYTES_AEGIS256 { - return Err(EncryptionError::DecryptionFailed); - } let tag_offset = after_nonce.len() - TAG_BYTES_AEGIS256; let ciphertext = &after_nonce[..tag_offset]; let tag: &[u8; TAG_BYTES_AEGIS256] = after_nonce[tag_offset..].try_into().unwrap(); let plaintext = Aegis256::::new(&key.expose_secret().0, nonce) - .decrypt(ciphertext, tag, aad) + .decrypt(ciphertext, tag, &full_aad) .map_err(|_| EncryptionError::DecryptionFailed)?; - Ok(Some(Bytes::from(plaintext))) + Ok(Bytes::from(plaintext)) } ALG_ID_AES256GCM => { // Layout after alg_id: [nonce:12][ciphertext+tag:n+16] - let rest = &body[1..]; if rest.len() < NONCE_BYTES_AES256GCM + TAG_BYTES_AES256GCM { return Err(EncryptionError::DecryptionFailed); } @@ -289,59 +308,39 @@ pub fn decrypt_record( nonce_generic, Payload { msg: ciphertext_with_tag, - aad, + aad: &full_aad, }, ) .map_err(|_| EncryptionError::DecryptionFailed)?; - Ok(Some(Bytes::from(plaintext))) + Ok(Bytes::from(plaintext)) } - _ => Ok(None), + _ => Err(EncryptionError::DecryptionFailed), } } -pub fn encrypt_append_input( - input: crate::types::stream::AppendInput, - stream_alg: Option, - directive: Option<&EncryptionDirective>, +pub fn encrypt_sequenced_records( + records: Vec>, + alg: EncryptionAlgorithm, + key: &EncryptionKey, aad: &[u8], -) -> Result { - let Some(EncryptionDirective::Key { alg, key }) = directive else { - return Ok(input); - }; - if stream_alg.is_none() { - return Ok(input); - } - let mut encrypted_records = Vec::with_capacity(input.records.len()); - for record in input.records.into_iter() { - let crate::types::stream::AppendRecordParts { timestamp, record } = record.into(); - let encrypted = match &*record { - crate::record::Record::Envelope(env) => { - let plaintext = - encode_record_plaintext(env.headers().to_vec(), env.body().clone())?; - let enc_body = encrypt_record(&plaintext, *alg, key, aad)?; - let enc_record = crate::record::Record::try_from_parts(vec![], enc_body) - .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; - crate::record::Metered::from(enc_record) - } - crate::record::Record::Command(_) => record, - }; - encrypted_records.push( - crate::types::stream::AppendRecordParts { - timestamp, - record: encrypted, - } - .try_into() - .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_string()))?, - ); - } - let records: crate::types::stream::AppendRecordBatch = encrypted_records - .try_into() - .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_string()))?; - Ok(crate::types::stream::AppendInput { - records, - match_seq_num: input.match_seq_num, - fencing_token: input.fencing_token, - }) +) -> Result>, EncryptionError> { + records + .into_iter() + .map(|msr| { + let crate::record::SequencedRecord { position, record } = msr.into_inner(); + let encrypted = match &record { + crate::record::Record::Envelope(env) => { + let plaintext = + encode_record_plaintext(env.headers().to_vec(), env.body().clone())?; + let enc_body = encrypt_record(&plaintext, alg, key, aad, position.seq_num)?; + crate::record::Record::try_from_parts(vec![], enc_body) + .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))? + } + crate::record::Record::Command(_) => record, + }; + Ok(crate::record::Metered::from(encrypted.sequenced(position))) + }) + .collect() } pub fn decrypt_read_batch( @@ -360,18 +359,14 @@ pub fn decrypt_read_batch( let crate::record::Record::Envelope(ref env) = sr.record else { return Ok(sr); }; - match decrypt_record(env.body(), key, aad)? { - None => Ok(sr), - Some(plaintext) => { - let (headers, body) = decode_record_plaintext(plaintext)?; - let record = crate::record::Record::try_from_parts(headers, body) - .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; - Ok(crate::record::SequencedRecord { - position: sr.position, - record, - }) - } - } + let plaintext = decrypt_record(env.body(), key, aad, sr.position.seq_num)?; + let (headers, body) = decode_record_plaintext(plaintext)?; + let record = crate::record::Record::try_from_parts(headers, body) + .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; + Ok(crate::record::SequencedRecord { + position: sr.position, + record, + }) }) .collect::>()?; Ok(crate::types::stream::ReadBatch { @@ -397,6 +392,7 @@ mod tests { } const AAD: &[u8] = b"test-basin/test-stream"; + const SEQ: u64 = 42; fn roundtrip(alg: EncryptionAlgorithm) { let headers = vec![Header { @@ -407,8 +403,8 @@ mod tests { let plaintext = encode_record_plaintext(headers.clone(), body.clone()).unwrap(); let key = make_key_fn(); - let ciphertext = encrypt_record(&plaintext, alg, &key, AAD).unwrap(); - let decrypted = decrypt_record(&ciphertext, &key, AAD).unwrap().unwrap(); + let ciphertext = encrypt_record(&plaintext, alg, &key, AAD, SEQ).unwrap(); + let decrypted = decrypt_record(&ciphertext, &key, AAD, SEQ).unwrap(); let (out_headers, out_body) = decode_record_plaintext(decrypted).unwrap(); assert_eq!(out_headers, headers); @@ -430,8 +426,8 @@ mod tests { let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); let ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD).unwrap(); - let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), AAD); + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ).unwrap(); + let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), AAD, SEQ); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } @@ -440,8 +436,8 @@ mod tests { let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); let ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aes256Gcm, &key, AAD).unwrap(); - let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), AAD); + encrypt_record(&plaintext, EncryptionAlgorithm::Aes256Gcm, &key, AAD, SEQ).unwrap(); + let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), AAD, SEQ); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } @@ -450,27 +446,49 @@ mod tests { let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); let ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD).unwrap(); - // Truncate to 3 bytes — too short to contain nonce+tag. + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ).unwrap(); let truncated = &ciphertext[..3]; - let result = decrypt_record(truncated, &key, AAD); + let result = decrypt_record(truncated, &key, AAD, SEQ); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } #[test] - fn unknown_first_byte_passthrough() { - // First byte 0x00 is not a known alg_id. + fn unknown_first_byte_fails() { let body = b"\x00some opaque bytes"; let key = make_key_fn(); - let result = decrypt_record(body, &key, AAD).unwrap(); - assert!(result.is_none()); + let result = decrypt_record(body, &key, AAD, SEQ); + assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } #[test] - fn empty_body_passthrough() { + fn empty_body_fails() { let key = make_key_fn(); - let result = decrypt_record(b"", &key, AAD).unwrap(); - assert!(result.is_none()); + let result = decrypt_record(b"", &key, AAD, SEQ); + assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); + } + + #[test] + fn alg_id_flip_detected() { + let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); + let key = make_key_fn(); + let mut ciphertext = + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ) + .unwrap() + .to_vec(); + assert_eq!(ciphertext[0], ALG_ID_AEGIS256); + ciphertext[0] = ALG_ID_AES256GCM; + let result = decrypt_record(&ciphertext, &key, AAD, SEQ); + assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); + } + + #[test] + fn wrong_seq_num_fails() { + let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); + let key = make_key_fn(); + let ciphertext = + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, 5).unwrap(); + let result = decrypt_record(&ciphertext, &key, AAD, 6); + assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } #[test] @@ -589,13 +607,28 @@ mod tests { } #[test] - fn check_directive_no_stream_encryption() { + fn check_directive_key_on_plaintext_stream_rejected() { let key = make_key_fn(); let directive = EncryptionDirective::Key { alg: EncryptionAlgorithm::Aegis256, key, }; let result = check_encryption_directive(None, Some(&directive)); + assert!(matches!( + result, + Err(EncryptionError::EncryptionNotExpected) + )); + } + + #[test] + fn check_directive_attest_on_plaintext_stream_ok() { + let result = check_encryption_directive(None, Some(&EncryptionDirective::Attest)); + assert!(result.unwrap().is_none()); + } + + #[test] + fn check_directive_none_on_plaintext_stream_ok() { + let result = check_encryption_directive(None, None); assert!(result.unwrap().is_none()); } } diff --git a/lite/src/backend/append.rs b/lite/src/backend/append.rs index b9972da4..b3c4fd07 100644 --- a/lite/src/backend/append.rs +++ b/lite/src/backend/append.rs @@ -14,7 +14,7 @@ use s2_common::{ }; use tokio::sync::oneshot; -use super::Backend; +use super::{Backend, streamer::AppendEncryption}; use crate::backend::error::{AppendError, AppendErrorInternal, StorageError}; impl Backend { @@ -23,13 +23,18 @@ impl Backend { basin: BasinName, stream: StreamName, input: AppendInput, + encryption: Option, ) -> Result { let client = self .streamer_client_with_auto_create::(&basin, &stream, |config| { config.create_stream_on_append }) .await?; - let ack = client.append_permit(input).await?.submit().await?; + let ack = client + .append_permit(input, encryption) + .await? + .submit() + .await?; Ok(ack) } @@ -38,6 +43,7 @@ impl Backend { basin: BasinName, stream: StreamName, inputs: impl Stream, + encryption: Option, ) -> Result>, AppendError> { let client = self .streamer_client_with_auto_create::(&basin, &stream, |config| { @@ -52,7 +58,7 @@ impl Backend { loop { tokio::select! { Some(input) = inputs.next(), if permit_opt.is_none() => { - permit_opt = Some(Box::pin(client.append_permit(input))); + permit_opt = Some(Box::pin(client.append_permit(input, encryption.clone()))); } Some(res) = OptionFuture::from(permit_opt.as_mut()) => { permit_opt = None; diff --git a/lite/src/backend/error.rs b/lite/src/backend/error.rs index 07578c7c..c3ec7b89 100644 --- a/lite/src/backend/error.rs +++ b/lite/src/backend/error.rs @@ -102,6 +102,8 @@ pub(super) enum AppendErrorInternal { ConditionFailed(#[from] AppendConditionFailedError), #[error(transparent)] TimestampMissing(#[from] AppendTimestampRequiredError), + #[error(transparent)] + Encryption(#[from] s2_common::encryption::EncryptionError), } #[derive(Debug, Clone, thiserror::Error)] @@ -168,6 +170,7 @@ impl From for AppendError { AppendErrorInternal::RequestDroppedError(e) => AppendError::RequestDroppedError(e), AppendErrorInternal::ConditionFailed(e) => AppendError::ConditionFailed(e), AppendErrorInternal::TimestampMissing(e) => AppendError::TimestampMissing(e), + AppendErrorInternal::Encryption(e) => AppendError::Encryption(e), } } } diff --git a/lite/src/backend/mod.rs b/lite/src/backend/mod.rs index 46851323..eaea103a 100644 --- a/lite/src/backend/mod.rs +++ b/lite/src/backend/mod.rs @@ -15,6 +15,8 @@ mod stream_id; pub use core::Backend; +pub use streamer::AppendEncryption; + pub const FOLLOWER_MAX_LAG: usize = 25; #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/lite/src/backend/read.rs b/lite/src/backend/read.rs index 425ed71a..7ad5c4ad 100644 --- a/lite/src/backend/read.rs +++ b/lite/src/backend/read.rs @@ -489,7 +489,7 @@ mod tests { fencing_token: None, }; let ack = backend - .append(basin.clone(), stream.clone(), input) + .append(basin.clone(), stream.clone(), input, None) .await .unwrap(); assert!(ack.end.seq_num > 0); @@ -655,7 +655,7 @@ mod tests { fencing_token: None, }; let ack = backend - .append(basin.clone(), stream.clone(), input) + .append(basin.clone(), stream.clone(), input, None) .await .unwrap(); delete_batch.delete(kv::stream_record_data::ser_key(stream_id, ack.start)); @@ -725,7 +725,7 @@ mod tests { fencing_token: None, }; backend - .append(basin.clone(), stream.clone(), initial_input) + .append(basin.clone(), stream.clone(), initial_input, None) .await .unwrap(); @@ -780,7 +780,10 @@ mod tests { match_seq_num: None, fencing_token: None, }; - backend.append(basin, stream, follow_input).await.unwrap(); + backend + .append(basin, stream, follow_input, None) + .await + .unwrap(); let next = session .as_mut() diff --git a/lite/src/backend/streamer.rs b/lite/src/backend/streamer.rs index 90ba047d..0517b44c 100644 --- a/lite/src/backend/streamer.rs +++ b/lite/src/backend/streamer.rs @@ -200,6 +200,23 @@ impl Streamer { .unwrap_or(self.stable_pos) } + fn sequence_and_encrypt( + &self, + input: AppendInput, + encryption: Option, + ) -> Result>, AppendErrorInternal> { + let records = self.sequence_records(input)?; + match encryption { + Some(AppendEncryption { + directive: s2_common::encryption::EncryptionDirective::Key { alg, ref key }, + ref aad, + }) => Ok(s2_common::encryption::encrypt_sequenced_records( + records, alg, key, aad, + )?), + _ => Ok(records), + } + } + fn sequence_records( &self, AppendInput { @@ -262,6 +279,7 @@ impl Streamer { fn handle_append( &mut self, input: AppendInput, + encryption: Option, session: Option, reply_tx: oneshot::Sender>, append_type: AppendType, @@ -269,7 +287,7 @@ impl Streamer { let Some(ticket) = append::admit(reply_tx, session) else { return; }; - match self.sequence_records(input) { + match self.sequence_and_encrypt(input, encryption) { Ok(sequenced_records) => { let retention = self.config.retention_policy.unwrap_or_default(); let doe_deadline = self.maybe_doe_deadline(retention.age()); @@ -410,12 +428,13 @@ impl Streamer { match msg { Message::Append { input, + encryption, session, reply_tx, append_type, } => { if self.trim_point.state.end < SeqNum::MAX { - self.handle_append(input, session, reply_tx, append_type); + self.handle_append(input, encryption, session, reply_tx, append_type); } } Message::Follow { @@ -473,9 +492,16 @@ impl Streamer { } } +#[derive(Clone)] +pub struct AppendEncryption { + pub directive: s2_common::encryption::EncryptionDirective, + pub aad: Vec, +} + enum Message { Append { input: AppendInput, + encryption: Option, session: Option, reply_tx: oneshot::Sender>, append_type: AppendType, @@ -563,6 +589,7 @@ impl StreamerClient { pub async fn append_permit( &self, input: AppendInput, + encryption: Option, ) -> Result, StreamerMissingInActionError> { let metered_size = input.records.metered_size(); metrics::observe_append_batch_size(input.records.len(), metered_size); @@ -582,6 +609,7 @@ impl StreamerClient { sema_permit, msg_tx: &self.msg_tx, input, + encryption, }) } @@ -602,7 +630,7 @@ impl StreamerClient { fencing_token: None, }; match self - .append_permit(input) + .append_permit(input, None) .await? .submit_internal(None, AppendType::Terminal) .await @@ -618,6 +646,9 @@ impl StreamerClient { } AppendErrorInternal::ConditionFailed(_) => unreachable!("unconditional write"), AppendErrorInternal::TimestampMissing(_) => unreachable!("Timestamp::MAX used"), + AppendErrorInternal::Encryption(_) => { + unreachable!("no encryption for terminal trim") + } }), } } @@ -632,11 +663,11 @@ fn timestamp_now() -> Timestamp { .expect("Milliseconds since Unix epoch fits into a u64") } -#[derive(Debug)] pub struct AppendPermit<'a> { sema_permit: SemaphorePermit<'a>, msg_tx: &'a mpsc::UnboundedSender, input: AppendInput, + encryption: Option, } impl AppendPermit<'_> { @@ -662,11 +693,13 @@ impl AppendPermit<'_> { sema_permit, msg_tx, input, + encryption, } = self; let (reply_tx, reply_rx) = oneshot::channel(); msg_tx .send(Message::Append { input, + encryption, session, reply_tx, append_type, @@ -1090,13 +1123,13 @@ mod tests { let mut follow_rx = streamer.follow_tx.subscribe(); let (tx1, mut rx1) = oneshot::channel(); - streamer.handle_append(append_input(b"p0"), None, tx1, AppendType::Regular); + streamer.handle_append(append_input(b"p0"), None, None, tx1, AppendType::Regular); let (tx2, mut rx2) = oneshot::channel(); - streamer.handle_append(append_input(b"p1"), None, tx2, AppendType::Regular); + streamer.handle_append(append_input(b"p1"), None, None, tx2, AppendType::Regular); let (tx3, mut rx3) = oneshot::channel(); - streamer.handle_append(append_input(b"p2"), None, tx3, AppendType::Regular); + streamer.handle_append(append_input(b"p2"), None, None, tx3, AppendType::Regular); let mut db_seqs = Vec::new(); while let Some(fut) = streamer.db_writes_pending.pop_front() { @@ -1183,6 +1216,7 @@ mod tests { streamer.handle_append( append_input(payload.as_bytes()), None, + None, tx, AppendType::Regular, ); diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index dfd01732..de6b8dcb 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -24,13 +24,12 @@ use s2_common::{ types::{ ValidationError, basin::BasinName, - config::EncryptionAlgorithm, stream::{ReadBatch, ReadEnd, ReadFrom, ReadSessionOutput, ReadStart, StreamName}, }, }; use crate::{ - backend::{Backend, error::ReadError}, + backend::{AppendEncryption, Backend, error::ReadError}, handlers::v1::error::ServiceError, }; @@ -44,7 +43,6 @@ pub fn router() -> axum::Router { struct EncryptionContext { aad: Vec, - stream_alg: Option, directive: Option, } @@ -64,7 +62,6 @@ impl EncryptionContext { .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; Ok(Self { aad: stream_aad(basin, stream).into_bytes(), - stream_alg: config.encryption_algorithm, directive, }) } @@ -90,16 +87,15 @@ impl EncryptionContext { }; Ok(Self { aad: stream_aad(basin, stream).into_bytes(), - stream_alg: config.encryption_algorithm, directive: checked, }) } - fn encrypt_input( - &self, - input: s2_common::types::stream::AppendInput, - ) -> Result { - encryption::encrypt_append_input(input, self.stream_alg, self.directive.as_ref(), &self.aad) + fn into_append_encryption(self) -> Option { + self.directive.map(|directive| AppendEncryption { + directive, + aad: self.aad, + }) } fn decrypt_batch(&self, batch: ReadBatch) -> Result { @@ -447,16 +443,14 @@ pub async fn append( }: AppendArgs, ) -> Result { let enc = EncryptionContext::resolve_for_append(&backend, &headers, &basin, &stream).await?; + let append_enc = enc.into_append_encryption(); match request { v1t::stream::AppendRequest::Unary { input, response_mime, } => { - let input = enc - .encrypt_input(input) - .map_err(crate::backend::error::AppendError::Encryption)?; - let ack = backend.append(basin, stream, input).await?; + let ack = backend.append(basin, stream, input, append_enc).await?; match response_mime { JsonOrProto::Json => { let ack: v1t::stream::AppendAck = ack.into(); @@ -472,22 +466,14 @@ pub async fn append( inputs, response_compression, } => { - let (err_tx, err_rx) = tokio::sync::oneshot::channel(); + let (err_tx, err_rx) = tokio::sync::oneshot::channel::(); let inputs = async_stream::stream! { tokio::pin!(inputs); let mut err_tx = Some(err_tx); while let Some(input) = inputs.next().await { match input { - Ok(input) => match enc.encrypt_input(input) { - Ok(encrypted) => yield encrypted, - Err(e) => { - if let Some(tx) = err_tx.take() { - let _ = tx.send(ServiceError::Append(e.into())); - } - break; - } - }, + Ok(input) => yield input, Err(e) => { if let Some(tx) = err_tx.take() { let _ = tx.send(e.into()); @@ -499,7 +485,7 @@ pub async fn append( }; let ack_stream = backend - .append_session(basin, stream, inputs) + .append_session(basin, stream, inputs, append_enc) .await? .map(|res| { res.map(v1t::stream::proto::AppendAck::from) diff --git a/lite/tests/backend/common.rs b/lite/tests/backend/common.rs index 79575a18..def47b60 100644 --- a/lite/tests/backend/common.rs +++ b/lite/tests/backend/common.rs @@ -143,7 +143,7 @@ pub async fn append_payloads( fencing_token: None, }; backend - .append(basin.clone(), stream.clone(), input) + .append(basin.clone(), stream.clone(), input, None) .await .expect("Failed to append payloads") } diff --git a/lite/tests/backend/control_plane/stream.rs b/lite/tests/backend/control_plane/stream.rs index c9d54339..cc1acbc6 100644 --- a/lite/tests/backend/control_plane/stream.rs +++ b/lite/tests/backend/control_plane/stream.rs @@ -260,7 +260,7 @@ async fn test_reconfigure_stream_updates_active_streamer() { match_seq_num: None, fencing_token: None, }; - let result = backend.append(basin_name, stream_name, input).await; + let result = backend.append(basin_name, stream_name, input, None).await; assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); } @@ -303,7 +303,7 @@ async fn test_create_stream_create_or_reconfigure_updates_active_streamer() { match_seq_num: None, fencing_token: None, }; - let result = backend.append(basin_name, stream_name, input).await; + let result = backend.append(basin_name, stream_name, input, None).await; assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); } @@ -430,7 +430,7 @@ async fn test_delete_stream_blocks_data_operations() { fencing_token: None, }; let append_result = backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await; assert!(matches!( append_result, diff --git a/lite/tests/backend/data_plane/append.rs b/lite/tests/backend/data_plane/append.rs index 5244c6b1..e7c079d7 100644 --- a/lite/tests/backend/data_plane/append.rs +++ b/lite/tests/backend/data_plane/append.rs @@ -66,7 +66,12 @@ async fn test_append_fencing_token_conditions() { }; let ack = backend - .append(basin_name.clone(), stream_name.clone(), matching_input) + .append( + basin_name.clone(), + stream_name.clone(), + matching_input, + None, + ) .await .expect("Expected append to succeed with matching fencing token"); @@ -85,7 +90,7 @@ async fn test_append_fencing_token_conditions() { }; let command_ack = backend - .append(basin_name.clone(), stream_name.clone(), command_input) + .append(basin_name.clone(), stream_name.clone(), command_input, None) .await .expect("Expected fencing command to succeed"); @@ -99,7 +104,12 @@ async fn test_append_fencing_token_conditions() { }; let result = backend - .append(basin_name.clone(), stream_name.clone(), mismatched_input) + .append( + basin_name.clone(), + stream_name.clone(), + mismatched_input, + None, + ) .await; let Err(AppendError::ConditionFailed(AppendConditionFailedError::FencingTokenMismatch { @@ -120,7 +130,7 @@ async fn test_append_fencing_token_conditions() { }; let refreshed_ack = backend - .append(basin_name, stream_name, refreshed_input) + .append(basin_name, stream_name, refreshed_input, None) .await .expect("Expected append to succeed with updated fencing token"); @@ -148,7 +158,12 @@ async fn test_append_requires_timestamp() { }; let result = backend - .append(basin_name.clone(), stream_name.clone(), missing_timestamp) + .append( + basin_name.clone(), + stream_name.clone(), + missing_timestamp, + None, + ) .await; assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); @@ -163,7 +178,7 @@ async fn test_append_requires_timestamp() { }; let ack = backend - .append(basin_name, stream_name, with_timestamp) + .append(basin_name, stream_name, with_timestamp, None) .await .expect("Expected append to succeed when timestamp is provided"); @@ -183,7 +198,7 @@ async fn test_append_with_seq_num_match() { }; let ack = backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append with matching seq_num"); @@ -196,7 +211,7 @@ async fn test_append_with_seq_num_match() { }; let ack2 = backend - .append(basin_name.clone(), stream_name.clone(), input2) + .append(basin_name.clone(), stream_name.clone(), input2, None) .await .expect("Failed to append with matching seq_num"); @@ -219,7 +234,7 @@ async fn test_append_with_seq_num_mismatch() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append first record"); @@ -230,7 +245,7 @@ async fn test_append_with_seq_num_mismatch() { }; let result = backend - .append(basin_name.clone(), stream_name.clone(), input2) + .append(basin_name.clone(), stream_name.clone(), input2, None) .await; assert!(matches!( @@ -270,7 +285,7 @@ async fn test_append_session_basic() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs) + .append_session(basin_name.clone(), stream_name.clone(), inputs, None) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -320,7 +335,7 @@ async fn test_append_session_auto_create_stream() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs) + .append_session(basin_name.clone(), stream_name.clone(), inputs, None) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -355,7 +370,7 @@ async fn test_append_session_empty() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs) + .append_session(basin_name.clone(), stream_name.clone(), inputs, None) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -401,7 +416,7 @@ async fn test_append_session_multiple_records_per_batch() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs) + .append_session(basin_name.clone(), stream_name.clone(), inputs, None) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -471,7 +486,7 @@ async fn test_append_session_with_seq_num_conditions() { ]); let session = backend - .append_session(basin_name.clone(), stream_name.clone(), inputs) + .append_session(basin_name.clone(), stream_name.clone(), inputs, None) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -509,7 +524,7 @@ async fn test_append_session_seq_num_mismatch() { }]); let session = backend - .append_session(basin_name, stream_name, inputs) + .append_session(basin_name, stream_name, inputs, None) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -543,7 +558,7 @@ async fn test_append_session_with_fencing_token() { ]); let session = backend - .append_session(basin_name, stream_name, inputs) + .append_session(basin_name, stream_name, inputs, None) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -583,7 +598,7 @@ async fn test_append_session_large_batches() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs) + .append_session(basin_name.clone(), stream_name.clone(), inputs, None) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -628,7 +643,7 @@ async fn test_append_session_pipeline_preserves_ack_tail_and_read_order() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs) + .append_session(basin_name.clone(), stream_name.clone(), inputs, None) .await .expect("Failed to create append session"); tokio::pin!(session); diff --git a/lite/tests/backend/data_plane/auto_create.rs b/lite/tests/backend/data_plane/auto_create.rs index 5a0ca5d3..2ceee8ca 100644 --- a/lite/tests/backend/data_plane/auto_create.rs +++ b/lite/tests/backend/data_plane/auto_create.rs @@ -104,7 +104,7 @@ async fn test_auto_create_disabled_append_fails() { fencing_token: None, }; - let result = backend.append(basin_name, stream_name, input).await; + let result = backend.append(basin_name, stream_name, input, None).await; assert!(matches!(result, Err(AppendError::StreamNotFound(_)))); } @@ -177,7 +177,7 @@ async fn test_auto_create_race_condition_append() { }; for _ in 0..5 { match backend - .append(basin_name.clone(), stream_name.clone(), input.clone()) + .append(basin_name.clone(), stream_name.clone(), input.clone(), None) .await { Ok(ack) => return Ok(ack), @@ -189,7 +189,7 @@ async fn test_auto_create_race_condition_append() { Err(e) => return Err(e), } } - backend.append(basin_name, stream_name, input).await + backend.append(basin_name, stream_name, input, None).await }); handles.push(handle); } diff --git a/lite/tests/backend/data_plane/mixed.rs b/lite/tests/backend/data_plane/mixed.rs index c4d566ad..f8945bba 100644 --- a/lite/tests/backend/data_plane/mixed.rs +++ b/lite/tests/backend/data_plane/mixed.rs @@ -39,7 +39,7 @@ async fn test_operations_on_nonexistent_basin() { fencing_token: None, }; let append_result = backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await; assert!(matches!(append_result, Err(AppendError::BasinNotFound(_)))); @@ -70,7 +70,7 @@ async fn test_concurrent_appends_to_same_stream() { match_seq_num: None, fencing_token: None, }; - backend.append(basin_name, stream_name, input).await + backend.append(basin_name, stream_name, input, None).await }); handles.push(handle); } diff --git a/lite/tests/backend/data_plane/read.rs b/lite/tests/backend/data_plane/read.rs index 0f6da1f7..ee759b22 100644 --- a/lite/tests/backend/data_plane/read.rs +++ b/lite/tests/backend/data_plane/read.rs @@ -168,7 +168,7 @@ async fn test_read_at_tail_without_follow_returns_unwritten() { fencing_token: None, }; let ack = backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("append"); @@ -270,7 +270,7 @@ async fn test_read_timestamp_range() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append records with timestamps"); @@ -336,7 +336,7 @@ async fn test_read_from_timestamp_includes_duplicate_timestamps() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append duplicate timestamp records"); @@ -542,7 +542,7 @@ async fn test_read_until_timestamp_basic() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append timestamped records"); @@ -602,7 +602,7 @@ async fn test_read_until_timestamp_exact_boundary() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append timestamped records"); @@ -651,7 +651,7 @@ async fn test_read_until_timestamp_before_all_records() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append timestamped records"); @@ -693,7 +693,7 @@ async fn test_read_until_timestamp_after_all_records() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append timestamped records"); @@ -749,7 +749,7 @@ async fn test_read_until_with_count_limit_count_wins() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append timestamped records"); @@ -799,7 +799,7 @@ async fn test_read_until_with_count_limit_timestamp_wins() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append timestamped records"); @@ -857,7 +857,7 @@ async fn test_read_until_with_bytes_limit_bytes_wins() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append timestamped records"); @@ -910,7 +910,7 @@ async fn test_read_until_with_bytes_limit_timestamp_wins() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append timestamped records"); @@ -964,7 +964,7 @@ async fn test_read_timestamp_range_with_from_and_until() { }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append timestamped records"); diff --git a/lite/tests/backend/data_plane/read_follow.rs b/lite/tests/backend/data_plane/read_follow.rs index 0d954e1f..04fcfaff 100644 --- a/lite/tests/backend/data_plane/read_follow.rs +++ b/lite/tests/backend/data_plane/read_follow.rs @@ -524,7 +524,7 @@ async fn test_follow_mode_with_timestamp_until() { fencing_token: None, }; backend - .append(basin_name.clone(), stream_name.clone(), input) + .append(basin_name.clone(), stream_name.clone(), input, None) .await .expect("Failed to append initial record"); @@ -557,7 +557,7 @@ async fn test_follow_mode_with_timestamp_until() { fencing_token: None, }; backend_clone - .append(basin_clone.clone(), stream_clone.clone(), input) + .append(basin_clone.clone(), stream_clone.clone(), input, None) .await .unwrap(); @@ -569,7 +569,7 @@ async fn test_follow_mode_with_timestamp_until() { fencing_token: None, }; backend_clone - .append(basin_clone, stream_clone, input) + .append(basin_clone, stream_clone, input, None) .await .unwrap(); }); From 17360b4bfc111c73519f120adda12c32879519a8 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 20 Mar 2026 21:32:32 +0530 Subject: [PATCH 06/42] . --- common/src/encryption.rs | 103 ++++++++++++++++++++++++++++++--------- 1 file changed, 81 insertions(+), 22 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index bf74aa45..4f01187c 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -7,9 +7,13 @@ //! ## Wire format (body of stored EnvelopeRecord) //! //! ```text -//! [alg_id: 1 byte] [nonce] [ciphertext] [tag] +//! [version: 1 byte] [alg_id: 1 byte] [nonce] [ciphertext] [tag] //! ``` //! +//! | version | Description | +//! |---------|--------------------------------------------------| +//! | 0x01 | Initial versioned format. AAD = base ‖ alg ‖ seq_num_le | +//! //! | alg_id | Algorithm | Nonce | Tag | //! |--------|-------------|--------|------| //! | 0x01 | AEGIS-256 | 32 B | 32 B | @@ -30,6 +34,10 @@ pub use crate::types::config::EncryptionAlgorithm; pub const S2_ENCRYPTION_HEADER: &str = "s2-encryption"; +/// Ciphertext envelope version. Stored as the first byte of the encrypted record body. +/// Authenticated via AAD so tampering is detected. +const CIPHERTEXT_V1: u8 = 0x01; + const ALG_ID_AEGIS256: u8 = 0x01; const ALG_ID_AES256GCM: u8 = 0x02; @@ -83,6 +91,8 @@ pub enum EncryptionError { EncryptionRequired(EncryptionAlgorithm), #[error("Encryption key provided but stream is plaintext")] EncryptionNotExpected, + #[error("Unsupported ciphertext version: {0:#04x}")] + UnsupportedVersion(u8), #[error("Decryption failed")] DecryptionFailed, #[error("Record encoding error: {0}")] @@ -195,15 +205,15 @@ pub fn decode_record_plaintext(bytes: Bytes) -> Result<(Vec
, Bytes), Enc .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) } -/// Build the effective AAD by appending the algorithm ID byte and seq_num to the -/// caller-provided base AAD. This binds the algorithm tag and stream position into -/// the AEAD authentication, so flipping the stored alg_id byte or reordering records -/// within a stream is detected as an authentication failure. +/// Build the effective AAD for V1 envelope format. The alg_id and seq_num are mixed +/// into the AAD so the AEAD tag binds the ciphertext to its algorithm and stream +/// position. The version byte is not included -- it's already gated by the dispatch +/// in `decrypt_record`, so a version flip is caught before AAD construction. /// -/// Layout: `[base_aad | alg_id: 1 byte | seq_num: 8 bytes LE]` -fn effective_aad(aad: &[u8], alg_id: u8, seq_num: crate::record::SeqNum) -> Vec { - let mut buf = Vec::with_capacity(aad.len() + 1 + 8); - buf.extend_from_slice(aad); +/// Layout: `[base_aad | alg_id | seq_num: 8 bytes LE]` +fn effective_aad_v1(base: &[u8], alg_id: u8, seq_num: crate::record::SeqNum) -> Vec { + let mut buf = Vec::with_capacity(base.len() + 1 + 8); + buf.extend_from_slice(base); buf.push(alg_id); buf.extend_from_slice(&seq_num.to_le_bytes()); buf @@ -218,15 +228,16 @@ pub fn encrypt_record( ) -> Result { match alg { EncryptionAlgorithm::Aegis256 => { - let full_aad = effective_aad(aad, ALG_ID_AEGIS256, seq_num); + let full_aad = effective_aad_v1(aad, ALG_ID_AEGIS256, seq_num); let nonce: [u8; NONCE_BYTES_AEGIS256] = random(); let (ciphertext, tag) = Aegis256::::new(&key.expose_secret().0, &nonce) .encrypt(plaintext, &full_aad); let mut out = BytesMut::with_capacity( - 1 + NONCE_BYTES_AEGIS256 + ciphertext.len() + TAG_BYTES_AEGIS256, + 2 + NONCE_BYTES_AEGIS256 + ciphertext.len() + TAG_BYTES_AEGIS256, ); + out.put_u8(CIPHERTEXT_V1); out.put_u8(ALG_ID_AEGIS256); out.put_slice(&nonce); out.put_slice(&ciphertext); @@ -234,7 +245,7 @@ pub fn encrypt_record( Ok(out.freeze()) } EncryptionAlgorithm::Aes256Gcm => { - let full_aad = effective_aad(aad, ALG_ID_AES256GCM, seq_num); + let full_aad = effective_aad_v1(aad, ALG_ID_AES256GCM, seq_num); let nonce: [u8; NONCE_BYTES_AES256GCM] = random(); let cipher = Aes256Gcm::new_from_slice(&key.expose_secret().0).map_err(|_| { EncryptionError::EncodingFailed("invalid AES key length".to_owned()) @@ -251,7 +262,8 @@ pub fn encrypt_record( .map_err(|_| EncryptionError::DecryptionFailed)?; let mut out = - BytesMut::with_capacity(1 + NONCE_BYTES_AES256GCM + ciphertext_with_tag.len()); + BytesMut::with_capacity(2 + NONCE_BYTES_AES256GCM + ciphertext_with_tag.len()); + out.put_u8(CIPHERTEXT_V1); out.put_u8(ALG_ID_AES256GCM); out.put_slice(&nonce); out.put_slice(&ciphertext_with_tag); @@ -268,16 +280,31 @@ pub fn decrypt_record( key: &EncryptionKey, aad: &[u8], seq_num: crate::record::SeqNum, +) -> Result { + let (&version, after_version) = body + .split_first() + .ok_or(EncryptionError::DecryptionFailed)?; + + match version { + CIPHERTEXT_V1 => decrypt_record_v1(after_version, key, aad, seq_num), + v => Err(EncryptionError::UnsupportedVersion(v)), + } +} + +fn decrypt_record_v1( + body: &[u8], + key: &EncryptionKey, + aad: &[u8], + seq_num: crate::record::SeqNum, ) -> Result { let (&alg_id, rest) = body .split_first() .ok_or(EncryptionError::DecryptionFailed)?; - let full_aad = effective_aad(aad, alg_id, seq_num); + let full_aad = effective_aad_v1(aad, alg_id, seq_num); match alg_id { ALG_ID_AEGIS256 => { - // Layout after alg_id: [nonce:32][ciphertext:n][tag:32] if rest.len() < NONCE_BYTES_AEGIS256 + TAG_BYTES_AEGIS256 { return Err(EncryptionError::DecryptionFailed); } @@ -294,7 +321,6 @@ pub fn decrypt_record( Ok(Bytes::from(plaintext)) } ALG_ID_AES256GCM => { - // Layout after alg_id: [nonce:12][ciphertext+tag:n+16] if rest.len() < NONCE_BYTES_AES256GCM + TAG_BYTES_AES256GCM { return Err(EncryptionError::DecryptionFailed); } @@ -447,17 +473,21 @@ mod tests { let key = make_key_fn(); let ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ).unwrap(); - let truncated = &ciphertext[..3]; + // Truncate to 4 bytes -- version + alg_id + 2 nonce bytes, too short. + let truncated = &ciphertext[..4]; let result = decrypt_record(truncated, &key, AAD, SEQ); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } #[test] - fn unknown_first_byte_fails() { - let body = b"\x00some opaque bytes"; + fn unsupported_version_fails() { let key = make_key_fn(); + let body = b"\xFFsome opaque bytes"; let result = decrypt_record(body, &key, AAD, SEQ); - assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); + assert!(matches!( + result, + Err(EncryptionError::UnsupportedVersion(0xFF)) + )); } #[test] @@ -467,6 +497,16 @@ mod tests { assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } + #[test] + fn version_byte_present() { + let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); + let key = make_key_fn(); + let ciphertext = + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ).unwrap(); + assert_eq!(ciphertext[0], CIPHERTEXT_V1); + assert_eq!(ciphertext[1], ALG_ID_AEGIS256); + } + #[test] fn alg_id_flip_detected() { let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); @@ -475,12 +515,31 @@ mod tests { encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ) .unwrap() .to_vec(); - assert_eq!(ciphertext[0], ALG_ID_AEGIS256); - ciphertext[0] = ALG_ID_AES256GCM; + assert_eq!(ciphertext[0], CIPHERTEXT_V1); + assert_eq!(ciphertext[1], ALG_ID_AEGIS256); + // Flip alg_id (byte 1), keep version intact. + ciphertext[1] = ALG_ID_AES256GCM; let result = decrypt_record(&ciphertext, &key, AAD, SEQ); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } + #[test] + fn version_flip_detected() { + let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); + let key = make_key_fn(); + let mut ciphertext = + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ) + .unwrap() + .to_vec(); + // Flip version byte to a hypothetical v2. + ciphertext[0] = 0x02; + let result = decrypt_record(&ciphertext, &key, AAD, SEQ); + assert!(matches!( + result, + Err(EncryptionError::UnsupportedVersion(0x02)) + )); + } + #[test] fn wrong_seq_num_fails() { let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); From 553cf51fd79605616122546a36f013706ccbb00a Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Sun, 22 Mar 2026 23:22:33 +0530 Subject: [PATCH 07/42] . --- Cargo.lock | 2 + api/src/v1/config.rs | 73 ----- cli/Cargo.toml | 1 + cli/src/cli.rs | 34 ++- cli/src/config.rs | 9 +- cli/src/error.rs | 4 + cli/src/main.rs | 64 ++++- cli/src/tui/app.rs | 12 + cli/src/tui/mod.rs | 2 +- cli/src/types.rs | 27 ++ common/src/encryption.rs | 137 ++------- common/src/types/config.rs | 37 +-- lite/Cargo.toml | 1 + lite/src/backend/core.rs | 2 - lite/src/backend/read.rs | 4 - lite/src/backend/streams.rs | 32 +-- lite/src/handlers/v1/records.rs | 40 +-- lite/src/handlers/v1/streams.rs | 21 +- lite/src/init.rs | 1 - lite/tests/backend/data_plane/encryption.rs | 301 ++++++++++++++++++++ lite/tests/backend/data_plane/mod.rs | 1 + sdk/src/api.rs | 35 ++- sdk/src/types.rs | 104 ++++--- 23 files changed, 562 insertions(+), 382 deletions(-) create mode 100644 lite/tests/backend/data_plane/encryption.rs diff --git a/Cargo.lock b/Cargo.lock index bdc85fc0..565510dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4687,6 +4687,7 @@ dependencies = [ "dirs", "eyre", "futures", + "hex", "http 1.4.0", "humantime", "indicatif", @@ -4783,6 +4784,7 @@ dependencies = [ "s2-api", "s2-common", "schemars", + "secrecy", "serde", "serde_json", "slatedb", diff --git a/api/src/v1/config.rs b/api/src/v1/config.rs index 8d891cd7..8a9ec38c 100644 --- a/api/src/v1/config.rs +++ b/api/src/v1/config.rs @@ -269,10 +269,6 @@ pub struct StreamConfig { /// Delete-on-empty configuration. #[serde(default)] pub delete_on_empty: Option, - /// Encryption algorithm. `"aegis-256"` | `"aes-256-gcm"` | absent (plaintext). - /// Immutable after stream creation. - #[serde(skip_serializing_if = "Option::is_none")] - pub encryption_algorithm: Option, } impl StreamConfig { @@ -282,7 +278,6 @@ impl StreamConfig { retention_policy, timestamping, delete_on_empty, - encryption_algorithm, } = config; let config = StreamConfig { @@ -290,9 +285,6 @@ impl StreamConfig { retention_policy: retention_policy.map(Into::into), timestamping: TimestampingConfig::to_opt(timestamping), delete_on_empty: DeleteOnEmptyConfig::to_opt(delete_on_empty), - encryption_algorithm: encryption_algorithm - .filter(|alg| *alg != types::config::EncryptionAlgorithm::None) - .map(|alg| alg.as_api_str().to_owned()), }; if config == Self::default() { None @@ -309,7 +301,6 @@ impl From for StreamConfig { retention_policy, timestamping, delete_on_empty, - encryption_algorithm, } = value; Self { @@ -317,7 +308,6 @@ impl From for StreamConfig { retention_policy: Some(retention_policy.into()), timestamping: Some(timestamping.into()), delete_on_empty: Some(delete_on_empty.into()), - encryption_algorithm: encryption_algorithm.map(|alg| alg.as_api_str().to_owned()), } } } @@ -331,7 +321,6 @@ impl TryFrom for types::config::OptionalStreamConfig { retention_policy, timestamping, delete_on_empty, - encryption_algorithm, } = value; let retention_policy = match retention_policy { @@ -339,21 +328,11 @@ impl TryFrom for types::config::OptionalStreamConfig { Some(policy) => Some(policy.try_into()?), }; - let encryption_algorithm = encryption_algorithm - .as_deref() - .map(|s| { - types::config::EncryptionAlgorithm::parse_api_str(s).ok_or_else(|| { - types::ValidationError(format!("unknown encryption algorithm: {s:?}")) - }) - }) - .transpose()?; - Ok(Self { storage_class: storage_class.map(Into::into), retention_policy, timestamping: timestamping.map(Into::into).unwrap_or_default(), delete_on_empty: delete_on_empty.map(Into::into).unwrap_or_default(), - encryption_algorithm, }) } } @@ -379,28 +358,6 @@ pub struct StreamReconfiguration { #[serde(default, skip_serializing_if = "Maybe::is_unspecified")] #[cfg_attr(feature = "utoipa", schema(value_type = Option))] pub delete_on_empty: Maybe>, - /// Encryption algorithm for the stream. Only used during stream creation; ignored on - /// reconfigure (encryption is immutable after creation). - /// `"aegis-256"` | `"aes-256-gcm"` | absent (plaintext). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub encryption_algorithm: Option, -} - -impl StreamReconfiguration { - /// Extract and validate the encryption algorithm, consuming the field. - pub fn take_encryption_algorithm( - &mut self, - ) -> Result, types::ValidationError> { - self.encryption_algorithm - .take() - .as_deref() - .map(|s| { - types::config::EncryptionAlgorithm::parse_api_str(s).ok_or_else(|| { - types::ValidationError(format!("unknown encryption algorithm: {s:?}")) - }) - }) - .transpose() - } } impl TryFrom for types::config::StreamReconfiguration { @@ -412,7 +369,6 @@ impl TryFrom for types::config::StreamReconfiguration { retention_policy, timestamping, delete_on_empty, - encryption_algorithm: _, // immutable; not part of StreamReconfiguration } = value; Ok(Self { @@ -438,7 +394,6 @@ impl From for StreamReconfiguration { retention_policy: retention_policy.map_opt(Into::into), timestamping: timestamping.map_opt(Into::into), delete_on_empty: delete_on_empty.map_opt(Into::into), - encryption_algorithm: None, } } } @@ -457,11 +412,6 @@ pub struct BasinConfig { #[serde(default)] #[cfg_attr(feature = "utoipa", schema(default = false))] pub create_stream_on_read: bool, - /// Allowlist of encryption algorithms permitted for streams in this basin. - /// `"none"` = plaintext allowed; `"aegis-256"` | `"aes-256-gcm"` = algorithm allowed. - /// Empty = all allowed (including plaintext). - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub allowed_encryption_algorithms: Vec, } impl TryFrom for types::config::BasinConfig { @@ -472,20 +422,8 @@ impl TryFrom for types::config::BasinConfig { default_stream_config, create_stream_on_append, create_stream_on_read, - allowed_encryption_algorithms, } = value; - let parsed_allowed_encryption_algorithms = allowed_encryption_algorithms - .iter() - .map(|s| { - types::config::EncryptionAlgorithm::parse_api_str(s).ok_or_else(|| { - types::ValidationError(format!( - "unknown encryption algorithm in allowed_encryption_algorithms: {s:?}" - )) - }) - }) - .collect::, _>>()?; - Ok(Self { default_stream_config: match default_stream_config { Some(config) => config.try_into()?, @@ -493,7 +431,6 @@ impl TryFrom for types::config::BasinConfig { }, create_stream_on_append, create_stream_on_read, - allowed_encryption_algorithms: parsed_allowed_encryption_algorithms, }) } } @@ -504,17 +441,12 @@ impl From for BasinConfig { default_stream_config, create_stream_on_append, create_stream_on_read, - allowed_encryption_algorithms, } = value; Self { default_stream_config: StreamConfig::to_opt(default_stream_config), create_stream_on_append, create_stream_on_read, - allowed_encryption_algorithms: allowed_encryption_algorithms - .into_iter() - .map(|alg| alg.as_api_str().to_owned()) - .collect(), } } } @@ -621,7 +553,6 @@ mod tests { retention_policy, timestamping, delete_on_empty, - encryption_algorithm: None, }, ) } @@ -638,7 +569,6 @@ mod tests { default_stream_config, create_stream_on_append, create_stream_on_read, - allowed_encryption_algorithms: vec![], } }, ) @@ -668,7 +598,6 @@ mod tests { retention_policy, timestamping, delete_on_empty, - encryption_algorithm: None, } }, ) @@ -733,7 +662,6 @@ mod tests { delete_on_empty: types::config::OptionalDeleteOnEmptyConfig { min_age: doe.map(Duration::from_secs), }, - encryption_algorithm: None, } }) } @@ -921,7 +849,6 @@ mod tests { }, create_stream_on_append: base_on_append, create_stream_on_read: base_on_read, - allowed_encryption_algorithms: vec![], }; let reconfig = types::config::BasinReconfiguration::default(); diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 9b54d5c8..4b2b1021 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -27,6 +27,7 @@ crossterm = "0.29" dirs = { workspace = true } eyre = { workspace = true } futures = { workspace = true } +hex = { workspace = true } http = { workspace = true } humantime = { workspace = true } indicatif = { workspace = true } diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 05e181c9..2c2b10c5 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -12,7 +12,7 @@ use crate::{ parse_records_output_source, }, types::{ - AccessTokenMatcher, BasinConfig, BasinMatcher, Interval, Operation, + AccessTokenMatcher, BasinConfig, BasinMatcher, EncryptionAlgorithm, Interval, Operation, PermittedOperationGroups, S2BasinAndMaybeStreamUri, S2BasinAndStreamUri, S2BasinUri, StorageClass, StreamConfig, StreamMatcher, }, @@ -464,6 +464,22 @@ pub struct AppendArgs { /// How long to wait for more records before flushing a batch. #[arg(long, default_value = "5ms")] pub linger: humantime::Duration, + + /// Hex-encoded 32-byte encryption key. Prefer S2_ENCRYPTION_KEY env var. + #[arg(long, env = "S2_ENCRYPTION_KEY", hide_env_values = true)] + pub encryption_key: Option, + + /// Read encryption key from file (first line, trimmed). + #[arg(long, conflicts_with = "encryption_key")] + pub encryption_key_file: Option, + + /// Encryption algorithm (default: aegis-256). + #[arg(long, value_enum)] + pub encryption_algorithm: Option, + + /// Attest client-side encryption (mutually exclusive with key/algorithm). + #[arg(long, conflicts_with_all = ["encryption_key", "encryption_key_file", "encryption_algorithm"])] + pub encryption_attest: bool, } #[derive(Args, Debug)] @@ -515,6 +531,22 @@ pub struct ReadArgs { /// Use "-" to write to stdout. #[arg(short = 'o', long, value_parser = parse_records_output_source, default_value = "-")] pub output: RecordsOut, + + /// Hex-encoded 32-byte encryption key. Prefer S2_ENCRYPTION_KEY env var. + #[arg(long, env = "S2_ENCRYPTION_KEY", hide_env_values = true)] + pub encryption_key: Option, + + /// Read encryption key from file (first line, trimmed). + #[arg(long, conflicts_with = "encryption_key")] + pub encryption_key_file: Option, + + /// Encryption algorithm (default: aegis-256). + #[arg(long, value_enum)] + pub encryption_algorithm: Option, + + /// Attest client-side encryption (mutually exclusive with key/algorithm). + #[arg(long, conflicts_with_all = ["encryption_key", "encryption_key_file", "encryption_algorithm"])] + pub encryption_attest: bool, } #[derive(Args, Debug)] diff --git a/cli/src/config.rs b/cli/src/config.rs index 0c575ab7..b7abb960 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -163,7 +163,10 @@ pub fn unset_config_value(key: ConfigKey) -> Result { save_cli_config(&config) } -pub fn sdk_config(config: &CliConfig) -> Result { +pub fn sdk_config( + config: &CliConfig, + encryption: Option, +) -> Result { let access_token = config .access_token .as_ref() @@ -210,6 +213,10 @@ pub fn sdk_config(config: &CliConfig) -> Result { sdk_config = sdk_config.with_insecure_skip_cert_verification(true); } + if let Some(enc) = encryption { + sdk_config = sdk_config.with_encryption(enc); + } + Ok(sdk_config) } diff --git a/cli/src/error.rs b/cli/src/error.rs index a7be6d7f..41c2db9b 100644 --- a/cli/src/error.rs +++ b/cli/src/error.rs @@ -66,6 +66,10 @@ pub enum CliError { ))] OperationWithTokenSource(OpKind, #[source] S2Error, TokenSource), + #[error("Invalid encryption key: {0}")] + #[diagnostic(help("Key must be exactly 64 hex characters (32 bytes)."))] + InvalidEncryptionKey(String), + #[error("S2 Lite server error: {0}")] #[diagnostic(help("{}", HELP))] LiteServer(String), diff --git a/cli/src/main.rs b/cli/src/main.rs index a6e2d77f..4f7edccd 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -32,9 +32,9 @@ use record_format::{ use s2_sdk::{ S2, types::{ - AppendRetryPolicy, CreateStreamInput, DeleteOnEmptyConfig, DeleteStreamInput, MeteredBytes, - Metric, RetentionPolicy, RetryConfig, StreamConfig as SdkStreamConfig, StreamName, - TimestampingConfig, TimestampingMode, + AppendRetryPolicy, CreateStreamInput, DeleteOnEmptyConfig, DeleteStreamInput, + EncryptionConfig, MeteredBytes, Metric, RetentionPolicy, RetryConfig, + StreamConfig as SdkStreamConfig, StreamName, TimestampingConfig, TimestampingMode, }, }; use strum::VariantNames; @@ -151,7 +151,8 @@ async fn run() -> Result<(), CliError> { } let cli_config = load_cli_config()?; - let sdk_config = sdk_config(&cli_config)?; + let encryption = resolve_command_encryption(&command)?; + let sdk_config = sdk_config(&cli_config, encryption)?; let s2 = S2::new(sdk_config.clone()).map_err(CliError::SdkInit)?; let token_source = access_token_source(&cli_config); let result: Result<(), CliError> = (async { @@ -796,3 +797,58 @@ fn print_metrics(metrics: &[Metric]) { } } } + +fn resolve_command_encryption(command: &Command) -> Result, CliError> { + struct EncryptionArgs<'a> { + key: &'a Option, + key_file: &'a Option, + alg: Option, + attest: bool, + } + + let args = match command { + Command::Append(a) => EncryptionArgs { + key: &a.encryption_key, + key_file: &a.encryption_key_file, + alg: a.encryption_algorithm, + attest: a.encryption_attest, + }, + Command::Read(a) => EncryptionArgs { + key: &a.encryption_key, + key_file: &a.encryption_key_file, + alg: a.encryption_algorithm, + attest: a.encryption_attest, + }, + _ => return Ok(None), + }; + + if args.attest { + return Ok(Some(EncryptionConfig::Attest)); + } + + let key = match (args.key, args.key_file) { + (Some(k), _) => k.clone(), + (_, Some(path)) => { + let contents = std::fs::read_to_string(path).map_err(|e| { + CliError::InvalidEncryptionKey(format!("cannot read key file: {e}")) + })?; + contents.lines().next().unwrap_or("").trim().to_owned() + } + _ => return Ok(None), + }; + + if key.len() != 64 { + return Err(CliError::InvalidEncryptionKey(format!( + "expected 64 hex characters, got {}", + key.len() + ))); + } + hex::decode(&key).map_err(|e| CliError::InvalidEncryptionKey(e.to_string()))?; + + let alg = args.alg.unwrap_or(types::EncryptionAlgorithm::Aegis256); + + Ok(Some(EncryptionConfig::Key { + alg: alg.into(), + key: key.into(), + })) +} diff --git a/cli/src/tui/app.rs b/cli/src/tui/app.rs index 7469462a..d3f6d411 100644 --- a/cli/src/tui/app.rs +++ b/cli/src/tui/app.rs @@ -4607,6 +4607,10 @@ impl App { until: None, format: RecordFormat::default(), output: RecordsOut::Stdout, + encryption_key: None, + encryption_key_file: None, + encryption_algorithm: None, + encryption_attest: false, }; match ops::read(&s2, &args).await { @@ -4679,6 +4683,10 @@ impl App { until: None, format: RecordFormat::default(), output: RecordsOut::Stdout, + encryption_key: None, + encryption_key_file: None, + encryption_algorithm: None, + encryption_attest: false, }; match ops::read(&s2, &args).await { @@ -4842,6 +4850,10 @@ impl App { until, format: record_format, output: output.clone(), + encryption_key: None, + encryption_key_file: None, + encryption_algorithm: None, + encryption_attest: false, }; // Open file writer if output file is specified diff --git a/cli/src/tui/mod.rs b/cli/src/tui/mod.rs index 35a5e3fb..1cc7f063 100644 --- a/cli/src/tui/mod.rs +++ b/cli/src/tui/mod.rs @@ -20,7 +20,7 @@ pub async fn run() -> Result<(), CliError> { // Load config and try to create SDK client // If access token is missing, we'll start with Setup screen instead of failing let cli_config = load_cli_config()?; - let s2 = match sdk_config(&cli_config) { + let s2 = match sdk_config(&cli_config, None) { Ok(sdk_cfg) => Some(s2_sdk::S2::new(sdk_cfg).map_err(CliError::SdkInit)?), Err(_) => None, // No access token - will show setup screen }; diff --git a/cli/src/types.rs b/cli/src/types.rs index e828ca0f..bfc22805 100644 --- a/cli/src/types.rs +++ b/cli/src/types.rs @@ -169,6 +169,33 @@ impl StreamConfig { } } +#[derive(ValueEnum, Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum EncryptionAlgorithm { + #[value(name = "aegis-256")] + Aegis256, + #[value(name = "aes-256-gcm")] + Aes256Gcm, +} + +impl From for sdk::types::EncryptionAlgorithm { + fn from(alg: EncryptionAlgorithm) -> Self { + match alg { + EncryptionAlgorithm::Aegis256 => Self::Aegis256, + EncryptionAlgorithm::Aes256Gcm => Self::Aes256Gcm, + } + } +} + +impl From for EncryptionAlgorithm { + fn from(alg: sdk::types::EncryptionAlgorithm) -> Self { + match alg { + sdk::types::EncryptionAlgorithm::Aegis256 => Self::Aegis256, + sdk::types::EncryptionAlgorithm::Aes256Gcm => Self::Aes256Gcm, + } + } +} + #[derive(ValueEnum, Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub enum StorageClass { diff --git a/common/src/encryption.rs b/common/src/encryption.rs index 4f01187c..2be63171 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -35,7 +35,8 @@ pub use crate::types::config::EncryptionAlgorithm; pub const S2_ENCRYPTION_HEADER: &str = "s2-encryption"; /// Ciphertext envelope version. Stored as the first byte of the encrypted record body. -/// Authenticated via AAD so tampering is detected. +/// Tampering is caught by the version dispatch in `decrypt_record` before AAD construction. +/// N.B. shares the value `0x01` with `ALG_ID_AEGIS256` but occupies a different byte offset. const CIPHERTEXT_V1: u8 = 0x01; const ALG_ID_AEGIS256: u8 = 0x01; @@ -69,7 +70,7 @@ fn make_key(bytes: [u8; 32]) -> EncryptionKey { /// Parsed `S2-Encryption` header directive. #[derive(Clone, Debug)] pub enum EncryptionDirective { - /// Client provides the key; server encrypts/decrypts. + /// Client provides the algorithm and key; server encrypts/decrypts. Key { alg: EncryptionAlgorithm, key: EncryptionKey, @@ -82,15 +83,6 @@ pub enum EncryptionDirective { pub enum EncryptionError { #[error("Malformed S2-Encryption header: {0}")] MalformedHeader(String), - #[error("Algorithm mismatch: stream requires {expected:?}, got {got:?}")] - AlgorithmMismatch { - expected: EncryptionAlgorithm, - got: EncryptionAlgorithm, - }, - #[error("Encryption required: stream has encryption={0:?} but no key was provided")] - EncryptionRequired(EncryptionAlgorithm), - #[error("Encryption key provided but stream is plaintext")] - EncryptionNotExpected, #[error("Unsupported ciphertext version: {0:#04x}")] UnsupportedVersion(u8), #[error("Decryption failed")] @@ -116,33 +108,26 @@ pub fn parse_s2_encryption_header( } let (alg_part, key_part) = s.split_once(';').ok_or_else(|| { - EncryptionError::MalformedHeader(format!("expected 'alg=...; key=...', got {s:?}")) + EncryptionError::MalformedHeader("expected 'alg=...; key=...'".to_owned()) })?; let alg_str = alg_part .trim() .strip_prefix("alg=") - .ok_or_else(|| { - EncryptionError::MalformedHeader(format!("expected 'alg=...', got {alg_part:?}")) - })? + .ok_or_else(|| EncryptionError::MalformedHeader("missing 'alg=' prefix".to_owned()))? .trim(); let key_hex = key_part .trim() .strip_prefix("key=") - .ok_or_else(|| { - EncryptionError::MalformedHeader(format!("expected 'key=...', got {key_part:?}")) - })? + .ok_or_else(|| EncryptionError::MalformedHeader("missing 'key=' prefix".to_owned()))? .trim(); - let alg = match EncryptionAlgorithm::parse_api_str(alg_str) { - Some(EncryptionAlgorithm::None) | None => { - return Err(EncryptionError::MalformedHeader(format!( - "unknown algorithm {alg_str:?}; expected 'aegis-256' or 'aes-256-gcm'" - ))); - } - Some(alg) => alg, - }; + let alg = EncryptionAlgorithm::parse_api_str(alg_str).ok_or_else(|| { + EncryptionError::MalformedHeader(format!( + "unknown algorithm {alg_str:?}; expected 'aegis-256' or 'aes-256-gcm'" + )) + })?; if key_hex.len() != 64 { return Err(EncryptionError::MalformedHeader(format!( @@ -151,12 +136,21 @@ pub fn parse_s2_encryption_header( ))); } - let key_bytes: Vec = hex::decode(key_hex) + let mut key_bytes: Vec = hex::decode(key_hex) .map_err(|e| EncryptionError::MalformedHeader(format!("key is not valid hex: {e}")))?; - let key_array: [u8; 32] = key_bytes - .try_into() - .map_err(|_| EncryptionError::MalformedHeader("key must be exactly 32 bytes".to_owned()))?; + let key_array: [u8; 32] = match key_bytes.as_slice().try_into() { + Ok(arr) => { + secrecy::zeroize::Zeroize::zeroize(&mut key_bytes); + arr + } + Err(_) => { + secrecy::zeroize::Zeroize::zeroize(&mut key_bytes); + return Err(EncryptionError::MalformedHeader( + "key must be exactly 32 bytes".to_owned(), + )); + } + }; Ok(Some(EncryptionDirective::Key { alg, @@ -164,32 +158,6 @@ pub fn parse_s2_encryption_header( })) } -pub fn check_encryption_directive<'a>( - stream_alg: Option, - directive: Option<&'a EncryptionDirective>, -) -> Result, EncryptionError> { - let Some(required_alg) = stream_alg else { - if matches!(directive, Some(EncryptionDirective::Key { .. })) { - return Err(EncryptionError::EncryptionNotExpected); - } - return Ok(None); - }; - - match directive { - None => Err(EncryptionError::EncryptionRequired(required_alg)), - Some(EncryptionDirective::Attest) => Ok(directive), - Some(EncryptionDirective::Key { alg, .. }) => { - if *alg != required_alg { - return Err(EncryptionError::AlgorithmMismatch { - expected: required_alg, - got: *alg, - }); - } - Ok(directive) - } - } -} - pub fn encode_record_plaintext( headers: Vec
, body: Bytes, @@ -259,7 +227,9 @@ pub fn encrypt_record( aad: &full_aad, }, ) - .map_err(|_| EncryptionError::DecryptionFailed)?; + .map_err(|_| { + EncryptionError::EncodingFailed("AES-256-GCM encryption failed".to_owned()) + })?; let mut out = BytesMut::with_capacity(2 + NONCE_BYTES_AES256GCM + ciphertext_with_tag.len()); @@ -269,9 +239,6 @@ pub fn encrypt_record( out.put_slice(&ciphertext_with_tag); Ok(out.freeze()) } - EncryptionAlgorithm::None => Err(EncryptionError::EncodingFailed( - "cannot encrypt with None algorithm".to_owned(), - )), } } @@ -640,54 +607,4 @@ mod tests { let result = parse_s2_encryption_header(&headers); assert!(matches!(result, Err(EncryptionError::MalformedHeader(_)))); } - - #[test] - fn check_directive_alg_mismatch() { - let key = make_key_fn(); - let directive = EncryptionDirective::Key { - alg: EncryptionAlgorithm::Aes256Gcm, - key, - }; - let result = - check_encryption_directive(Some(EncryptionAlgorithm::Aegis256), Some(&directive)); - assert!(matches!( - result, - Err(EncryptionError::AlgorithmMismatch { .. }) - )); - } - - #[test] - fn check_directive_required_but_absent() { - let result = check_encryption_directive(Some(EncryptionAlgorithm::Aegis256), None); - assert!(matches!( - result, - Err(EncryptionError::EncryptionRequired(_)) - )); - } - - #[test] - fn check_directive_key_on_plaintext_stream_rejected() { - let key = make_key_fn(); - let directive = EncryptionDirective::Key { - alg: EncryptionAlgorithm::Aegis256, - key, - }; - let result = check_encryption_directive(None, Some(&directive)); - assert!(matches!( - result, - Err(EncryptionError::EncryptionNotExpected) - )); - } - - #[test] - fn check_directive_attest_on_plaintext_stream_ok() { - let result = check_encryption_directive(None, Some(&EncryptionDirective::Attest)); - assert!(result.unwrap().is_none()); - } - - #[test] - fn check_directive_none_on_plaintext_stream_ok() { - let result = check_encryption_directive(None, None); - assert!(result.unwrap().is_none()); - } } diff --git a/common/src/types/config.rs b/common/src/types/config.rs index 107639c3..686032d2 100644 --- a/common/src/types/config.rs +++ b/common/src/types/config.rs @@ -33,16 +33,9 @@ use enum_ordinalize::Ordinalize; use crate::maybe::Maybe; -/// Encryption algorithm for stream records. -/// -/// When used in `StreamConfig.encryption_algorithm`, `None` is represented by Rust's `Option::None` -/// (meaning plaintext). When used in `BasinConfig.allowed_encryption_algorithms`, the `None` -/// variant is a sentinel that explicitly permits plaintext streams. +/// Encryption algorithm for record data, specified per-request via the `S2-Encryption` header. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EncryptionAlgorithm { - /// Sentinel: plaintext is explicitly allowed (used only in `allowed_encryption_algorithms` - /// lists). - None, Aegis256, Aes256Gcm, } @@ -51,19 +44,17 @@ impl EncryptionAlgorithm { /// Wire-format string used in JSON API and `S2-Encryption` header. pub fn as_api_str(self) -> &'static str { match self { - Self::None => "none", Self::Aegis256 => "aegis-256", Self::Aes256Gcm => "aes-256-gcm", } } - /// Parse from wire-format string. Returns `None` for unrecognised values. + /// Parse from wire-format string. pub fn parse_api_str(s: &str) -> Option { match s { - "none" => Some(Self::None), "aegis-256" => Some(Self::Aegis256), "aes-256-gcm" => Some(Self::Aes256Gcm), - _ => Option::None, + _ => None, } } } @@ -141,8 +132,6 @@ pub struct StreamConfig { pub retention_policy: RetentionPolicy, pub timestamping: TimestampingConfig, pub delete_on_empty: DeleteOnEmptyConfig, - /// Encryption algorithm for this stream. `None` = plaintext. Immutable after creation. - pub encryption_algorithm: Option, } #[derive(Debug, Clone, Default)] @@ -267,8 +256,6 @@ pub struct OptionalStreamConfig { pub retention_policy: Option, pub timestamping: OptionalTimestampingConfig, pub delete_on_empty: OptionalDeleteOnEmptyConfig, - /// Encryption algorithm. `None` = not specified (falls back to basin default or plaintext). - pub encryption_algorithm: Option, } impl OptionalStreamConfig { @@ -313,16 +300,11 @@ impl OptionalStreamConfig { let delete_on_empty = self.delete_on_empty.merge(basin_defaults.delete_on_empty); - let encryption_algorithm = self - .encryption_algorithm - .or(basin_defaults.encryption_algorithm); - StreamConfig { storage_class, retention_policy, timestamping, delete_on_empty, - encryption_algorithm, } } } @@ -334,8 +316,6 @@ impl From for StreamReconfiguration { retention_policy, timestamping, delete_on_empty, - encryption_algorithm: _, /* encryption_algorithm is immutable; not represented in - * StreamReconfiguration */ } = value; Self { @@ -354,7 +334,6 @@ impl From for StreamConfig { retention_policy, timestamping, delete_on_empty, - encryption_algorithm, } = value; Self { @@ -362,7 +341,6 @@ impl From for StreamConfig { retention_policy: retention_policy.unwrap_or_default(), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), - encryption_algorithm, } } } @@ -374,7 +352,6 @@ impl From for OptionalStreamConfig { retention_policy, timestamping, delete_on_empty, - encryption_algorithm, } = value; Self { @@ -382,7 +359,6 @@ impl From for OptionalStreamConfig { retention_policy: Some(retention_policy), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), - encryption_algorithm, } } } @@ -392,10 +368,6 @@ pub struct BasinConfig { pub default_stream_config: OptionalStreamConfig, pub create_stream_on_append: bool, pub create_stream_on_read: bool, - /// Allowlist of encryption algorithms for streams in this basin. - /// Empty = all allowed (including plaintext). Use `EncryptionAlgorithm::None` to - /// explicitly permit plaintext when the list is non-empty. - pub allowed_encryption_algorithms: Vec, } impl BasinConfig { @@ -420,8 +392,6 @@ impl BasinConfig { self.create_stream_on_read = create_stream_on_read; } - // allowed_encryption_algorithms is not reconfigurable via BasinReconfiguration (S2-ops - // managed). self } } @@ -432,7 +402,6 @@ impl From for BasinReconfiguration { default_stream_config, create_stream_on_append, create_stream_on_read, - allowed_encryption_algorithms: _, // not reconfigurable via BasinReconfiguration } = value; Self { diff --git a/lite/Cargo.toml b/lite/Cargo.toml index 58e8ee4c..1cd11fd2 100644 --- a/lite/Cargo.toml +++ b/lite/Cargo.toml @@ -63,5 +63,6 @@ utoipa = { version = "5.4", optional = true, features = ["time"] } [dev-dependencies] proptest = { workspace = true } +secrecy = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util", "time"] } uuid = { workspace = true, features = ["v4"] } diff --git a/lite/src/backend/core.rs b/lite/src/backend/core.rs index feedcaa1..c22d143e 100644 --- a/lite/src/backend/core.rs +++ b/lite/src/backend/core.rs @@ -301,7 +301,6 @@ impl Backend { basin.clone(), stream.clone(), OptionalStreamConfig::default(), - None, CreateMode::CreateOnly(None), ) .await @@ -452,7 +451,6 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), - None, CreateMode::CreateOnly(None), ) .await diff --git a/lite/src/backend/read.rs b/lite/src/backend/read.rs index 7ad5c4ad..8d990d73 100644 --- a/lite/src/backend/read.rs +++ b/lite/src/backend/read.rs @@ -468,7 +468,6 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), - None, CreateMode::CreateOnly(None), ) .await @@ -546,7 +545,6 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), - None, CreateMode::CreateOnly(None), ) .await @@ -602,7 +600,6 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), - None, CreateMode::CreateOnly(None), ) .await @@ -701,7 +698,6 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), - None, CreateMode::CreateOnly(None), ) .await diff --git a/lite/src/backend/streams.rs b/lite/src/backend/streams.rs index ba48862f..80c69ddb 100644 --- a/lite/src/backend/streams.rs +++ b/lite/src/backend/streams.rs @@ -3,7 +3,7 @@ use s2_common::{ record::StreamPosition, types::{ basin::BasinName, - config::{EncryptionAlgorithm, OptionalStreamConfig, StreamReconfiguration}, + config::{OptionalStreamConfig, StreamReconfiguration}, resources::{CreateMode, ListItemsRequestParts, Page, RequestToken}, stream::{ListStreamsRequest, StreamInfo, StreamName}, }, @@ -82,7 +82,6 @@ impl Backend { basin: BasinName, stream: StreamName, config: impl Into, - encryption: Option, mode: CreateMode, ) -> Result, CreateStreamError> { let config = config.into(); @@ -152,8 +151,7 @@ impl Backend { let (resolved, created_at) = match existing_meta_opt { Some(existing) => (existing.config.reconfigure(config), existing.created_at), None => { - let mut cfg = OptionalStreamConfig::default().reconfigure(config); - cfg.encryption_algorithm = encryption; + let cfg = OptionalStreamConfig::default().reconfigure(config); (cfg, OffsetDateTime::now_utc()) } }; @@ -161,25 +159,6 @@ impl Backend { .merge(basin_meta.config.default_stream_config) .into(); - if !is_reconfigure && !basin_meta.config.allowed_encryption_algorithms.is_empty() { - let is_allowed = match resolved.encryption_algorithm { - None => basin_meta - .config - .allowed_encryption_algorithms - .contains(&EncryptionAlgorithm::None), - Some(alg) => basin_meta - .config - .allowed_encryption_algorithms - .contains(&alg), - }; - if !is_allowed { - return Err(CreateStreamError::InvalidConfig(format!( - "encryption algorithm {:?} not permitted for this basin", - resolved.encryption_algorithm - ))); - } - } - let meta = kv::stream_meta::StreamMeta { config: resolved.clone(), created_at, @@ -310,15 +289,8 @@ impl Backend { .min_age .filter(|age| !age.is_zero()); - let original_encryption = meta.config.encryption_algorithm; meta.config = meta.config.reconfigure(reconfig); - if meta.config.encryption_algorithm != original_encryption { - return Err(ReconfigureStreamError::ImmutableField( - "encryption_algorithm", - )); - } - txn.put(&meta_key, kv::stream_meta::ser_value(&meta))?; let stream_id = StreamId::new(&basin, &stream); diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index de6b8dcb..208e0051 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -15,8 +15,7 @@ use s2_api::{ use s2_common::{ caps::RECORD_BATCH_MAX, encryption::{ - self, EncryptionDirective, EncryptionError, check_encryption_directive, - parse_s2_encryption_header, stream_aad, + self, EncryptionDirective, EncryptionError, parse_s2_encryption_header, stream_aad, }, http::extract::Header, read_extent::{CountOrBytes, ReadLimit}, @@ -47,50 +46,19 @@ struct EncryptionContext { } impl EncryptionContext { - async fn resolve_for_append( - backend: &Backend, + fn resolve( headers: &http::HeaderMap, basin: &BasinName, stream: &StreamName, ) -> Result { let directive = parse_s2_encryption_header(headers) .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; - let config = backend - .get_stream_config(basin.clone(), stream.clone()) - .await?; - check_encryption_directive(config.encryption_algorithm, directive.as_ref()) - .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; Ok(Self { aad: stream_aad(basin, stream).into_bytes(), directive, }) } - async fn resolve_for_read( - backend: &Backend, - headers: &http::HeaderMap, - basin: &BasinName, - stream: &StreamName, - ) -> Result { - let directive = parse_s2_encryption_header(headers) - .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; - let config = backend - .get_stream_config(basin.clone(), stream.clone()) - .await?; - let checked = - match check_encryption_directive(config.encryption_algorithm, directive.as_ref()) { - Ok(d) => d.cloned(), - Err(EncryptionError::EncryptionRequired(_)) => None, - Err(e) => { - return Err(ServiceError::Validation(ValidationError(e.to_string()))); - } - }; - Ok(Self { - aad: stream_aad(basin, stream).into_bytes(), - directive: checked, - }) - } - fn into_append_encryption(self) -> Option { self.directive.map(|directive| AppendEncryption { directive, @@ -247,7 +215,7 @@ pub async fn read( request, }: ReadArgs, ) -> Result { - let enc = EncryptionContext::resolve_for_read(&backend, &headers, &basin, &stream).await?; + let enc = EncryptionContext::resolve(&headers, &basin, &stream)?; let start: ReadStart = start.try_into()?; match request { @@ -442,7 +410,7 @@ pub async fn append( request, }: AppendArgs, ) -> Result { - let enc = EncryptionContext::resolve_for_append(&backend, &headers, &basin, &stream).await?; + let enc = EncryptionContext::resolve(&headers, &basin, &stream)?; let append_enc = enc.into_append_encryption(); match request { diff --git a/lite/src/handlers/v1/streams.rs b/lite/src/handlers/v1/streams.rs index 1a9e1207..13bee62c 100644 --- a/lite/src/handlers/v1/streams.rs +++ b/lite/src/handlers/v1/streams.rs @@ -121,13 +121,11 @@ pub async fn create_stream( .map(TryInto::try_into) .transpose()? .unwrap_or_default(); - let encryption = config.encryption_algorithm; let info = backend .create_stream( basin, request.stream, config, - encryption, CreateMode::CreateOnly(request_token), ) .await?; @@ -217,21 +215,12 @@ pub async fn create_or_reconfigure_stream( config: JsonOpt(config), }: CreateOrReconfigureArgs, ) -> Result<(StatusCode, Json), ServiceError> { - let (encryption, config): (_, StreamReconfiguration) = match config { - Some(mut api_config) => { - let enc = api_config.take_encryption_algorithm()?; - (enc, api_config.try_into()?) - } - None => (None, StreamReconfiguration::default()), - }; + let config: StreamReconfiguration = config + .map(TryInto::try_into) + .transpose()? + .unwrap_or_default(); let info = backend - .create_stream( - basin, - stream, - config, - encryption, - CreateMode::CreateOrReconfigure, - ) + .create_stream(basin, stream, config, CreateMode::CreateOrReconfigure) .await?; let status = if info.is_created() { StatusCode::CREATED diff --git a/lite/src/init.rs b/lite/src/init.rs index 68fa49e9..c7a1a76b 100644 --- a/lite/src/init.rs +++ b/lite/src/init.rs @@ -388,7 +388,6 @@ pub async fn apply(backend: &Backend, spec: ResourcesSpec) -> eyre::Result<()> { basin.clone(), stream.clone(), reconfiguration, - None, CreateMode::CreateOrReconfigure, ) .await diff --git a/lite/tests/backend/data_plane/encryption.rs b/lite/tests/backend/data_plane/encryption.rs new file mode 100644 index 00000000..6b8270e1 --- /dev/null +++ b/lite/tests/backend/data_plane/encryption.rs @@ -0,0 +1,301 @@ +use bytes::Bytes; +use futures::StreamExt; +use s2_common::{ + encryption::{ + EncryptionAlgorithm, EncryptionDirective, EncryptionKey, KeyBytes, decrypt_read_batch, + stream_aad, + }, + read_extent::{ReadLimit, ReadUntil}, + record::Record, + types::{ + config::OptionalStreamConfig, + stream::{AppendInput, ReadEnd, ReadFrom, ReadSessionOutput, ReadStart}, + }, +}; +use s2_lite::backend::AppendEncryption; +use secrecy::SecretBox; + +use super::common::*; + +fn test_key() -> EncryptionKey { + SecretBox::new(Box::new(KeyBytes([0x42u8; 32]))) +} + +fn test_key_2() -> EncryptionKey { + SecretBox::new(Box::new(KeyBytes([0x99u8; 32]))) +} + +fn make_append_encryption( + alg: EncryptionAlgorithm, + key: EncryptionKey, + basin: &s2_common::types::basin::BasinName, + stream: &s2_common::types::stream::StreamName, +) -> AppendEncryption { + AppendEncryption { + directive: EncryptionDirective::Key { alg, key }, + aad: stream_aad(basin, stream).into_bytes(), + } +} + +#[tokio::test] +async fn test_encrypt_append_and_decrypt_read_aegis256() { + let (backend, basin, stream) = + setup_backend_with_stream("enc-aegis", "stream", OptionalStreamConfig::default()).await; + + let enc = make_append_encryption(EncryptionAlgorithm::Aegis256, test_key(), &basin, &stream); + + let input = AppendInput { + records: create_test_record_batch(vec![ + Bytes::from_static(b"secret 1"), + Bytes::from_static(b"secret 2"), + ]), + match_seq_num: None, + fencing_token: None, + }; + + let ack = backend + .append(basin.clone(), stream.clone(), input, Some(enc)) + .await + .expect("encrypted append should succeed"); + assert_eq!(ack.start.seq_num, 0); + assert_eq!(ack.end.seq_num, 2); + + // Read back raw (encrypted) records. + let session = backend + .read( + basin.clone(), + stream.clone(), + ReadStart { + from: ReadFrom::SeqNum(0), + clamp: false, + }, + ReadEnd { + limit: ReadLimit::Count(10), + until: ReadUntil::Unbounded, + wait: None, + }, + ) + .await + .expect("read session"); + + let mut batches = Vec::new(); + tokio::pin!(session); + while let Some(output) = session.next().await { + match output.expect("read output") { + ReadSessionOutput::Batch(batch) => batches.push(batch), + ReadSessionOutput::Heartbeat(_) => {} + } + } + assert!(!batches.is_empty()); + + // Raw records should NOT match plaintext (they're encrypted). + let Record::Envelope(ref env) = batches[0].records[0].record else { + panic!("expected envelope record"); + }; + assert_ne!(env.body().as_ref(), b"secret 1"); + + // Decrypt and verify plaintext matches. + let directive = EncryptionDirective::Key { + alg: EncryptionAlgorithm::Aegis256, + key: test_key(), + }; + let aad = stream_aad(&basin, &stream).into_bytes(); + for batch in batches { + let decrypted = + decrypt_read_batch(batch, Some(&directive), &aad).expect("decryption should succeed"); + for sr in decrypted.records.iter() { + let Record::Envelope(ref env) = sr.record else { + panic!("expected envelope record"); + }; + let text = std::str::from_utf8(env.body()).expect("valid utf8"); + assert!( + text == "secret 1" || text == "secret 2", + "unexpected body: {text}" + ); + } + } +} + +#[tokio::test] +async fn test_encrypt_append_and_decrypt_read_aes256gcm() { + let (backend, basin, stream) = + setup_backend_with_stream("enc-aes", "stream", OptionalStreamConfig::default()).await; + + let enc = make_append_encryption(EncryptionAlgorithm::Aes256Gcm, test_key(), &basin, &stream); + + let input = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"aes payload")]), + match_seq_num: None, + fencing_token: None, + }; + + backend + .append(basin.clone(), stream.clone(), input, Some(enc)) + .await + .expect("encrypted append should succeed"); + + let session = backend + .read( + basin.clone(), + stream.clone(), + ReadStart { + from: ReadFrom::SeqNum(0), + clamp: false, + }, + ReadEnd { + limit: ReadLimit::Count(10), + until: ReadUntil::Unbounded, + wait: None, + }, + ) + .await + .expect("read session"); + + let directive = EncryptionDirective::Key { + alg: EncryptionAlgorithm::Aes256Gcm, + key: test_key(), + }; + let aad = stream_aad(&basin, &stream).into_bytes(); + + tokio::pin!(session); + while let Some(output) = session.next().await { + match output.expect("read output") { + ReadSessionOutput::Batch(batch) => { + let decrypted = decrypt_read_batch(batch, Some(&directive), &aad) + .expect("decryption should succeed"); + for sr in decrypted.records.iter() { + let Record::Envelope(ref env) = sr.record else { + panic!("expected envelope record"); + }; + assert_eq!(env.body().as_ref(), b"aes payload"); + } + } + ReadSessionOutput::Heartbeat(_) => {} + } + } +} + +#[tokio::test] +async fn test_wrong_key_fails_decryption() { + let (backend, basin, stream) = + setup_backend_with_stream("enc-wrongkey", "stream", OptionalStreamConfig::default()).await; + + let enc = make_append_encryption(EncryptionAlgorithm::Aegis256, test_key(), &basin, &stream); + + let input = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"secret")]), + match_seq_num: None, + fencing_token: None, + }; + + backend + .append(basin.clone(), stream.clone(), input, Some(enc)) + .await + .expect("append should succeed"); + + let session = backend + .read( + basin.clone(), + stream.clone(), + ReadStart { + from: ReadFrom::SeqNum(0), + clamp: false, + }, + ReadEnd { + limit: ReadLimit::Count(10), + until: ReadUntil::Unbounded, + wait: None, + }, + ) + .await + .expect("read session"); + + let wrong_directive = EncryptionDirective::Key { + alg: EncryptionAlgorithm::Aegis256, + key: test_key_2(), + }; + let aad = stream_aad(&basin, &stream).into_bytes(); + + tokio::pin!(session); + while let Some(output) = session.next().await { + match output.expect("read output") { + ReadSessionOutput::Batch(batch) => { + let result = decrypt_read_batch(batch, Some(&wrong_directive), &aad); + assert!(result.is_err(), "decryption with wrong key should fail"); + } + ReadSessionOutput::Heartbeat(_) => {} + } + } +} + +#[tokio::test] +async fn test_mixed_encrypted_and_plaintext_append() { + let (backend, basin, stream) = + setup_backend_with_stream("enc-mixed", "stream", OptionalStreamConfig::default()).await; + + // First append: plaintext. + let input1 = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"plaintext")]), + match_seq_num: None, + fencing_token: None, + }; + backend + .append(basin.clone(), stream.clone(), input1, None) + .await + .expect("plaintext append"); + + // Second append: encrypted. + let enc = make_append_encryption(EncryptionAlgorithm::Aegis256, test_key(), &basin, &stream); + let input2 = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"encrypted")]), + match_seq_num: None, + fencing_token: None, + }; + backend + .append(basin.clone(), stream.clone(), input2, Some(enc)) + .await + .expect("encrypted append"); + + // Read all records. + let session = backend + .read( + basin.clone(), + stream.clone(), + ReadStart { + from: ReadFrom::SeqNum(0), + clamp: false, + }, + ReadEnd { + limit: ReadLimit::Count(10), + until: ReadUntil::Unbounded, + wait: None, + }, + ) + .await + .expect("read session"); + + let mut records = Vec::new(); + tokio::pin!(session); + while let Some(output) = session.next().await { + match output.expect("read output") { + ReadSessionOutput::Batch(batch) => { + for sr in batch.records.into_inner() { + records.push(sr); + } + } + ReadSessionOutput::Heartbeat(_) => {} + } + } + + assert_eq!(records.len(), 2); + // Record 0: plaintext, body should be readable directly. + let Record::Envelope(ref env0) = records[0].record else { + panic!("expected envelope record"); + }; + assert_eq!(env0.body().as_ref(), b"plaintext"); + // Record 1: encrypted, body should NOT be plaintext. + let Record::Envelope(ref env1) = records[1].record else { + panic!("expected envelope record"); + }; + assert_ne!(env1.body().as_ref(), b"encrypted"); +} diff --git a/lite/tests/backend/data_plane/mod.rs b/lite/tests/backend/data_plane/mod.rs index 8e9689a0..1c730d3f 100644 --- a/lite/tests/backend/data_plane/mod.rs +++ b/lite/tests/backend/data_plane/mod.rs @@ -2,6 +2,7 @@ use super::common; mod append; mod auto_create; +mod encryption; mod mixed; mod read; mod read_follow; diff --git a/sdk/src/api.rs b/sdk/src/api.rs index 59b054b4..dd688146 100644 --- a/sdk/src/api.rs +++ b/sdk/src/api.rs @@ -9,14 +9,14 @@ use http::{ header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE, InvalidHeaderValue}, }; use prost::{self, Message}; +#[cfg(feature = "_hidden")] +use s2_api::v1::basin::CreateOrReconfigureBasinRequest; use s2_api::v1::{ access::{ AccessTokenInfo, IssueAccessTokenResponse, ListAccessTokensRequest, ListAccessTokensResponse, }, - basin::{ - BasinInfo, CreateBasinRequest, ListBasinsRequest, ListBasinsResponse, - }, + basin::{BasinInfo, CreateBasinRequest, ListBasinsRequest, ListBasinsResponse}, config::{BasinConfig, BasinReconfiguration, StreamConfig, StreamReconfiguration}, metrics::{ AccountMetricSetRequest, BasinMetricSetRequest, MetricSetResponse, StreamMetricSetRequest, @@ -32,13 +32,10 @@ use secrecy::ExposeSecret; use tokio_util::codec::Decoder; use tracing::{debug, warn}; use url::Url; -#[cfg(feature = "_hidden")] -use s2_api::v1::basin::CreateOrReconfigureBasinRequest; - -use crate::frame_signal::FrameSignal; use crate::{ client::{self, StreamingResponse, UnaryResponse}, + frame_signal::FrameSignal, retry::{RetryBackoff, RetryBackoffBuilder}, types::{ AccessTokenId, AppendRetryPolicy, BasinAuthority, BasinName, Compression, RetryConfig, @@ -51,6 +48,7 @@ const CONTENT_TYPE_PROTO: &str = "application/protobuf"; const ACCEPT_PROTO: &str = "application/protobuf"; const S2_REQUEST_TOKEN: &str = "s2-request-token"; const S2_BASIN: &str = "s2-basin"; +const S2_ENCRYPTION: &str = "s2-encryption"; const RETRY_AFTER_MS_HEADER: &str = "retry-after-ms"; #[derive(Debug, Clone)] @@ -394,7 +392,8 @@ impl BasinClient { .query(&start) .query(&end); if let Some(wait) = end.wait { - builder = builder.timeout(self.client.request_timeout + Duration::from_secs(wait.into())); + builder = + builder.timeout(self.client.request_timeout + Duration::from_secs(wait.into())); } let request = builder.build()?; let response = self @@ -829,6 +828,17 @@ impl BaseClient { Compression::None => {} } + if let Some(ref enc) = config.encryption { + use crate::types::EncryptionConfig; + let header_value = match enc { + EncryptionConfig::Key { alg, key } => { + format!("alg={}; key={}", alg.as_api_str(), key.expose_secret()) + } + EncryptionConfig::Attest => "attest".to_owned(), + }; + default_headers.insert(S2_ENCRYPTION, header_value.try_into()?); + } + let client = client::Pool::new(connector); Ok(Self { @@ -963,10 +973,7 @@ impl<'a> RequestBuilder<'a> { r }; - let response = self - .client - .execute_unary(attempt_request) - .await; + let response = self.client.execute_unary(attempt_request).await; let (err, retry_after) = match response { Ok(resp) => { @@ -1183,7 +1190,9 @@ mod tests { // Server errors that do NOT guarantee no mutation. assert!(!server_error(StatusCode::INTERNAL_SERVER_ERROR, "internal").has_no_side_effects()); assert!(!server_error(StatusCode::BAD_GATEWAY, "other").has_no_side_effects()); - assert!(!server_error(StatusCode::SERVICE_UNAVAILABLE, "unavailable").has_no_side_effects()); + assert!( + !server_error(StatusCode::SERVICE_UNAVAILABLE, "unavailable").has_no_side_effects() + ); } #[test] diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 85bdc1dd..07422eaa 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -385,6 +385,33 @@ impl RetryConfig { } } +/// Encryption configuration for the `S2-Encryption` request header. +#[derive(Clone)] +pub enum EncryptionConfig { + /// Server-side encryption with caller-provided algorithm and key. + Key { + /// Encryption algorithm. + alg: EncryptionAlgorithm, + /// Hex-encoded 32-byte key (64 hex characters). + key: SecretString, + }, + /// Client handles encryption; server passes bytes through. + Attest, +} + +impl fmt::Debug for EncryptionConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Key { alg, .. } => f + .debug_struct("Key") + .field("alg", alg) + .field("key", &"[REDACTED]") + .finish(), + Self::Attest => write!(f, "Attest"), + } + } +} + #[derive(Debug, Clone)] #[non_exhaustive] /// Configuration for [`S2`](crate::S2). @@ -397,6 +424,7 @@ pub struct S2Config { pub(crate) compression: Compression, pub(crate) user_agent: HeaderValue, pub(crate) insecure_skip_cert_verification: bool, + pub(crate) encryption: Option, } impl S2Config { @@ -413,6 +441,7 @@ impl S2Config { .parse() .expect("valid user agent"), insecure_skip_cert_verification: false, + encryption: None, } } @@ -476,6 +505,15 @@ impl S2Config { } } + /// Set the encryption configuration. This sets the `S2-Encryption` header on all requests. + /// The server ignores this header on non-data-path endpoints (e.g. list-basins, create-stream). + pub fn with_encryption(self, encryption: EncryptionConfig) -> Self { + Self { + encryption: Some(encryption), + ..self + } + } + #[doc(hidden)] #[cfg(feature = "_hidden")] pub fn with_user_agent(self, user_agent: impl Into) -> Result { @@ -542,6 +580,16 @@ pub enum EncryptionAlgorithm { Aes256Gcm, } +impl EncryptionAlgorithm { + /// Wire-format string for the `S2-Encryption` header. + pub fn as_api_str(self) -> &'static str { + match self { + Self::Aegis256 => "aegis-256", + Self::Aes256Gcm => "aes-256-gcm", + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] /// Retention policy for records in a stream. pub enum RetentionPolicy { @@ -714,10 +762,6 @@ pub struct StreamConfig { /// /// See [`DeleteOnEmptyConfig`] for defaults. pub delete_on_empty: Option, - /// Encryption algorithm for this stream. Immutable after creation. - /// - /// Defaults to `None` (plaintext). - pub encryption_algorithm: Option, } impl StreamConfig { @@ -757,14 +801,6 @@ impl StreamConfig { ..self } } - - /// Set the encryption algorithm for the stream. Immutable after creation. - pub fn with_encryption_algorithm(self, alg: EncryptionAlgorithm) -> Self { - Self { - encryption_algorithm: Some(alg), - ..self - } - } } impl From for StreamConfig { @@ -774,34 +810,17 @@ impl From for StreamConfig { retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), - encryption_algorithm: value.encryption_algorithm.as_deref().and_then(|s| { - use s2_common::types::config::EncryptionAlgorithm as E; - match E::parse_api_str(s)? { - E::Aegis256 => Some(EncryptionAlgorithm::Aegis256), - E::Aes256Gcm => Some(EncryptionAlgorithm::Aes256Gcm), - E::None => Option::None, - } - }), } } } impl From for api::config::StreamConfig { fn from(value: StreamConfig) -> Self { - use s2_common::types::config::EncryptionAlgorithm as E; Self { storage_class: value.storage_class.map(Into::into), retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), - encryption_algorithm: value.encryption_algorithm.map(|a| { - match a { - EncryptionAlgorithm::Aegis256 => E::Aegis256, - EncryptionAlgorithm::Aes256Gcm => E::Aes256Gcm, - } - .as_api_str() - .to_owned() - }), } } } @@ -822,9 +841,6 @@ pub struct BasinConfig { /// /// Defaults to `false`. pub create_stream_on_read: bool, - /// Allowlist of encryption algorithms permitted for streams in this basin. - /// Empty means all algorithms (including plaintext) are allowed. - pub allowed_encryption_algorithms: Vec, } impl BasinConfig { @@ -861,43 +877,20 @@ impl BasinConfig { impl From for BasinConfig { fn from(value: api::config::BasinConfig) -> Self { - use s2_common::types::config::EncryptionAlgorithm as E; Self { default_stream_config: value.default_stream_config.map(Into::into), create_stream_on_append: value.create_stream_on_append, create_stream_on_read: value.create_stream_on_read, - allowed_encryption_algorithms: value - .allowed_encryption_algorithms - .iter() - .filter_map(|s| match E::parse_api_str(s)? { - E::Aegis256 => Some(EncryptionAlgorithm::Aegis256), - E::Aes256Gcm => Some(EncryptionAlgorithm::Aes256Gcm), - E::None => Option::None, - }) - .collect(), } } } impl From for api::config::BasinConfig { fn from(value: BasinConfig) -> Self { - use s2_common::types::config::EncryptionAlgorithm as E; Self { default_stream_config: value.default_stream_config.map(Into::into), create_stream_on_append: value.create_stream_on_append, create_stream_on_read: value.create_stream_on_read, - allowed_encryption_algorithms: value - .allowed_encryption_algorithms - .into_iter() - .map(|a| { - match a { - EncryptionAlgorithm::Aegis256 => E::Aegis256, - EncryptionAlgorithm::Aes256Gcm => E::Aes256Gcm, - } - .as_api_str() - .to_owned() - }) - .collect(), } } } @@ -1372,7 +1365,6 @@ impl From for api::config::StreamReconfiguration { retention_policy: value.retention_policy.map(|m| m.map(Into::into)), timestamping: value.timestamping.map(|m| m.map(Into::into)), delete_on_empty: value.delete_on_empty.map(|m| m.map(Into::into)), - encryption_algorithm: None, } } } From 080b82d22772ae26b837c62df7f7069218267d88 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 09:08:47 +0530 Subject: [PATCH 08/42] . --- Cargo.lock | 1 - cli/Cargo.toml | 1 - cli/src/main.rs | 10 ++++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 565510dd..109f8d5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4687,7 +4687,6 @@ dependencies = [ "dirs", "eyre", "futures", - "hex", "http 1.4.0", "humantime", "indicatif", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4b2b1021..9b54d5c8 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -27,7 +27,6 @@ crossterm = "0.29" dirs = { workspace = true } eyre = { workspace = true } futures = { workspace = true } -hex = { workspace = true } http = { workspace = true } humantime = { workspace = true } indicatif = { workspace = true } diff --git a/cli/src/main.rs b/cli/src/main.rs index 4f7edccd..5fc036fb 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -837,13 +837,11 @@ fn resolve_command_encryption(command: &Command) -> Result return Ok(None), }; - if key.len() != 64 { - return Err(CliError::InvalidEncryptionKey(format!( - "expected 64 hex characters, got {}", - key.len() - ))); + if key.len() != 64 || !key.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(CliError::InvalidEncryptionKey( + "key must be exactly 64 hex characters (32 bytes)".to_owned(), + )); } - hex::decode(&key).map_err(|e| CliError::InvalidEncryptionKey(e.to_string()))?; let alg = args.alg.unwrap_or(types::EncryptionAlgorithm::Aegis256); From cbe7ae87a41d41ee7c2796f9e269cf0c26fc1b9c Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 13:07:38 +0530 Subject: [PATCH 09/42] . --- common/src/encryption.rs | 32 ++++++++++----------- common/src/types/config.rs | 23 ++------------- lite/src/backend/streamer.rs | 10 +++++-- lite/src/handlers/v1/records.rs | 6 ++-- lite/tests/backend/data_plane/encryption.rs | 8 +++--- sdk/src/api.rs | 2 +- sdk/src/types.rs | 11 ++++--- 7 files changed, 39 insertions(+), 53 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index 2be63171..f021a98f 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -10,9 +10,9 @@ //! [version: 1 byte] [alg_id: 1 byte] [nonce] [ciphertext] [tag] //! ``` //! -//! | version | Description | -//! |---------|--------------------------------------------------| -//! | 0x01 | Initial versioned format. AAD = base ‖ alg ‖ seq_num_le | +//! | version | Description | +//! |---------|-------------------------------------------------------| +//! | 0x01 | Initial versioned format. AAD = base ‖ seq_num_le | //! //! | alg_id | Algorithm | Nonce | Tag | //! |--------|-------------|--------|------| @@ -36,7 +36,6 @@ pub const S2_ENCRYPTION_HEADER: &str = "s2-encryption"; /// Ciphertext envelope version. Stored as the first byte of the encrypted record body. /// Tampering is caught by the version dispatch in `decrypt_record` before AAD construction. -/// N.B. shares the value `0x01` with `ALG_ID_AEGIS256` but occupies a different byte offset. const CIPHERTEXT_V1: u8 = 0x01; const ALG_ID_AEGIS256: u8 = 0x01; @@ -123,7 +122,7 @@ pub fn parse_s2_encryption_header( .ok_or_else(|| EncryptionError::MalformedHeader("missing 'key=' prefix".to_owned()))? .trim(); - let alg = EncryptionAlgorithm::parse_api_str(alg_str).ok_or_else(|| { + let alg: EncryptionAlgorithm = alg_str.parse().map_err(|_| { EncryptionError::MalformedHeader(format!( "unknown algorithm {alg_str:?}; expected 'aegis-256' or 'aes-256-gcm'" )) @@ -173,16 +172,17 @@ pub fn decode_record_plaintext(bytes: Bytes) -> Result<(Vec
, Bytes), Enc .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) } -/// Build the effective AAD for V1 envelope format. The alg_id and seq_num are mixed -/// into the AAD so the AEAD tag binds the ciphertext to its algorithm and stream -/// position. The version byte is not included -- it's already gated by the dispatch -/// in `decrypt_record`, so a version flip is caught before AAD construction. +/// Build the effective AAD for V1 envelope format. The seq_num is mixed into the AAD +/// so the AEAD tag binds the ciphertext to its stream position, preventing reordering. /// -/// Layout: `[base_aad | alg_id | seq_num: 8 bytes LE]` -fn effective_aad_v1(base: &[u8], alg_id: u8, seq_num: crate::record::SeqNum) -> Vec { - let mut buf = Vec::with_capacity(base.len() + 1 + 8); +/// The version and alg_id bytes are not included -- version is gated by dispatch before +/// AAD construction, and alg_id flips cause decryption failure due to nonce/tag size +/// mismatches. +/// +/// Layout: `[base_aad | seq_num: 8 bytes LE]` +fn effective_aad_v1(base: &[u8], seq_num: crate::record::SeqNum) -> Vec { + let mut buf = Vec::with_capacity(base.len() + 8); buf.extend_from_slice(base); - buf.push(alg_id); buf.extend_from_slice(&seq_num.to_le_bytes()); buf } @@ -196,7 +196,7 @@ pub fn encrypt_record( ) -> Result { match alg { EncryptionAlgorithm::Aegis256 => { - let full_aad = effective_aad_v1(aad, ALG_ID_AEGIS256, seq_num); + let full_aad = effective_aad_v1(aad, seq_num); let nonce: [u8; NONCE_BYTES_AEGIS256] = random(); let (ciphertext, tag) = Aegis256::::new(&key.expose_secret().0, &nonce) @@ -213,7 +213,7 @@ pub fn encrypt_record( Ok(out.freeze()) } EncryptionAlgorithm::Aes256Gcm => { - let full_aad = effective_aad_v1(aad, ALG_ID_AES256GCM, seq_num); + let full_aad = effective_aad_v1(aad, seq_num); let nonce: [u8; NONCE_BYTES_AES256GCM] = random(); let cipher = Aes256Gcm::new_from_slice(&key.expose_secret().0).map_err(|_| { EncryptionError::EncodingFailed("invalid AES key length".to_owned()) @@ -268,7 +268,7 @@ fn decrypt_record_v1( .split_first() .ok_or(EncryptionError::DecryptionFailed)?; - let full_aad = effective_aad_v1(aad, alg_id, seq_num); + let full_aad = effective_aad_v1(aad, seq_num); match alg_id { ALG_ID_AEGIS256 => { diff --git a/common/src/types/config.rs b/common/src/types/config.rs index 686032d2..701e8f5b 100644 --- a/common/src/types/config.rs +++ b/common/src/types/config.rs @@ -34,31 +34,14 @@ use enum_ordinalize::Ordinalize; use crate::maybe::Maybe; /// Encryption algorithm for record data, specified per-request via the `S2-Encryption` header. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display, strum::EnumString)] pub enum EncryptionAlgorithm { + #[strum(serialize = "aegis-256")] Aegis256, + #[strum(serialize = "aes-256-gcm")] Aes256Gcm, } -impl EncryptionAlgorithm { - /// Wire-format string used in JSON API and `S2-Encryption` header. - pub fn as_api_str(self) -> &'static str { - match self { - Self::Aegis256 => "aegis-256", - Self::Aes256Gcm => "aes-256-gcm", - } - } - - /// Parse from wire-format string. - pub fn parse_api_str(s: &str) -> Option { - match s { - "aegis-256" => Some(Self::Aegis256), - "aes-256-gcm" => Some(Self::Aes256Gcm), - _ => None, - } - } -} - #[derive( Debug, Default, diff --git a/lite/src/backend/streamer.rs b/lite/src/backend/streamer.rs index 0517b44c..779c9a25 100644 --- a/lite/src/backend/streamer.rs +++ b/lite/src/backend/streamer.rs @@ -494,8 +494,14 @@ impl Streamer { #[derive(Clone)] pub struct AppendEncryption { - pub directive: s2_common::encryption::EncryptionDirective, - pub aad: Vec, + directive: s2_common::encryption::EncryptionDirective, + aad: Vec, +} + +impl AppendEncryption { + pub fn new(directive: s2_common::encryption::EncryptionDirective, aad: Vec) -> Self { + Self { directive, aad } + } } enum Message { diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index 208e0051..c29e9518 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -60,10 +60,8 @@ impl EncryptionContext { } fn into_append_encryption(self) -> Option { - self.directive.map(|directive| AppendEncryption { - directive, - aad: self.aad, - }) + self.directive + .map(|directive| AppendEncryption::new(directive, self.aad)) } fn decrypt_batch(&self, batch: ReadBatch) -> Result { diff --git a/lite/tests/backend/data_plane/encryption.rs b/lite/tests/backend/data_plane/encryption.rs index 6b8270e1..1a04a727 100644 --- a/lite/tests/backend/data_plane/encryption.rs +++ b/lite/tests/backend/data_plane/encryption.rs @@ -31,10 +31,10 @@ fn make_append_encryption( basin: &s2_common::types::basin::BasinName, stream: &s2_common::types::stream::StreamName, ) -> AppendEncryption { - AppendEncryption { - directive: EncryptionDirective::Key { alg, key }, - aad: stream_aad(basin, stream).into_bytes(), - } + AppendEncryption::new( + EncryptionDirective::Key { alg, key }, + stream_aad(basin, stream).into_bytes(), + ) } #[tokio::test] diff --git a/sdk/src/api.rs b/sdk/src/api.rs index dd688146..3a17f495 100644 --- a/sdk/src/api.rs +++ b/sdk/src/api.rs @@ -832,7 +832,7 @@ impl BaseClient { use crate::types::EncryptionConfig; let header_value = match enc { EncryptionConfig::Key { alg, key } => { - format!("alg={}; key={}", alg.as_api_str(), key.expose_secret()) + format!("alg={alg}; key={}", key.expose_secret()) } EncryptionConfig::Attest => "attest".to_owned(), }; diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 07422eaa..1e18aa8f 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -571,8 +571,8 @@ impl From for api::config::StorageClass { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] /// Encryption algorithm for stream records. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EncryptionAlgorithm { /// AEGIS-256 authenticated encryption. Aegis256, @@ -580,13 +580,12 @@ pub enum EncryptionAlgorithm { Aes256Gcm, } -impl EncryptionAlgorithm { - /// Wire-format string for the `S2-Encryption` header. - pub fn as_api_str(self) -> &'static str { - match self { +impl fmt::Display for EncryptionAlgorithm { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { Self::Aegis256 => "aegis-256", Self::Aes256Gcm => "aes-256-gcm", - } + }) } } From a065eab2431b66549504659863baa7097231ee32 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 13:20:40 +0530 Subject: [PATCH 10/42] . --- Cargo.toml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 55e34399..7947ac21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,8 @@ repository = "https://github.com/s2-streamstore/s2" homepage = "https://s2.dev" [workspace.dependencies] +aegis = "0.9" +aes-gcm = "0.10" async-stream = "0.3" async-trait = "0.1" aws-config = "1" @@ -31,6 +33,7 @@ enumset = "1.1" eyre = "0.6" flate2 = "1.1" futures = "0.3" +hex = "0.4" http = "1.4" humantime = "2.3" indexmap = "2.13" @@ -55,6 +58,7 @@ s2-common = { path = "common", version = "0.30" } s2-lite = { path = "lite", version = "0.29" } s2-sdk = { path = "sdk", version = "0.26" } schemars = "1.2" +secrecy = "0.10.3" serde = "1.0" serde_json = "1.0" slatedb = "0.11.2" @@ -74,12 +78,6 @@ uuid = "1.22" xxhash-rust = "0.8" zstd = "0.13" -# Encryption -aegis = "0.9" -aes-gcm = "0.10" -hex = "0.4" -secrecy = "0.10.3" - [patch.crates-io] utoipa = { git = "https://github.com/infiniteregrets/utoipa", rev = "9cd181d40ac0a50551ebe1995a4d73e8685890d6" } From 8f755a8ecda446c3d2b9d7bfe01b7f514e4ca61e Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 19:45:52 +0530 Subject: [PATCH 11/42] . --- common/src/encryption.rs | 62 +++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index f021a98f..5661ccd8 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -1,10 +1,4 @@ -//! Server-side record encryption for HIPAA/BAA compliance. -//! -//! The `S2-Encryption` header carries the algorithm and key for each request. -//! The server encrypts on append and decrypts on read; the key never persists -//! beyond the request lifetime. -//! -//! ## Wire format (body of stored EnvelopeRecord) +//! # Ciphertext format //! //! ```text //! [version: 1 byte] [alg_id: 1 byte] [nonce] [ciphertext] [tag] @@ -29,13 +23,14 @@ use http::HeaderMap; use rand::random; use secrecy::{CloneableSecret, ExposeSecret, SecretBox}; -use crate::record::{Encodable as _, EnvelopeRecord, Header}; pub use crate::types::config::EncryptionAlgorithm; +use crate::{ + record::{self, Encodable as _, EnvelopeRecord, Header}, + types, +}; pub const S2_ENCRYPTION_HEADER: &str = "s2-encryption"; -/// Ciphertext envelope version. Stored as the first byte of the encrypted record body. -/// Tampering is caught by the version dispatch in `decrypt_record` before AAD construction. const CIPHERTEXT_V1: u8 = 0x01; const ALG_ID_AEGIS256: u8 = 0x01; @@ -47,7 +42,6 @@ const TAG_BYTES_AEGIS256: usize = 32; const NONCE_BYTES_AES256GCM: usize = 12; const TAG_BYTES_AES256GCM: usize = 16; -/// Newtype for a 32-byte encryption key that allows cloning and zeroizes on drop. #[derive(Clone)] pub struct KeyBytes(pub [u8; 32]); @@ -59,22 +53,18 @@ impl secrecy::zeroize::Zeroize for KeyBytes { impl CloneableSecret for KeyBytes {} -/// A cloneable, debug-redacted wrapper around a 32-byte key. pub type EncryptionKey = SecretBox; fn make_key(bytes: [u8; 32]) -> EncryptionKey { SecretBox::new(Box::new(KeyBytes(bytes))) } -/// Parsed `S2-Encryption` header directive. #[derive(Clone, Debug)] pub enum EncryptionDirective { - /// Client provides the algorithm and key; server encrypts/decrypts. Key { alg: EncryptionAlgorithm, key: EncryptionKey, }, - /// Client handles encryption itself; server passes bytes through. Attest, } @@ -98,15 +88,15 @@ pub fn parse_s2_encryption_header( None => return Ok(None), }; - let s = value + let header_str = value .to_str() .map_err(|_| EncryptionError::MalformedHeader("header is not valid UTF-8".to_owned()))?; - if s.trim() == "attest" { + if header_str.trim() == "attest" { return Ok(Some(EncryptionDirective::Attest)); } - let (alg_part, key_part) = s.split_once(';').ok_or_else(|| { + let (alg_part, key_part) = header_str.split_once(';').ok_or_else(|| { EncryptionError::MalformedHeader("expected 'alg=...; key=...'".to_owned()) })?; @@ -180,7 +170,7 @@ pub fn decode_record_plaintext(bytes: Bytes) -> Result<(Vec
, Bytes), Enc /// mismatches. /// /// Layout: `[base_aad | seq_num: 8 bytes LE]` -fn effective_aad_v1(base: &[u8], seq_num: crate::record::SeqNum) -> Vec { +fn effective_aad_v1(base: &[u8], seq_num: record::SeqNum) -> Vec { let mut buf = Vec::with_capacity(base.len() + 8); buf.extend_from_slice(base); buf.extend_from_slice(&seq_num.to_le_bytes()); @@ -192,7 +182,7 @@ pub fn encrypt_record( alg: EncryptionAlgorithm, key: &EncryptionKey, aad: &[u8], - seq_num: crate::record::SeqNum, + seq_num: record::SeqNum, ) -> Result { match alg { EncryptionAlgorithm::Aegis256 => { @@ -246,7 +236,7 @@ pub fn decrypt_record( body: &[u8], key: &EncryptionKey, aad: &[u8], - seq_num: crate::record::SeqNum, + seq_num: record::SeqNum, ) -> Result { let (&version, after_version) = body .split_first() @@ -262,7 +252,7 @@ fn decrypt_record_v1( body: &[u8], key: &EncryptionKey, aad: &[u8], - seq_num: crate::record::SeqNum, + seq_num: record::SeqNum, ) -> Result { let (&alg_id, rest) = body .split_first() @@ -312,58 +302,58 @@ fn decrypt_record_v1( } pub fn encrypt_sequenced_records( - records: Vec>, + records: Vec>, alg: EncryptionAlgorithm, key: &EncryptionKey, aad: &[u8], -) -> Result>, EncryptionError> { +) -> Result>, EncryptionError> { records .into_iter() .map(|msr| { - let crate::record::SequencedRecord { position, record } = msr.into_inner(); + let record::SequencedRecord { position, record } = msr.into_inner(); let encrypted = match &record { - crate::record::Record::Envelope(env) => { + record::Record::Envelope(env) => { let plaintext = encode_record_plaintext(env.headers().to_vec(), env.body().clone())?; let enc_body = encrypt_record(&plaintext, alg, key, aad, position.seq_num)?; crate::record::Record::try_from_parts(vec![], enc_body) .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))? } - crate::record::Record::Command(_) => record, + record::Record::Command(_) => record, }; - Ok(crate::record::Metered::from(encrypted.sequenced(position))) + Ok(record::Metered::from(encrypted.sequenced(position))) }) .collect() } pub fn decrypt_read_batch( - batch: crate::types::stream::ReadBatch, + batch: types::stream::ReadBatch, directive: Option<&EncryptionDirective>, aad: &[u8], -) -> Result { +) -> Result { let Some(EncryptionDirective::Key { key, .. }) = directive else { return Ok(batch); }; - let records: Vec = batch + let records: Vec = batch .records .into_inner() .into_iter() .map(|sr| { - let crate::record::Record::Envelope(ref env) = sr.record else { + let record::Record::Envelope(ref env) = sr.record else { return Ok(sr); }; let plaintext = decrypt_record(env.body(), key, aad, sr.position.seq_num)?; let (headers, body) = decode_record_plaintext(plaintext)?; - let record = crate::record::Record::try_from_parts(headers, body) + let record = record::Record::try_from_parts(headers, body) .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; - Ok(crate::record::SequencedRecord { + Ok(record::SequencedRecord { position: sr.position, record, }) }) .collect::>()?; - Ok(crate::types::stream::ReadBatch { - records: crate::record::Metered::from(records), + Ok(types::stream::ReadBatch { + records: record::Metered::from(records), tail: batch.tail, }) } From 94eb362b06faefbbe647fcbf94e0b7fa800b7fc7 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 20:10:45 +0530 Subject: [PATCH 12/42] . --- sdk/src/types.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 1e18aa8f..27d0ea72 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -385,17 +385,17 @@ impl RetryConfig { } } -/// Encryption configuration for the `S2-Encryption` request header. +/// Encryption configuration. #[derive(Clone)] pub enum EncryptionConfig { - /// Server-side encryption with caller-provided algorithm and key. + /// Algorithm and key. Key { /// Encryption algorithm. alg: EncryptionAlgorithm, - /// Hex-encoded 32-byte key (64 hex characters). + /// Hex-encoded 32-byte key. key: SecretString, }, - /// Client handles encryption; server passes bytes through. + /// Attest mode. Attest, } @@ -505,8 +505,7 @@ impl S2Config { } } - /// Set the encryption configuration. This sets the `S2-Encryption` header on all requests. - /// The server ignores this header on non-data-path endpoints (e.g. list-basins, create-stream). + /// Set the encryption configuration. pub fn with_encryption(self, encryption: EncryptionConfig) -> Self { Self { encryption: Some(encryption), @@ -571,12 +570,12 @@ impl From for api::config::StorageClass { } } -/// Encryption algorithm for stream records. +/// Encryption algorithm. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EncryptionAlgorithm { - /// AEGIS-256 authenticated encryption. + /// AEGIS-256 Aegis256, - /// AES-256-GCM authenticated encryption (NIST-compliant). + /// AES-256-GCM Aes256Gcm, } From 9955bbb2db61bd341e60d8ff8d463233c02bd480 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 20:13:49 +0530 Subject: [PATCH 13/42] . --- common/src/encryption.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index 5661ccd8..e01970b7 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -162,13 +162,7 @@ pub fn decode_record_plaintext(bytes: Bytes) -> Result<(Vec
, Bytes), Enc .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) } -/// Build the effective AAD for V1 envelope format. The seq_num is mixed into the AAD -/// so the AEAD tag binds the ciphertext to its stream position, preventing reordering. -/// -/// The version and alg_id bytes are not included -- version is gated by dispatch before -/// AAD construction, and alg_id flips cause decryption failure due to nonce/tag size -/// mismatches. -/// +/// The seq_num is mixed into the AAD, so the AEAD tag binds the ciphertext to its stream position. /// Layout: `[base_aad | seq_num: 8 bytes LE]` fn effective_aad_v1(base: &[u8], seq_num: record::SeqNum) -> Vec { let mut buf = Vec::with_capacity(base.len() + 8); From 176e3473e0a8569d236f001431a42e01851e484e Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 20:15:30 +0530 Subject: [PATCH 14/42] . --- cli/src/cli.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 2c2b10c5..665500db 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -465,11 +465,11 @@ pub struct AppendArgs { #[arg(long, default_value = "5ms")] pub linger: humantime::Duration, - /// Hex-encoded 32-byte encryption key. Prefer S2_ENCRYPTION_KEY env var. + /// Hex-encoded 32-byte encryption key. Alternatively, set S2_ENCRYPTION_KEY env var. #[arg(long, env = "S2_ENCRYPTION_KEY", hide_env_values = true)] pub encryption_key: Option, - /// Read encryption key from file (first line, trimmed). + /// Read encryption key from file. #[arg(long, conflicts_with = "encryption_key")] pub encryption_key_file: Option, @@ -477,7 +477,7 @@ pub struct AppendArgs { #[arg(long, value_enum)] pub encryption_algorithm: Option, - /// Attest client-side encryption (mutually exclusive with key/algorithm). + /// Attest client-side encryption. #[arg(long, conflicts_with_all = ["encryption_key", "encryption_key_file", "encryption_algorithm"])] pub encryption_attest: bool, } From e2a59a89af332437a9992f203562cbc6c2baaf54 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 21:43:13 +0530 Subject: [PATCH 15/42] . --- sdk/src/api.rs | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/sdk/src/api.rs b/sdk/src/api.rs index 3a17f495..13d58705 100644 --- a/sdk/src/api.rs +++ b/sdk/src/api.rs @@ -357,6 +357,8 @@ impl BasinClient { .header(ACCEPT, ACCEPT_PROTO) .body(input.encode_to_vec()) .build()?; + let mut request = request; + self.client.set_encryption_header(&mut request); let response = self .request(request) .with_append_retry_policy(append_retry_policy) @@ -395,7 +397,8 @@ impl BasinClient { builder = builder.timeout(self.client.request_timeout + Duration::from_secs(wait.into())); } - let request = builder.build()?; + let mut request = builder.build()?; + self.client.set_encryption_header(&mut request); let response = self .request(request) .error_handler(read_response_error_handler) @@ -437,9 +440,11 @@ impl BasinClient { .timeout(self.client.request_timeout); request_builder = add_basin_header_if_required(request_builder, &self.config.endpoints, &self.name); + let mut request = request_builder.build()?; + self.client.set_encryption_header(&mut request); let response = self .client - .init_streaming(request_builder.build()?) + .init_streaming(request) .await? .into_result() .await?; @@ -488,9 +493,11 @@ impl BasinClient { .timeout(self.client.request_timeout); request_builder = add_basin_header_if_required(request_builder, &self.config.endpoints, &self.name); + let mut request = request_builder.build()?; + self.client.set_encryption_header(&mut request); let response = self .client - .init_streaming(request_builder.build()?) + .init_streaming(request) .await? .into_result() .await?; @@ -781,6 +788,7 @@ pub type Streaming = Pin>>>; pub struct BaseClient { client: Arc, default_headers: HeaderMap, + encryption_header: Option, request_timeout: Duration, retry_builder: RetryBackoffBuilder, compression: Compression, @@ -828,28 +836,39 @@ impl BaseClient { Compression::None => {} } - if let Some(ref enc) = config.encryption { - use crate::types::EncryptionConfig; - let header_value = match enc { - EncryptionConfig::Key { alg, key } => { - format!("alg={alg}; key={}", key.expose_secret()) - } - EncryptionConfig::Attest => "attest".to_owned(), - }; - default_headers.insert(S2_ENCRYPTION, header_value.try_into()?); - } + let encryption_header = config + .encryption + .as_ref() + .map(|enc| { + use crate::types::EncryptionConfig; + let value = match enc { + EncryptionConfig::Key { alg, key } => { + format!("alg={alg}; key={}", key.expose_secret()) + } + EncryptionConfig::Attest => "attest".to_owned(), + }; + value.try_into() + }) + .transpose()?; let client = client::Pool::new(connector); Ok(Self { client: Arc::new(client), default_headers, + encryption_header, request_timeout: config.request_timeout, retry_builder: retry_builder(&config.retry), compression: config.compression, }) } + pub fn set_encryption_header(&self, request: &mut client::Request) { + if let Some(ref value) = self.encryption_header { + request.headers_mut().insert(S2_ENCRYPTION, value.clone()); + } + } + pub fn get(&self, url: Url) -> client::RequestBuilder { client::RequestBuilder::get(url) .timeout(self.request_timeout) From 0c40ff89043f027b9c9725e8975ffbd7f386d36a Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 23:21:51 +0530 Subject: [PATCH 16/42] . --- lite/src/handlers/v1/records.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index c29e9518..6fbbc74d 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -460,7 +460,7 @@ pub async fn append( let input_err_stream = futures::stream::once(err_rx).filter_map(|res| async move { match res { - Ok(err) => Some(Err(err.into())), + Ok(err) => Some(Err(err)), Err(_) => None, } }); From 8f90bff361085522822a486dfd721d6c0cf7f341 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Mon, 23 Mar 2026 23:38:58 +0530 Subject: [PATCH 17/42] . --- cli/src/config.rs | 9 +-------- cli/src/main.rs | 6 ++++-- cli/src/tui/mod.rs | 2 +- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index b7abb960..0c575ab7 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -163,10 +163,7 @@ pub fn unset_config_value(key: ConfigKey) -> Result { save_cli_config(&config) } -pub fn sdk_config( - config: &CliConfig, - encryption: Option, -) -> Result { +pub fn sdk_config(config: &CliConfig) -> Result { let access_token = config .access_token .as_ref() @@ -213,10 +210,6 @@ pub fn sdk_config( sdk_config = sdk_config.with_insecure_skip_cert_verification(true); } - if let Some(enc) = encryption { - sdk_config = sdk_config.with_encryption(enc); - } - Ok(sdk_config) } diff --git a/cli/src/main.rs b/cli/src/main.rs index 5fc036fb..5a8e778d 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -151,8 +151,10 @@ async fn run() -> Result<(), CliError> { } let cli_config = load_cli_config()?; - let encryption = resolve_command_encryption(&command)?; - let sdk_config = sdk_config(&cli_config, encryption)?; + let mut sdk_config = sdk_config(&cli_config)?; + if let Some(enc) = resolve_command_encryption(&command)? { + sdk_config = sdk_config.with_encryption(enc); + } let s2 = S2::new(sdk_config.clone()).map_err(CliError::SdkInit)?; let token_source = access_token_source(&cli_config); let result: Result<(), CliError> = (async { diff --git a/cli/src/tui/mod.rs b/cli/src/tui/mod.rs index 1cc7f063..35a5e3fb 100644 --- a/cli/src/tui/mod.rs +++ b/cli/src/tui/mod.rs @@ -20,7 +20,7 @@ pub async fn run() -> Result<(), CliError> { // Load config and try to create SDK client // If access token is missing, we'll start with Setup screen instead of failing let cli_config = load_cli_config()?; - let s2 = match sdk_config(&cli_config, None) { + let s2 = match sdk_config(&cli_config) { Ok(sdk_cfg) => Some(s2_sdk::S2::new(sdk_cfg).map_err(CliError::SdkInit)?), Err(_) => None, // No access token - will show setup screen }; From a815b317107fe0bc4af0819f0ae61d274356a79e Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 24 Mar 2026 00:53:15 +0530 Subject: [PATCH 18/42] .. --- common/src/types/config.rs | 2 +- lite/src/backend/core.rs | 1 - lite/src/backend/error.rs | 4 ---- lite/src/handlers/v1/error.rs | 3 --- 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/common/src/types/config.rs b/common/src/types/config.rs index 701e8f5b..96645135 100644 --- a/common/src/types/config.rs +++ b/common/src/types/config.rs @@ -33,7 +33,7 @@ use enum_ordinalize::Ordinalize; use crate::maybe::Maybe; -/// Encryption algorithm for record data, specified per-request via the `S2-Encryption` header. +/// Encryption algorithm. #[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display, strum::EnumString)] pub enum EncryptionAlgorithm { #[strum(serialize = "aegis-256")] diff --git a/lite/src/backend/core.rs b/lite/src/backend/core.rs index c22d143e..16286445 100644 --- a/lite/src/backend/core.rs +++ b/lite/src/backend/core.rs @@ -311,7 +311,6 @@ impl Backend { CreateStreamError::BasinDeletionPending(e) => Err(e)?, CreateStreamError::StreamDeletionPending(e) => Err(e)?, CreateStreamError::BasinNotFound(e) => Err(e)?, - CreateStreamError::InvalidConfig(_) => {} CreateStreamError::StreamAlreadyExists(_) => {} } } diff --git a/lite/src/backend/error.rs b/lite/src/backend/error.rs index c3ec7b89..1c21fe2b 100644 --- a/lite/src/backend/error.rs +++ b/lite/src/backend/error.rs @@ -288,8 +288,6 @@ pub enum CreateStreamError { StreamAlreadyExists(#[from] StreamAlreadyExistsError), #[error(transparent)] StreamDeletionPending(#[from] StreamDeletionPendingError), - #[error("{0}")] - InvalidConfig(String), } impl From for CreateStreamError { @@ -429,8 +427,6 @@ pub enum ReconfigureStreamError { StreamNotFound(#[from] StreamNotFoundError), #[error(transparent)] StreamDeletionPending(#[from] StreamDeletionPendingError), - #[error("immutable field: {0}")] - ImmutableField(&'static str), } impl From for ReconfigureStreamError { diff --git a/lite/src/handlers/v1/error.rs b/lite/src/handlers/v1/error.rs index 4ca98760..b4c561a6 100644 --- a/lite/src/handlers/v1/error.rs +++ b/lite/src/handlers/v1/error.rs @@ -16,7 +16,6 @@ use crate::backend::error::{ AppendConditionFailedError, AppendError, CheckTailError, CreateBasinError, CreateStreamError, DeleteBasinError, DeleteStreamError, GetBasinConfigError, GetStreamConfigError, ListBasinsError, ListStreamsError, ReadError, ReconfigureBasinError, ReconfigureStreamError, - ReconfigureStreamError::ImmutableField, }; #[derive(Debug, thiserror::Error)] @@ -155,7 +154,6 @@ impl ServiceError { CreateStreamError::StreamDeletionPending(e) => { standard(ErrorCode::StreamDeletionPending, e.to_string()) } - CreateStreamError::InvalidConfig(e) => standard(ErrorCode::Invalid, e.to_string()), }, ServiceError::GetStreamConfig(e) => match e { GetStreamConfigError::Storage(e) => standard(ErrorCode::Storage, e.to_string()), @@ -191,7 +189,6 @@ impl ServiceError { ReconfigureStreamError::StreamDeletionPending(e) => { standard(ErrorCode::StreamDeletionPending, e.to_string()) } - ImmutableField(_) => standard(ErrorCode::Invalid, e.to_string()), }, ServiceError::CheckTail(e) => match e { CheckTailError::Storage(e) => standard(ErrorCode::Storage, e.to_string()), From 43fdc54ce8935fd7823f18dfb54591b98fed5f6f Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 24 Mar 2026 01:30:18 +0530 Subject: [PATCH 19/42] . --- lite/src/backend/streams.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lite/src/backend/streams.rs b/lite/src/backend/streams.rs index 80c69ddb..dc10007c 100644 --- a/lite/src/backend/streams.rs +++ b/lite/src/backend/streams.rs @@ -150,10 +150,10 @@ impl Backend { let is_reconfigure = existing_meta_opt.is_some(); let (resolved, created_at) = match existing_meta_opt { Some(existing) => (existing.config.reconfigure(config), existing.created_at), - None => { - let cfg = OptionalStreamConfig::default().reconfigure(config); - (cfg, OffsetDateTime::now_utc()) - } + None => ( + OptionalStreamConfig::default().reconfigure(config), + OffsetDateTime::now_utc(), + ), }; let resolved: OptionalStreamConfig = resolved .merge(basin_meta.config.default_stream_config) From 41ed54f070eee06b17d3ff2d214dc00f06946926 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 24 Mar 2026 02:27:26 +0530 Subject: [PATCH 20/42] . --- Cargo.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 109f8d5d..65f1eb7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "aegis" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae1572243695de9c6c8d16c7889899abac907d14c148f1939d837122bbeca79" +checksum = "78412fa53e6da95324e8902c3641b3ff32ab45258582ea997eb9169c68ffa219" dependencies = [ "cc", "softaes", @@ -388,9 +388,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.1" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -399,9 +399,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e" +checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" dependencies = [ "cc", "cmake", @@ -1009,9 +1009,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.56" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "jobserver", @@ -3892,7 +3892,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "itertools", "log", "multimap", @@ -4605,9 +4605,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "aws-lc-rs", "ring", From 7ebd7c5068ad928d644e27989b5904a77c5c699c Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 24 Mar 2026 10:40:13 +0530 Subject: [PATCH 21/42] . --- cli/src/cli.rs | 23 ++++++++--------------- cli/src/main.rs | 32 ++++++++++---------------------- cli/src/tui/app.rs | 15 +++------------ lite/src/handlers/v1/records.rs | 10 ++++++---- 4 files changed, 27 insertions(+), 53 deletions(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 665500db..1c2fd90c 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -465,6 +465,12 @@ pub struct AppendArgs { #[arg(long, default_value = "5ms")] pub linger: humantime::Duration, + #[command(flatten)] + pub encryption: EncryptionArgs, +} + +#[derive(Args, Debug, Clone, Default)] +pub struct EncryptionArgs { /// Hex-encoded 32-byte encryption key. Alternatively, set S2_ENCRYPTION_KEY env var. #[arg(long, env = "S2_ENCRYPTION_KEY", hide_env_values = true)] pub encryption_key: Option, @@ -532,21 +538,8 @@ pub struct ReadArgs { #[arg(short = 'o', long, value_parser = parse_records_output_source, default_value = "-")] pub output: RecordsOut, - /// Hex-encoded 32-byte encryption key. Prefer S2_ENCRYPTION_KEY env var. - #[arg(long, env = "S2_ENCRYPTION_KEY", hide_env_values = true)] - pub encryption_key: Option, - - /// Read encryption key from file (first line, trimmed). - #[arg(long, conflicts_with = "encryption_key")] - pub encryption_key_file: Option, - - /// Encryption algorithm (default: aegis-256). - #[arg(long, value_enum)] - pub encryption_algorithm: Option, - - /// Attest client-side encryption (mutually exclusive with key/algorithm). - #[arg(long, conflicts_with_all = ["encryption_key", "encryption_key_file", "encryption_algorithm"])] - pub encryption_attest: bool, + #[command(flatten)] + pub encryption: EncryptionArgs, } #[derive(Args, Debug)] diff --git a/cli/src/main.rs b/cli/src/main.rs index 5a8e778d..d14592e7 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -801,34 +801,20 @@ fn print_metrics(metrics: &[Metric]) { } fn resolve_command_encryption(command: &Command) -> Result, CliError> { - struct EncryptionArgs<'a> { - key: &'a Option, - key_file: &'a Option, - alg: Option, - attest: bool, - } - let args = match command { - Command::Append(a) => EncryptionArgs { - key: &a.encryption_key, - key_file: &a.encryption_key_file, - alg: a.encryption_algorithm, - attest: a.encryption_attest, - }, - Command::Read(a) => EncryptionArgs { - key: &a.encryption_key, - key_file: &a.encryption_key_file, - alg: a.encryption_algorithm, - attest: a.encryption_attest, - }, + Command::Append(a) => &a.encryption, + Command::Read(a) => &a.encryption, _ => return Ok(None), }; + resolve_encryption(args) +} - if args.attest { +fn resolve_encryption(args: &cli::EncryptionArgs) -> Result, CliError> { + if args.encryption_attest { return Ok(Some(EncryptionConfig::Attest)); } - let key = match (args.key, args.key_file) { + let key = match (&args.encryption_key, &args.encryption_key_file) { (Some(k), _) => k.clone(), (_, Some(path)) => { let contents = std::fs::read_to_string(path).map_err(|e| { @@ -845,7 +831,9 @@ fn resolve_command_encryption(command: &Command) -> Result Result { let directive = parse_s2_encryption_header(headers) .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; - Ok(Self { - aad: stream_aad(basin, stream).into_bytes(), - directive, - }) + let aad = if directive.is_some() { + stream_aad(basin, stream).into_bytes() + } else { + Vec::new() + }; + Ok(Self { aad, directive }) } fn into_append_encryption(self) -> Option { From 327c7090af79bc07458c2dd41844057faf55ffb9 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 24 Mar 2026 12:57:41 +0530 Subject: [PATCH 22/42] . --- common/src/types/config.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/src/types/config.rs b/common/src/types/config.rs index 96645135..afadcae4 100644 --- a/common/src/types/config.rs +++ b/common/src/types/config.rs @@ -36,8 +36,10 @@ use crate::maybe::Maybe; /// Encryption algorithm. #[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display, strum::EnumString)] pub enum EncryptionAlgorithm { + /// AEGIS-256 #[strum(serialize = "aegis-256")] Aegis256, + /// AES-256-GCM #[strum(serialize = "aes-256-gcm")] Aes256Gcm, } From 4a4261908d5d397df0447285ebd5c969fb6d3559 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Thu, 26 Mar 2026 13:21:15 +0530 Subject: [PATCH 23/42] refactor(encryption): use stream_id as AAD, move encryption to handler layer Drop seq_num from AAD per team decision. AAD is now stream_id (BLAKE3 hash of basin+stream), matching StreamId::new in lite. This allows encryption to happen in the HTTP handler before the backend, removing all encryption plumbing from the streamer pipeline. - Replace effective_aad_v1(base, seq_num) with stream_id_aad(basin, stream) - Replace encrypt_sequenced_records with encrypt_append_input (pre-sequencing) - Remove AppendEncryption struct from streamer/backend - Encrypt in handler before backend.append, decrypt after backend.read Co-Authored-By: Claude Opus 4.6 (1M context) --- common/src/encryption.rs | 162 +++++++++++-------- lite/src/backend/append.rs | 12 +- lite/src/backend/error.rs | 5 - lite/src/backend/mod.rs | 2 - lite/src/backend/read.rs | 11 +- lite/src/backend/streamer.rs | 53 +----- lite/src/handlers/v1/error.rs | 1 - lite/src/handlers/v1/records.rs | 44 +++-- lite/tests/backend/common.rs | 2 +- lite/tests/backend/control_plane/stream.rs | 6 +- lite/tests/backend/data_plane/append.rs | 53 +++--- lite/tests/backend/data_plane/auto_create.rs | 6 +- lite/tests/backend/data_plane/encryption.rs | 57 ++++--- lite/tests/backend/data_plane/mixed.rs | 4 +- lite/tests/backend/data_plane/read.rs | 24 +-- lite/tests/backend/data_plane/read_follow.rs | 6 +- 16 files changed, 209 insertions(+), 239 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index e01970b7..582575a9 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -4,9 +4,9 @@ //! [version: 1 byte] [alg_id: 1 byte] [nonce] [ciphertext] [tag] //! ``` //! -//! | version | Description | -//! |---------|-------------------------------------------------------| -//! | 0x01 | Initial versioned format. AAD = base ‖ seq_num_le | +//! | version | Description | +//! |---------|--------------------------------------------------| +//! | 0x01 | Initial versioned format. AAD = stream_id bytes | //! //! | alg_id | Algorithm | Nonce | Tag | //! |--------|-------------|--------|------| @@ -25,8 +25,12 @@ use secrecy::{CloneableSecret, ExposeSecret, SecretBox}; pub use crate::types::config::EncryptionAlgorithm; use crate::{ - record::{self, Encodable as _, EnvelopeRecord, Header}, - types, + bash::Bash, + record::{self, Encodable as _, EnvelopeRecord, Header, Metered, Record}, + types::{ + self, + stream::{AppendInput, AppendRecord, AppendRecordBatch, AppendRecordParts}, + }, }; pub const S2_ENCRYPTION_HEADER: &str = "s2-encryption"; @@ -162,13 +166,13 @@ pub fn decode_record_plaintext(bytes: Bytes) -> Result<(Vec
, Bytes), Enc .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) } -/// The seq_num is mixed into the AAD, so the AEAD tag binds the ciphertext to its stream position. -/// Layout: `[base_aad | seq_num: 8 bytes LE]` -fn effective_aad_v1(base: &[u8], seq_num: record::SeqNum) -> Vec { - let mut buf = Vec::with_capacity(base.len() + 8); - buf.extend_from_slice(base); - buf.extend_from_slice(&seq_num.to_le_bytes()); - buf +/// Compute stream_id AAD: BLAKE3 hash of `basin ‖ 0x00 ‖ stream ‖ 0x00`. +/// Matches `StreamId::new(basin, stream)` in lite. +pub fn stream_id_aad( + basin: &(impl AsRef<[u8]> + ?Sized), + stream: &(impl AsRef<[u8]> + ?Sized), +) -> [u8; 32] { + *Bash::delimited(&[basin.as_ref(), stream.as_ref()], 0).as_bytes() } pub fn encrypt_record( @@ -176,15 +180,13 @@ pub fn encrypt_record( alg: EncryptionAlgorithm, key: &EncryptionKey, aad: &[u8], - seq_num: record::SeqNum, ) -> Result { match alg { EncryptionAlgorithm::Aegis256 => { - let full_aad = effective_aad_v1(aad, seq_num); let nonce: [u8; NONCE_BYTES_AEGIS256] = random(); let (ciphertext, tag) = Aegis256::::new(&key.expose_secret().0, &nonce) - .encrypt(plaintext, &full_aad); + .encrypt(plaintext, aad); let mut out = BytesMut::with_capacity( 2 + NONCE_BYTES_AEGIS256 + ciphertext.len() + TAG_BYTES_AEGIS256, @@ -197,7 +199,6 @@ pub fn encrypt_record( Ok(out.freeze()) } EncryptionAlgorithm::Aes256Gcm => { - let full_aad = effective_aad_v1(aad, seq_num); let nonce: [u8; NONCE_BYTES_AES256GCM] = random(); let cipher = Aes256Gcm::new_from_slice(&key.expose_secret().0).map_err(|_| { EncryptionError::EncodingFailed("invalid AES key length".to_owned()) @@ -208,7 +209,7 @@ pub fn encrypt_record( nonce_generic, Payload { msg: plaintext, - aad: &full_aad, + aad, }, ) .map_err(|_| { @@ -230,14 +231,13 @@ pub fn decrypt_record( body: &[u8], key: &EncryptionKey, aad: &[u8], - seq_num: record::SeqNum, ) -> Result { let (&version, after_version) = body .split_first() .ok_or(EncryptionError::DecryptionFailed)?; match version { - CIPHERTEXT_V1 => decrypt_record_v1(after_version, key, aad, seq_num), + CIPHERTEXT_V1 => decrypt_record_v1(after_version, key, aad), v => Err(EncryptionError::UnsupportedVersion(v)), } } @@ -246,14 +246,11 @@ fn decrypt_record_v1( body: &[u8], key: &EncryptionKey, aad: &[u8], - seq_num: record::SeqNum, ) -> Result { let (&alg_id, rest) = body .split_first() .ok_or(EncryptionError::DecryptionFailed)?; - let full_aad = effective_aad_v1(aad, seq_num); - match alg_id { ALG_ID_AEGIS256 => { if rest.len() < NONCE_BYTES_AEGIS256 + TAG_BYTES_AEGIS256 { @@ -267,7 +264,7 @@ fn decrypt_record_v1( let tag: &[u8; TAG_BYTES_AEGIS256] = after_nonce[tag_offset..].try_into().unwrap(); let plaintext = Aegis256::::new(&key.expose_secret().0, nonce) - .decrypt(ciphertext, tag, &full_aad) + .decrypt(ciphertext, tag, aad) .map_err(|_| EncryptionError::DecryptionFailed)?; Ok(Bytes::from(plaintext)) } @@ -285,7 +282,7 @@ fn decrypt_record_v1( nonce_generic, Payload { msg: ciphertext_with_tag, - aad: &full_aad, + aad, }, ) .map_err(|_| EncryptionError::DecryptionFailed)?; @@ -295,29 +292,49 @@ fn decrypt_record_v1( } } -pub fn encrypt_sequenced_records( - records: Vec>, +pub fn encrypt_append_input( + input: AppendInput, alg: EncryptionAlgorithm, key: &EncryptionKey, aad: &[u8], -) -> Result>, EncryptionError> { - records +) -> Result { + let encrypted_records: Vec = input + .records .into_iter() - .map(|msr| { - let record::SequencedRecord { position, record } = msr.into_inner(); - let encrypted = match &record { - record::Record::Envelope(env) => { + .map(|record| { + let AppendRecordParts { + timestamp, + record: metered_record, + } = record.into(); + let inner_record = metered_record.into_inner(); + let encrypted = match &inner_record { + Record::Envelope(env) => { let plaintext = encode_record_plaintext(env.headers().to_vec(), env.body().clone())?; - let enc_body = encrypt_record(&plaintext, alg, key, aad, position.seq_num)?; - crate::record::Record::try_from_parts(vec![], enc_body) + let enc_body = encrypt_record(&plaintext, alg, key, aad)?; + Record::try_from_parts(vec![], enc_body) .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))? } - record::Record::Command(_) => record, + Record::Command(_) => inner_record, }; - Ok(record::Metered::from(encrypted.sequenced(position))) + AppendRecordParts { + timestamp, + record: Metered::from(encrypted), + } + .try_into() + .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_owned())) }) - .collect() + .collect::>()?; + + let records: AppendRecordBatch = encrypted_records + .try_into() + .map_err(|e: &str| EncryptionError::EncodingFailed(e.to_owned()))?; + + Ok(AppendInput { + records, + match_seq_num: input.match_seq_num, + fencing_token: input.fencing_token, + }) } pub fn decrypt_read_batch( @@ -336,7 +353,7 @@ pub fn decrypt_read_batch( let record::Record::Envelope(ref env) = sr.record else { return Ok(sr); }; - let plaintext = decrypt_record(env.body(), key, aad, sr.position.seq_num)?; + let plaintext = decrypt_record(env.body(), key, aad)?; let (headers, body) = decode_record_plaintext(plaintext)?; let record = record::Record::try_from_parts(headers, body) .map_err(|e| EncryptionError::EncodingFailed(e.to_string()))?; @@ -352,10 +369,6 @@ pub fn decrypt_read_batch( }) } -pub fn stream_aad(basin: &impl std::fmt::Display, stream: &impl std::fmt::Display) -> String { - format!("{basin}/{stream}") -} - #[cfg(test)] mod tests { use super::*; @@ -368,8 +381,9 @@ mod tests { make_key([0x99u8; 32]) } - const AAD: &[u8] = b"test-basin/test-stream"; - const SEQ: u64 = 42; + fn test_aad() -> [u8; 32] { + stream_id_aad("test-basin", "test-stream") + } fn roundtrip(alg: EncryptionAlgorithm) { let headers = vec![Header { @@ -378,10 +392,11 @@ mod tests { }]; let body = Bytes::from_static(b"secret payload"); + let aad = test_aad(); let plaintext = encode_record_plaintext(headers.clone(), body.clone()).unwrap(); let key = make_key_fn(); - let ciphertext = encrypt_record(&plaintext, alg, &key, AAD, SEQ).unwrap(); - let decrypted = decrypt_record(&ciphertext, &key, AAD, SEQ).unwrap(); + let ciphertext = encrypt_record(&plaintext, alg, &key, &aad).unwrap(); + let decrypted = decrypt_record(&ciphertext, &key, &aad).unwrap(); let (out_headers, out_body) = decode_record_plaintext(decrypted).unwrap(); assert_eq!(out_headers, headers); @@ -400,41 +415,44 @@ mod tests { #[test] fn wrong_key_fails_aegis256() { + let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); let ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ).unwrap(); - let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), AAD, SEQ); + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad).unwrap(); + let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), &aad); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } #[test] fn wrong_key_fails_aes256gcm() { + let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); let ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aes256Gcm, &key, AAD, SEQ).unwrap(); - let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), AAD, SEQ); + encrypt_record(&plaintext, EncryptionAlgorithm::Aes256Gcm, &key, &aad).unwrap(); + let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), &aad); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } #[test] fn truncated_ciphertext_fails_no_panic() { + let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); let ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ).unwrap(); - // Truncate to 4 bytes -- version + alg_id + 2 nonce bytes, too short. + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad).unwrap(); let truncated = &ciphertext[..4]; - let result = decrypt_record(truncated, &key, AAD, SEQ); + let result = decrypt_record(truncated, &key, &aad); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } #[test] fn unsupported_version_fails() { + let aad = test_aad(); let key = make_key_fn(); let body = b"\xFFsome opaque bytes"; - let result = decrypt_record(body, &key, AAD, SEQ); + let result = decrypt_record(body, &key, &aad); assert!(matches!( result, Err(EncryptionError::UnsupportedVersion(0xFF)) @@ -443,48 +461,48 @@ mod tests { #[test] fn empty_body_fails() { + let aad = test_aad(); let key = make_key_fn(); - let result = decrypt_record(b"", &key, AAD, SEQ); + let result = decrypt_record(b"", &key, &aad); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } #[test] fn version_byte_present() { + let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); let ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ).unwrap(); + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad).unwrap(); assert_eq!(ciphertext[0], CIPHERTEXT_V1); assert_eq!(ciphertext[1], ALG_ID_AEGIS256); } #[test] fn alg_id_flip_detected() { + let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); - let mut ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ) - .unwrap() - .to_vec(); + let mut ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad) + .unwrap() + .to_vec(); assert_eq!(ciphertext[0], CIPHERTEXT_V1); assert_eq!(ciphertext[1], ALG_ID_AEGIS256); - // Flip alg_id (byte 1), keep version intact. ciphertext[1] = ALG_ID_AES256GCM; - let result = decrypt_record(&ciphertext, &key, AAD, SEQ); + let result = decrypt_record(&ciphertext, &key, &aad); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } #[test] fn version_flip_detected() { + let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); - let mut ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, SEQ) - .unwrap() - .to_vec(); - // Flip version byte to a hypothetical v2. + let mut ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad) + .unwrap() + .to_vec(); ciphertext[0] = 0x02; - let result = decrypt_record(&ciphertext, &key, AAD, SEQ); + let result = decrypt_record(&ciphertext, &key, &aad); assert!(matches!( result, Err(EncryptionError::UnsupportedVersion(0x02)) @@ -492,12 +510,14 @@ mod tests { } #[test] - fn wrong_seq_num_fails() { + fn wrong_aad_fails() { + let aad = test_aad(); + let other_aad = stream_id_aad("other-basin", "other-stream"); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); let key = make_key_fn(); let ciphertext = - encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, AAD, 5).unwrap(); - let result = decrypt_record(&ciphertext, &key, AAD, 6); + encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad).unwrap(); + let result = decrypt_record(&ciphertext, &key, &other_aad); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } diff --git a/lite/src/backend/append.rs b/lite/src/backend/append.rs index b3c4fd07..b9972da4 100644 --- a/lite/src/backend/append.rs +++ b/lite/src/backend/append.rs @@ -14,7 +14,7 @@ use s2_common::{ }; use tokio::sync::oneshot; -use super::{Backend, streamer::AppendEncryption}; +use super::Backend; use crate::backend::error::{AppendError, AppendErrorInternal, StorageError}; impl Backend { @@ -23,18 +23,13 @@ impl Backend { basin: BasinName, stream: StreamName, input: AppendInput, - encryption: Option, ) -> Result { let client = self .streamer_client_with_auto_create::(&basin, &stream, |config| { config.create_stream_on_append }) .await?; - let ack = client - .append_permit(input, encryption) - .await? - .submit() - .await?; + let ack = client.append_permit(input).await?.submit().await?; Ok(ack) } @@ -43,7 +38,6 @@ impl Backend { basin: BasinName, stream: StreamName, inputs: impl Stream, - encryption: Option, ) -> Result>, AppendError> { let client = self .streamer_client_with_auto_create::(&basin, &stream, |config| { @@ -58,7 +52,7 @@ impl Backend { loop { tokio::select! { Some(input) = inputs.next(), if permit_opt.is_none() => { - permit_opt = Some(Box::pin(client.append_permit(input, encryption.clone()))); + permit_opt = Some(Box::pin(client.append_permit(input))); } Some(res) = OptionFuture::from(permit_opt.as_mut()) => { permit_opt = None; diff --git a/lite/src/backend/error.rs b/lite/src/backend/error.rs index 1c21fe2b..dc98a65f 100644 --- a/lite/src/backend/error.rs +++ b/lite/src/backend/error.rs @@ -102,8 +102,6 @@ pub(super) enum AppendErrorInternal { ConditionFailed(#[from] AppendConditionFailedError), #[error(transparent)] TimestampMissing(#[from] AppendTimestampRequiredError), - #[error(transparent)] - Encryption(#[from] s2_common::encryption::EncryptionError), } #[derive(Debug, Clone, thiserror::Error)] @@ -156,8 +154,6 @@ pub enum AppendError { ConditionFailed(#[from] AppendConditionFailedError), #[error(transparent)] TimestampMissing(#[from] AppendTimestampRequiredError), - #[error(transparent)] - Encryption(#[from] s2_common::encryption::EncryptionError), } impl From for AppendError { @@ -170,7 +166,6 @@ impl From for AppendError { AppendErrorInternal::RequestDroppedError(e) => AppendError::RequestDroppedError(e), AppendErrorInternal::ConditionFailed(e) => AppendError::ConditionFailed(e), AppendErrorInternal::TimestampMissing(e) => AppendError::TimestampMissing(e), - AppendErrorInternal::Encryption(e) => AppendError::Encryption(e), } } } diff --git a/lite/src/backend/mod.rs b/lite/src/backend/mod.rs index eaea103a..46851323 100644 --- a/lite/src/backend/mod.rs +++ b/lite/src/backend/mod.rs @@ -15,8 +15,6 @@ mod stream_id; pub use core::Backend; -pub use streamer::AppendEncryption; - pub const FOLLOWER_MAX_LAG: usize = 25; #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/lite/src/backend/read.rs b/lite/src/backend/read.rs index 8d990d73..5578a9f8 100644 --- a/lite/src/backend/read.rs +++ b/lite/src/backend/read.rs @@ -488,7 +488,7 @@ mod tests { fencing_token: None, }; let ack = backend - .append(basin.clone(), stream.clone(), input, None) + .append(basin.clone(), stream.clone(), input) .await .unwrap(); assert!(ack.end.seq_num > 0); @@ -652,7 +652,7 @@ mod tests { fencing_token: None, }; let ack = backend - .append(basin.clone(), stream.clone(), input, None) + .append(basin.clone(), stream.clone(), input) .await .unwrap(); delete_batch.delete(kv::stream_record_data::ser_key(stream_id, ack.start)); @@ -721,7 +721,7 @@ mod tests { fencing_token: None, }; backend - .append(basin.clone(), stream.clone(), initial_input, None) + .append(basin.clone(), stream.clone(), initial_input) .await .unwrap(); @@ -776,10 +776,7 @@ mod tests { match_seq_num: None, fencing_token: None, }; - backend - .append(basin, stream, follow_input, None) - .await - .unwrap(); + backend.append(basin, stream, follow_input).await.unwrap(); let next = session .as_mut() diff --git a/lite/src/backend/streamer.rs b/lite/src/backend/streamer.rs index 779c9a25..767a4fbf 100644 --- a/lite/src/backend/streamer.rs +++ b/lite/src/backend/streamer.rs @@ -200,23 +200,6 @@ impl Streamer { .unwrap_or(self.stable_pos) } - fn sequence_and_encrypt( - &self, - input: AppendInput, - encryption: Option, - ) -> Result>, AppendErrorInternal> { - let records = self.sequence_records(input)?; - match encryption { - Some(AppendEncryption { - directive: s2_common::encryption::EncryptionDirective::Key { alg, ref key }, - ref aad, - }) => Ok(s2_common::encryption::encrypt_sequenced_records( - records, alg, key, aad, - )?), - _ => Ok(records), - } - } - fn sequence_records( &self, AppendInput { @@ -279,7 +262,6 @@ impl Streamer { fn handle_append( &mut self, input: AppendInput, - encryption: Option, session: Option, reply_tx: oneshot::Sender>, append_type: AppendType, @@ -287,7 +269,7 @@ impl Streamer { let Some(ticket) = append::admit(reply_tx, session) else { return; }; - match self.sequence_and_encrypt(input, encryption) { + match self.sequence_records(input) { Ok(sequenced_records) => { let retention = self.config.retention_policy.unwrap_or_default(); let doe_deadline = self.maybe_doe_deadline(retention.age()); @@ -428,13 +410,12 @@ impl Streamer { match msg { Message::Append { input, - encryption, session, reply_tx, append_type, } => { if self.trim_point.state.end < SeqNum::MAX { - self.handle_append(input, encryption, session, reply_tx, append_type); + self.handle_append(input, session, reply_tx, append_type); } } Message::Follow { @@ -492,22 +473,9 @@ impl Streamer { } } -#[derive(Clone)] -pub struct AppendEncryption { - directive: s2_common::encryption::EncryptionDirective, - aad: Vec, -} - -impl AppendEncryption { - pub fn new(directive: s2_common::encryption::EncryptionDirective, aad: Vec) -> Self { - Self { directive, aad } - } -} - enum Message { Append { input: AppendInput, - encryption: Option, session: Option, reply_tx: oneshot::Sender>, append_type: AppendType, @@ -595,7 +563,6 @@ impl StreamerClient { pub async fn append_permit( &self, input: AppendInput, - encryption: Option, ) -> Result, StreamerMissingInActionError> { let metered_size = input.records.metered_size(); metrics::observe_append_batch_size(input.records.len(), metered_size); @@ -615,7 +582,6 @@ impl StreamerClient { sema_permit, msg_tx: &self.msg_tx, input, - encryption, }) } @@ -636,7 +602,7 @@ impl StreamerClient { fencing_token: None, }; match self - .append_permit(input, None) + .append_permit(input) .await? .submit_internal(None, AppendType::Terminal) .await @@ -652,9 +618,6 @@ impl StreamerClient { } AppendErrorInternal::ConditionFailed(_) => unreachable!("unconditional write"), AppendErrorInternal::TimestampMissing(_) => unreachable!("Timestamp::MAX used"), - AppendErrorInternal::Encryption(_) => { - unreachable!("no encryption for terminal trim") - } }), } } @@ -673,7 +636,6 @@ pub struct AppendPermit<'a> { sema_permit: SemaphorePermit<'a>, msg_tx: &'a mpsc::UnboundedSender, input: AppendInput, - encryption: Option, } impl AppendPermit<'_> { @@ -699,13 +661,11 @@ impl AppendPermit<'_> { sema_permit, msg_tx, input, - encryption, } = self; let (reply_tx, reply_rx) = oneshot::channel(); msg_tx .send(Message::Append { input, - encryption, session, reply_tx, append_type, @@ -1129,13 +1089,13 @@ mod tests { let mut follow_rx = streamer.follow_tx.subscribe(); let (tx1, mut rx1) = oneshot::channel(); - streamer.handle_append(append_input(b"p0"), None, None, tx1, AppendType::Regular); + streamer.handle_append(append_input(b"p0"), None, tx1, AppendType::Regular); let (tx2, mut rx2) = oneshot::channel(); - streamer.handle_append(append_input(b"p1"), None, None, tx2, AppendType::Regular); + streamer.handle_append(append_input(b"p1"), None, tx2, AppendType::Regular); let (tx3, mut rx3) = oneshot::channel(); - streamer.handle_append(append_input(b"p2"), None, None, tx3, AppendType::Regular); + streamer.handle_append(append_input(b"p2"), None, tx3, AppendType::Regular); let mut db_seqs = Vec::new(); while let Some(fut) = streamer.db_writes_pending.pop_front() { @@ -1222,7 +1182,6 @@ mod tests { streamer.handle_append( append_input(payload.as_bytes()), None, - None, tx, AppendType::Regular, ); diff --git a/lite/src/handlers/v1/error.rs b/lite/src/handlers/v1/error.rs index b4c561a6..1c97dd45 100644 --- a/lite/src/handlers/v1/error.rs +++ b/lite/src/handlers/v1/error.rs @@ -243,7 +243,6 @@ impl ServiceError { } => v1t::stream::AppendConditionFailed::SeqNumMismatch(*assigned_seq_num), }), AppendError::TimestampMissing(e) => standard(ErrorCode::Invalid, e.to_string()), - AppendError::Encryption(e) => standard(ErrorCode::Invalid, e.to_string()), }, ServiceError::Read(e) => match e { ReadError::Storage(e) => standard(ErrorCode::Storage, e.to_string()), diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index 14093a07..8e84e76e 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -15,7 +15,7 @@ use s2_api::{ use s2_common::{ caps::RECORD_BATCH_MAX, encryption::{ - self, EncryptionDirective, EncryptionError, parse_s2_encryption_header, stream_aad, + self, EncryptionDirective, EncryptionError, parse_s2_encryption_header, stream_id_aad, }, http::extract::Header, read_extent::{CountOrBytes, ReadLimit}, @@ -23,12 +23,14 @@ use s2_common::{ types::{ ValidationError, basin::BasinName, - stream::{ReadBatch, ReadEnd, ReadFrom, ReadSessionOutput, ReadStart, StreamName}, + stream::{ + AppendInput, ReadBatch, ReadEnd, ReadFrom, ReadSessionOutput, ReadStart, StreamName, + }, }, }; use crate::{ - backend::{AppendEncryption, Backend, error::ReadError}, + backend::{Backend, error::ReadError}, handlers::v1::error::ServiceError, }; @@ -41,7 +43,7 @@ pub fn router() -> axum::Router { } struct EncryptionContext { - aad: Vec, + aad: [u8; 32], directive: Option, } @@ -53,17 +55,17 @@ impl EncryptionContext { ) -> Result { let directive = parse_s2_encryption_header(headers) .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; - let aad = if directive.is_some() { - stream_aad(basin, stream).into_bytes() - } else { - Vec::new() - }; + let aad = stream_id_aad(basin.as_ref(), stream.as_ref()); Ok(Self { aad, directive }) } - fn into_append_encryption(self) -> Option { - self.directive - .map(|directive| AppendEncryption::new(directive, self.aad)) + fn encrypt_input(&self, input: AppendInput) -> Result { + match &self.directive { + Some(EncryptionDirective::Key { alg, key }) => { + encryption::encrypt_append_input(input, *alg, key, &self.aad) + } + _ => Ok(input), + } } fn decrypt_batch(&self, batch: ReadBatch) -> Result { @@ -411,14 +413,16 @@ pub async fn append( }: AppendArgs, ) -> Result { let enc = EncryptionContext::resolve(&headers, &basin, &stream)?; - let append_enc = enc.into_append_encryption(); match request { v1t::stream::AppendRequest::Unary { input, response_mime, } => { - let ack = backend.append(basin, stream, input, append_enc).await?; + let input = enc + .encrypt_input(input) + .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; + let ack = backend.append(basin, stream, input).await?; match response_mime { JsonOrProto::Json => { let ack: v1t::stream::AppendAck = ack.into(); @@ -441,7 +445,15 @@ pub async fn append( let mut err_tx = Some(err_tx); while let Some(input) = inputs.next().await { match input { - Ok(input) => yield input, + Ok(input) => match enc.encrypt_input(input) { + Ok(encrypted) => yield encrypted, + Err(e) => { + if let Some(tx) = err_tx.take() { + let _ = tx.send(ServiceError::Validation(ValidationError(e.to_string()))); + } + break; + } + }, Err(e) => { if let Some(tx) = err_tx.take() { let _ = tx.send(e.into()); @@ -453,7 +465,7 @@ pub async fn append( }; let ack_stream = backend - .append_session(basin, stream, inputs, append_enc) + .append_session(basin, stream, inputs) .await? .map(|res| { res.map(v1t::stream::proto::AppendAck::from) diff --git a/lite/tests/backend/common.rs b/lite/tests/backend/common.rs index def47b60..79575a18 100644 --- a/lite/tests/backend/common.rs +++ b/lite/tests/backend/common.rs @@ -143,7 +143,7 @@ pub async fn append_payloads( fencing_token: None, }; backend - .append(basin.clone(), stream.clone(), input, None) + .append(basin.clone(), stream.clone(), input) .await .expect("Failed to append payloads") } diff --git a/lite/tests/backend/control_plane/stream.rs b/lite/tests/backend/control_plane/stream.rs index cc1acbc6..c9d54339 100644 --- a/lite/tests/backend/control_plane/stream.rs +++ b/lite/tests/backend/control_plane/stream.rs @@ -260,7 +260,7 @@ async fn test_reconfigure_stream_updates_active_streamer() { match_seq_num: None, fencing_token: None, }; - let result = backend.append(basin_name, stream_name, input, None).await; + let result = backend.append(basin_name, stream_name, input).await; assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); } @@ -303,7 +303,7 @@ async fn test_create_stream_create_or_reconfigure_updates_active_streamer() { match_seq_num: None, fencing_token: None, }; - let result = backend.append(basin_name, stream_name, input, None).await; + let result = backend.append(basin_name, stream_name, input).await; assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); } @@ -430,7 +430,7 @@ async fn test_delete_stream_blocks_data_operations() { fencing_token: None, }; let append_result = backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await; assert!(matches!( append_result, diff --git a/lite/tests/backend/data_plane/append.rs b/lite/tests/backend/data_plane/append.rs index e7c079d7..5244c6b1 100644 --- a/lite/tests/backend/data_plane/append.rs +++ b/lite/tests/backend/data_plane/append.rs @@ -66,12 +66,7 @@ async fn test_append_fencing_token_conditions() { }; let ack = backend - .append( - basin_name.clone(), - stream_name.clone(), - matching_input, - None, - ) + .append(basin_name.clone(), stream_name.clone(), matching_input) .await .expect("Expected append to succeed with matching fencing token"); @@ -90,7 +85,7 @@ async fn test_append_fencing_token_conditions() { }; let command_ack = backend - .append(basin_name.clone(), stream_name.clone(), command_input, None) + .append(basin_name.clone(), stream_name.clone(), command_input) .await .expect("Expected fencing command to succeed"); @@ -104,12 +99,7 @@ async fn test_append_fencing_token_conditions() { }; let result = backend - .append( - basin_name.clone(), - stream_name.clone(), - mismatched_input, - None, - ) + .append(basin_name.clone(), stream_name.clone(), mismatched_input) .await; let Err(AppendError::ConditionFailed(AppendConditionFailedError::FencingTokenMismatch { @@ -130,7 +120,7 @@ async fn test_append_fencing_token_conditions() { }; let refreshed_ack = backend - .append(basin_name, stream_name, refreshed_input, None) + .append(basin_name, stream_name, refreshed_input) .await .expect("Expected append to succeed with updated fencing token"); @@ -158,12 +148,7 @@ async fn test_append_requires_timestamp() { }; let result = backend - .append( - basin_name.clone(), - stream_name.clone(), - missing_timestamp, - None, - ) + .append(basin_name.clone(), stream_name.clone(), missing_timestamp) .await; assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); @@ -178,7 +163,7 @@ async fn test_append_requires_timestamp() { }; let ack = backend - .append(basin_name, stream_name, with_timestamp, None) + .append(basin_name, stream_name, with_timestamp) .await .expect("Expected append to succeed when timestamp is provided"); @@ -198,7 +183,7 @@ async fn test_append_with_seq_num_match() { }; let ack = backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append with matching seq_num"); @@ -211,7 +196,7 @@ async fn test_append_with_seq_num_match() { }; let ack2 = backend - .append(basin_name.clone(), stream_name.clone(), input2, None) + .append(basin_name.clone(), stream_name.clone(), input2) .await .expect("Failed to append with matching seq_num"); @@ -234,7 +219,7 @@ async fn test_append_with_seq_num_mismatch() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append first record"); @@ -245,7 +230,7 @@ async fn test_append_with_seq_num_mismatch() { }; let result = backend - .append(basin_name.clone(), stream_name.clone(), input2, None) + .append(basin_name.clone(), stream_name.clone(), input2) .await; assert!(matches!( @@ -285,7 +270,7 @@ async fn test_append_session_basic() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs, None) + .append_session(basin_name.clone(), stream_name.clone(), inputs) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -335,7 +320,7 @@ async fn test_append_session_auto_create_stream() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs, None) + .append_session(basin_name.clone(), stream_name.clone(), inputs) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -370,7 +355,7 @@ async fn test_append_session_empty() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs, None) + .append_session(basin_name.clone(), stream_name.clone(), inputs) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -416,7 +401,7 @@ async fn test_append_session_multiple_records_per_batch() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs, None) + .append_session(basin_name.clone(), stream_name.clone(), inputs) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -486,7 +471,7 @@ async fn test_append_session_with_seq_num_conditions() { ]); let session = backend - .append_session(basin_name.clone(), stream_name.clone(), inputs, None) + .append_session(basin_name.clone(), stream_name.clone(), inputs) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -524,7 +509,7 @@ async fn test_append_session_seq_num_mismatch() { }]); let session = backend - .append_session(basin_name, stream_name, inputs, None) + .append_session(basin_name, stream_name, inputs) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -558,7 +543,7 @@ async fn test_append_session_with_fencing_token() { ]); let session = backend - .append_session(basin_name, stream_name, inputs, None) + .append_session(basin_name, stream_name, inputs) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -598,7 +583,7 @@ async fn test_append_session_large_batches() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs, None) + .append_session(basin_name.clone(), stream_name.clone(), inputs) .await .expect("Failed to create append session"); tokio::pin!(session); @@ -643,7 +628,7 @@ async fn test_append_session_pipeline_preserves_ack_tail_and_read_order() { let session = backend .clone() - .append_session(basin_name.clone(), stream_name.clone(), inputs, None) + .append_session(basin_name.clone(), stream_name.clone(), inputs) .await .expect("Failed to create append session"); tokio::pin!(session); diff --git a/lite/tests/backend/data_plane/auto_create.rs b/lite/tests/backend/data_plane/auto_create.rs index 2ceee8ca..5a0ca5d3 100644 --- a/lite/tests/backend/data_plane/auto_create.rs +++ b/lite/tests/backend/data_plane/auto_create.rs @@ -104,7 +104,7 @@ async fn test_auto_create_disabled_append_fails() { fencing_token: None, }; - let result = backend.append(basin_name, stream_name, input, None).await; + let result = backend.append(basin_name, stream_name, input).await; assert!(matches!(result, Err(AppendError::StreamNotFound(_)))); } @@ -177,7 +177,7 @@ async fn test_auto_create_race_condition_append() { }; for _ in 0..5 { match backend - .append(basin_name.clone(), stream_name.clone(), input.clone(), None) + .append(basin_name.clone(), stream_name.clone(), input.clone()) .await { Ok(ack) => return Ok(ack), @@ -189,7 +189,7 @@ async fn test_auto_create_race_condition_append() { Err(e) => return Err(e), } } - backend.append(basin_name, stream_name, input, None).await + backend.append(basin_name, stream_name, input).await }); handles.push(handle); } diff --git a/lite/tests/backend/data_plane/encryption.rs b/lite/tests/backend/data_plane/encryption.rs index 1a04a727..7e2534ed 100644 --- a/lite/tests/backend/data_plane/encryption.rs +++ b/lite/tests/backend/data_plane/encryption.rs @@ -3,7 +3,7 @@ use futures::StreamExt; use s2_common::{ encryption::{ EncryptionAlgorithm, EncryptionDirective, EncryptionKey, KeyBytes, decrypt_read_batch, - stream_aad, + encrypt_append_input, stream_id_aad, }, read_extent::{ReadLimit, ReadUntil}, record::Record, @@ -12,7 +12,6 @@ use s2_common::{ stream::{AppendInput, ReadEnd, ReadFrom, ReadSessionOutput, ReadStart}, }, }; -use s2_lite::backend::AppendEncryption; use secrecy::SecretBox; use super::common::*; @@ -25,16 +24,20 @@ fn test_key_2() -> EncryptionKey { SecretBox::new(Box::new(KeyBytes([0x99u8; 32]))) } -fn make_append_encryption( - alg: EncryptionAlgorithm, - key: EncryptionKey, +fn test_aad( basin: &s2_common::types::basin::BasinName, stream: &s2_common::types::stream::StreamName, -) -> AppendEncryption { - AppendEncryption::new( - EncryptionDirective::Key { alg, key }, - stream_aad(basin, stream).into_bytes(), - ) +) -> [u8; 32] { + stream_id_aad(basin.as_ref(), stream.as_ref()) +} + +fn encrypt_input( + input: AppendInput, + alg: EncryptionAlgorithm, + key: &EncryptionKey, + aad: &[u8], +) -> AppendInput { + encrypt_append_input(input, alg, key, aad).expect("encryption should succeed") } #[tokio::test] @@ -42,7 +45,8 @@ async fn test_encrypt_append_and_decrypt_read_aegis256() { let (backend, basin, stream) = setup_backend_with_stream("enc-aegis", "stream", OptionalStreamConfig::default()).await; - let enc = make_append_encryption(EncryptionAlgorithm::Aegis256, test_key(), &basin, &stream); + let aad = test_aad(&basin, &stream); + let key = test_key(); let input = AppendInput { records: create_test_record_batch(vec![ @@ -53,14 +57,15 @@ async fn test_encrypt_append_and_decrypt_read_aegis256() { fencing_token: None, }; + let encrypted = encrypt_input(input, EncryptionAlgorithm::Aegis256, &key, &aad); + let ack = backend - .append(basin.clone(), stream.clone(), input, Some(enc)) + .append(basin.clone(), stream.clone(), encrypted) .await .expect("encrypted append should succeed"); assert_eq!(ack.start.seq_num, 0); assert_eq!(ack.end.seq_num, 2); - // Read back raw (encrypted) records. let session = backend .read( basin.clone(), @@ -99,7 +104,6 @@ async fn test_encrypt_append_and_decrypt_read_aegis256() { alg: EncryptionAlgorithm::Aegis256, key: test_key(), }; - let aad = stream_aad(&basin, &stream).into_bytes(); for batch in batches { let decrypted = decrypt_read_batch(batch, Some(&directive), &aad).expect("decryption should succeed"); @@ -121,7 +125,8 @@ async fn test_encrypt_append_and_decrypt_read_aes256gcm() { let (backend, basin, stream) = setup_backend_with_stream("enc-aes", "stream", OptionalStreamConfig::default()).await; - let enc = make_append_encryption(EncryptionAlgorithm::Aes256Gcm, test_key(), &basin, &stream); + let aad = test_aad(&basin, &stream); + let key = test_key(); let input = AppendInput { records: create_test_record_batch(vec![Bytes::from_static(b"aes payload")]), @@ -129,8 +134,10 @@ async fn test_encrypt_append_and_decrypt_read_aes256gcm() { fencing_token: None, }; + let encrypted = encrypt_input(input, EncryptionAlgorithm::Aes256Gcm, &key, &aad); + backend - .append(basin.clone(), stream.clone(), input, Some(enc)) + .append(basin.clone(), stream.clone(), encrypted) .await .expect("encrypted append should succeed"); @@ -155,7 +162,6 @@ async fn test_encrypt_append_and_decrypt_read_aes256gcm() { alg: EncryptionAlgorithm::Aes256Gcm, key: test_key(), }; - let aad = stream_aad(&basin, &stream).into_bytes(); tokio::pin!(session); while let Some(output) = session.next().await { @@ -180,7 +186,8 @@ async fn test_wrong_key_fails_decryption() { let (backend, basin, stream) = setup_backend_with_stream("enc-wrongkey", "stream", OptionalStreamConfig::default()).await; - let enc = make_append_encryption(EncryptionAlgorithm::Aegis256, test_key(), &basin, &stream); + let aad = test_aad(&basin, &stream); + let key = test_key(); let input = AppendInput { records: create_test_record_batch(vec![Bytes::from_static(b"secret")]), @@ -188,8 +195,10 @@ async fn test_wrong_key_fails_decryption() { fencing_token: None, }; + let encrypted = encrypt_input(input, EncryptionAlgorithm::Aegis256, &key, &aad); + backend - .append(basin.clone(), stream.clone(), input, Some(enc)) + .append(basin.clone(), stream.clone(), encrypted) .await .expect("append should succeed"); @@ -214,7 +223,6 @@ async fn test_wrong_key_fails_decryption() { alg: EncryptionAlgorithm::Aegis256, key: test_key_2(), }; - let aad = stream_aad(&basin, &stream).into_bytes(); tokio::pin!(session); while let Some(output) = session.next().await { @@ -233,6 +241,9 @@ async fn test_mixed_encrypted_and_plaintext_append() { let (backend, basin, stream) = setup_backend_with_stream("enc-mixed", "stream", OptionalStreamConfig::default()).await; + let aad = test_aad(&basin, &stream); + let key = test_key(); + // First append: plaintext. let input1 = AppendInput { records: create_test_record_batch(vec![Bytes::from_static(b"plaintext")]), @@ -240,19 +251,19 @@ async fn test_mixed_encrypted_and_plaintext_append() { fencing_token: None, }; backend - .append(basin.clone(), stream.clone(), input1, None) + .append(basin.clone(), stream.clone(), input1) .await .expect("plaintext append"); // Second append: encrypted. - let enc = make_append_encryption(EncryptionAlgorithm::Aegis256, test_key(), &basin, &stream); let input2 = AppendInput { records: create_test_record_batch(vec![Bytes::from_static(b"encrypted")]), match_seq_num: None, fencing_token: None, }; + let encrypted = encrypt_input(input2, EncryptionAlgorithm::Aegis256, &key, &aad); backend - .append(basin.clone(), stream.clone(), input2, Some(enc)) + .append(basin.clone(), stream.clone(), encrypted) .await .expect("encrypted append"); diff --git a/lite/tests/backend/data_plane/mixed.rs b/lite/tests/backend/data_plane/mixed.rs index f8945bba..c4d566ad 100644 --- a/lite/tests/backend/data_plane/mixed.rs +++ b/lite/tests/backend/data_plane/mixed.rs @@ -39,7 +39,7 @@ async fn test_operations_on_nonexistent_basin() { fencing_token: None, }; let append_result = backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await; assert!(matches!(append_result, Err(AppendError::BasinNotFound(_)))); @@ -70,7 +70,7 @@ async fn test_concurrent_appends_to_same_stream() { match_seq_num: None, fencing_token: None, }; - backend.append(basin_name, stream_name, input, None).await + backend.append(basin_name, stream_name, input).await }); handles.push(handle); } diff --git a/lite/tests/backend/data_plane/read.rs b/lite/tests/backend/data_plane/read.rs index ee759b22..0f6da1f7 100644 --- a/lite/tests/backend/data_plane/read.rs +++ b/lite/tests/backend/data_plane/read.rs @@ -168,7 +168,7 @@ async fn test_read_at_tail_without_follow_returns_unwritten() { fencing_token: None, }; let ack = backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("append"); @@ -270,7 +270,7 @@ async fn test_read_timestamp_range() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append records with timestamps"); @@ -336,7 +336,7 @@ async fn test_read_from_timestamp_includes_duplicate_timestamps() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append duplicate timestamp records"); @@ -542,7 +542,7 @@ async fn test_read_until_timestamp_basic() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append timestamped records"); @@ -602,7 +602,7 @@ async fn test_read_until_timestamp_exact_boundary() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append timestamped records"); @@ -651,7 +651,7 @@ async fn test_read_until_timestamp_before_all_records() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append timestamped records"); @@ -693,7 +693,7 @@ async fn test_read_until_timestamp_after_all_records() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append timestamped records"); @@ -749,7 +749,7 @@ async fn test_read_until_with_count_limit_count_wins() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append timestamped records"); @@ -799,7 +799,7 @@ async fn test_read_until_with_count_limit_timestamp_wins() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append timestamped records"); @@ -857,7 +857,7 @@ async fn test_read_until_with_bytes_limit_bytes_wins() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append timestamped records"); @@ -910,7 +910,7 @@ async fn test_read_until_with_bytes_limit_timestamp_wins() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append timestamped records"); @@ -964,7 +964,7 @@ async fn test_read_timestamp_range_with_from_and_until() { }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append timestamped records"); diff --git a/lite/tests/backend/data_plane/read_follow.rs b/lite/tests/backend/data_plane/read_follow.rs index 04fcfaff..0d954e1f 100644 --- a/lite/tests/backend/data_plane/read_follow.rs +++ b/lite/tests/backend/data_plane/read_follow.rs @@ -524,7 +524,7 @@ async fn test_follow_mode_with_timestamp_until() { fencing_token: None, }; backend - .append(basin_name.clone(), stream_name.clone(), input, None) + .append(basin_name.clone(), stream_name.clone(), input) .await .expect("Failed to append initial record"); @@ -557,7 +557,7 @@ async fn test_follow_mode_with_timestamp_until() { fencing_token: None, }; backend_clone - .append(basin_clone.clone(), stream_clone.clone(), input, None) + .append(basin_clone.clone(), stream_clone.clone(), input) .await .unwrap(); @@ -569,7 +569,7 @@ async fn test_follow_mode_with_timestamp_until() { fencing_token: None, }; backend_clone - .append(basin_clone, stream_clone, input, None) + .append(basin_clone, stream_clone, input) .await .unwrap(); }); From 425e0e16c74b47002982a6dc910727cc0447251f Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Thu, 26 Mar 2026 13:29:43 +0530 Subject: [PATCH 24/42] chore: remove backend-level encryption tests These tests manually encrypted before backend.append() and decrypted after backend.read(), but encryption now happens in the handler layer. The roundtrip logic is already covered by unit tests in common/src/encryption.rs. Co-Authored-By: Claude Opus 4.6 (1M context) --- lite/tests/backend/data_plane/encryption.rs | 312 -------------------- lite/tests/backend/data_plane/mod.rs | 1 - 2 files changed, 313 deletions(-) delete mode 100644 lite/tests/backend/data_plane/encryption.rs diff --git a/lite/tests/backend/data_plane/encryption.rs b/lite/tests/backend/data_plane/encryption.rs deleted file mode 100644 index 7e2534ed..00000000 --- a/lite/tests/backend/data_plane/encryption.rs +++ /dev/null @@ -1,312 +0,0 @@ -use bytes::Bytes; -use futures::StreamExt; -use s2_common::{ - encryption::{ - EncryptionAlgorithm, EncryptionDirective, EncryptionKey, KeyBytes, decrypt_read_batch, - encrypt_append_input, stream_id_aad, - }, - read_extent::{ReadLimit, ReadUntil}, - record::Record, - types::{ - config::OptionalStreamConfig, - stream::{AppendInput, ReadEnd, ReadFrom, ReadSessionOutput, ReadStart}, - }, -}; -use secrecy::SecretBox; - -use super::common::*; - -fn test_key() -> EncryptionKey { - SecretBox::new(Box::new(KeyBytes([0x42u8; 32]))) -} - -fn test_key_2() -> EncryptionKey { - SecretBox::new(Box::new(KeyBytes([0x99u8; 32]))) -} - -fn test_aad( - basin: &s2_common::types::basin::BasinName, - stream: &s2_common::types::stream::StreamName, -) -> [u8; 32] { - stream_id_aad(basin.as_ref(), stream.as_ref()) -} - -fn encrypt_input( - input: AppendInput, - alg: EncryptionAlgorithm, - key: &EncryptionKey, - aad: &[u8], -) -> AppendInput { - encrypt_append_input(input, alg, key, aad).expect("encryption should succeed") -} - -#[tokio::test] -async fn test_encrypt_append_and_decrypt_read_aegis256() { - let (backend, basin, stream) = - setup_backend_with_stream("enc-aegis", "stream", OptionalStreamConfig::default()).await; - - let aad = test_aad(&basin, &stream); - let key = test_key(); - - let input = AppendInput { - records: create_test_record_batch(vec![ - Bytes::from_static(b"secret 1"), - Bytes::from_static(b"secret 2"), - ]), - match_seq_num: None, - fencing_token: None, - }; - - let encrypted = encrypt_input(input, EncryptionAlgorithm::Aegis256, &key, &aad); - - let ack = backend - .append(basin.clone(), stream.clone(), encrypted) - .await - .expect("encrypted append should succeed"); - assert_eq!(ack.start.seq_num, 0); - assert_eq!(ack.end.seq_num, 2); - - let session = backend - .read( - basin.clone(), - stream.clone(), - ReadStart { - from: ReadFrom::SeqNum(0), - clamp: false, - }, - ReadEnd { - limit: ReadLimit::Count(10), - until: ReadUntil::Unbounded, - wait: None, - }, - ) - .await - .expect("read session"); - - let mut batches = Vec::new(); - tokio::pin!(session); - while let Some(output) = session.next().await { - match output.expect("read output") { - ReadSessionOutput::Batch(batch) => batches.push(batch), - ReadSessionOutput::Heartbeat(_) => {} - } - } - assert!(!batches.is_empty()); - - // Raw records should NOT match plaintext (they're encrypted). - let Record::Envelope(ref env) = batches[0].records[0].record else { - panic!("expected envelope record"); - }; - assert_ne!(env.body().as_ref(), b"secret 1"); - - // Decrypt and verify plaintext matches. - let directive = EncryptionDirective::Key { - alg: EncryptionAlgorithm::Aegis256, - key: test_key(), - }; - for batch in batches { - let decrypted = - decrypt_read_batch(batch, Some(&directive), &aad).expect("decryption should succeed"); - for sr in decrypted.records.iter() { - let Record::Envelope(ref env) = sr.record else { - panic!("expected envelope record"); - }; - let text = std::str::from_utf8(env.body()).expect("valid utf8"); - assert!( - text == "secret 1" || text == "secret 2", - "unexpected body: {text}" - ); - } - } -} - -#[tokio::test] -async fn test_encrypt_append_and_decrypt_read_aes256gcm() { - let (backend, basin, stream) = - setup_backend_with_stream("enc-aes", "stream", OptionalStreamConfig::default()).await; - - let aad = test_aad(&basin, &stream); - let key = test_key(); - - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"aes payload")]), - match_seq_num: None, - fencing_token: None, - }; - - let encrypted = encrypt_input(input, EncryptionAlgorithm::Aes256Gcm, &key, &aad); - - backend - .append(basin.clone(), stream.clone(), encrypted) - .await - .expect("encrypted append should succeed"); - - let session = backend - .read( - basin.clone(), - stream.clone(), - ReadStart { - from: ReadFrom::SeqNum(0), - clamp: false, - }, - ReadEnd { - limit: ReadLimit::Count(10), - until: ReadUntil::Unbounded, - wait: None, - }, - ) - .await - .expect("read session"); - - let directive = EncryptionDirective::Key { - alg: EncryptionAlgorithm::Aes256Gcm, - key: test_key(), - }; - - tokio::pin!(session); - while let Some(output) = session.next().await { - match output.expect("read output") { - ReadSessionOutput::Batch(batch) => { - let decrypted = decrypt_read_batch(batch, Some(&directive), &aad) - .expect("decryption should succeed"); - for sr in decrypted.records.iter() { - let Record::Envelope(ref env) = sr.record else { - panic!("expected envelope record"); - }; - assert_eq!(env.body().as_ref(), b"aes payload"); - } - } - ReadSessionOutput::Heartbeat(_) => {} - } - } -} - -#[tokio::test] -async fn test_wrong_key_fails_decryption() { - let (backend, basin, stream) = - setup_backend_with_stream("enc-wrongkey", "stream", OptionalStreamConfig::default()).await; - - let aad = test_aad(&basin, &stream); - let key = test_key(); - - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"secret")]), - match_seq_num: None, - fencing_token: None, - }; - - let encrypted = encrypt_input(input, EncryptionAlgorithm::Aegis256, &key, &aad); - - backend - .append(basin.clone(), stream.clone(), encrypted) - .await - .expect("append should succeed"); - - let session = backend - .read( - basin.clone(), - stream.clone(), - ReadStart { - from: ReadFrom::SeqNum(0), - clamp: false, - }, - ReadEnd { - limit: ReadLimit::Count(10), - until: ReadUntil::Unbounded, - wait: None, - }, - ) - .await - .expect("read session"); - - let wrong_directive = EncryptionDirective::Key { - alg: EncryptionAlgorithm::Aegis256, - key: test_key_2(), - }; - - tokio::pin!(session); - while let Some(output) = session.next().await { - match output.expect("read output") { - ReadSessionOutput::Batch(batch) => { - let result = decrypt_read_batch(batch, Some(&wrong_directive), &aad); - assert!(result.is_err(), "decryption with wrong key should fail"); - } - ReadSessionOutput::Heartbeat(_) => {} - } - } -} - -#[tokio::test] -async fn test_mixed_encrypted_and_plaintext_append() { - let (backend, basin, stream) = - setup_backend_with_stream("enc-mixed", "stream", OptionalStreamConfig::default()).await; - - let aad = test_aad(&basin, &stream); - let key = test_key(); - - // First append: plaintext. - let input1 = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"plaintext")]), - match_seq_num: None, - fencing_token: None, - }; - backend - .append(basin.clone(), stream.clone(), input1) - .await - .expect("plaintext append"); - - // Second append: encrypted. - let input2 = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"encrypted")]), - match_seq_num: None, - fencing_token: None, - }; - let encrypted = encrypt_input(input2, EncryptionAlgorithm::Aegis256, &key, &aad); - backend - .append(basin.clone(), stream.clone(), encrypted) - .await - .expect("encrypted append"); - - // Read all records. - let session = backend - .read( - basin.clone(), - stream.clone(), - ReadStart { - from: ReadFrom::SeqNum(0), - clamp: false, - }, - ReadEnd { - limit: ReadLimit::Count(10), - until: ReadUntil::Unbounded, - wait: None, - }, - ) - .await - .expect("read session"); - - let mut records = Vec::new(); - tokio::pin!(session); - while let Some(output) = session.next().await { - match output.expect("read output") { - ReadSessionOutput::Batch(batch) => { - for sr in batch.records.into_inner() { - records.push(sr); - } - } - ReadSessionOutput::Heartbeat(_) => {} - } - } - - assert_eq!(records.len(), 2); - // Record 0: plaintext, body should be readable directly. - let Record::Envelope(ref env0) = records[0].record else { - panic!("expected envelope record"); - }; - assert_eq!(env0.body().as_ref(), b"plaintext"); - // Record 1: encrypted, body should NOT be plaintext. - let Record::Envelope(ref env1) = records[1].record else { - panic!("expected envelope record"); - }; - assert_ne!(env1.body().as_ref(), b"encrypted"); -} diff --git a/lite/tests/backend/data_plane/mod.rs b/lite/tests/backend/data_plane/mod.rs index 1c730d3f..8e9689a0 100644 --- a/lite/tests/backend/data_plane/mod.rs +++ b/lite/tests/backend/data_plane/mod.rs @@ -2,7 +2,6 @@ use super::common; mod append; mod auto_create; -mod encryption; mod mixed; mod read; mod read_follow; From 1dcf22b38f1c66993c2562ed620572dd02520cb4 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Thu, 26 Mar 2026 13:57:55 +0530 Subject: [PATCH 25/42] chore: simplify encryption context - Compute stream_id AAD lazily (only when encryption header present) - Remove unused secrecy dev-dependency from lite Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 1 - lite/Cargo.toml | 1 - lite/src/handlers/v1/records.rs | 14 ++++++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65f1eb7e..aa0c02ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4783,7 +4783,6 @@ dependencies = [ "s2-api", "s2-common", "schemars", - "secrecy", "serde", "serde_json", "slatedb", diff --git a/lite/Cargo.toml b/lite/Cargo.toml index 1cd11fd2..58e8ee4c 100644 --- a/lite/Cargo.toml +++ b/lite/Cargo.toml @@ -63,6 +63,5 @@ utoipa = { version = "5.4", optional = true, features = ["time"] } [dev-dependencies] proptest = { workspace = true } -secrecy = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util", "time"] } uuid = { workspace = true, features = ["v4"] } diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index 8e84e76e..b43fb4d9 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -43,7 +43,7 @@ pub fn router() -> axum::Router { } struct EncryptionContext { - aad: [u8; 32], + aad: Option<[u8; 32]>, directive: Option, } @@ -55,21 +55,27 @@ impl EncryptionContext { ) -> Result { let directive = parse_s2_encryption_header(headers) .map_err(|e| ServiceError::Validation(ValidationError(e.to_string())))?; - let aad = stream_id_aad(basin.as_ref(), stream.as_ref()); + let aad = directive + .as_ref() + .map(|_| stream_id_aad(basin.as_ref(), stream.as_ref())); Ok(Self { aad, directive }) } + fn aad(&self) -> &[u8] { + self.aad.as_ref().map_or(&[], |a| a.as_slice()) + } + fn encrypt_input(&self, input: AppendInput) -> Result { match &self.directive { Some(EncryptionDirective::Key { alg, key }) => { - encryption::encrypt_append_input(input, *alg, key, &self.aad) + encryption::encrypt_append_input(input, *alg, key, self.aad()) } _ => Ok(input), } } fn decrypt_batch(&self, batch: ReadBatch) -> Result { - encryption::decrypt_read_batch(batch, self.directive.as_ref(), &self.aad) + encryption::decrypt_read_batch(batch, self.directive.as_ref(), self.aad()) } } From 6bad4c722c6c4431ced5133b39cbeb64bc8a0655 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Thu, 26 Mar 2026 14:05:24 +0530 Subject: [PATCH 26/42] fix: restore Debug derive on AppendPermit Co-Authored-By: Claude Opus 4.6 (1M context) --- lite/src/backend/streamer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lite/src/backend/streamer.rs b/lite/src/backend/streamer.rs index 767a4fbf..90ba047d 100644 --- a/lite/src/backend/streamer.rs +++ b/lite/src/backend/streamer.rs @@ -632,6 +632,7 @@ fn timestamp_now() -> Timestamp { .expect("Milliseconds since Unix epoch fits into a u64") } +#[derive(Debug)] pub struct AppendPermit<'a> { sema_permit: SemaphorePermit<'a>, msg_tx: &'a mpsc::UnboundedSender, From edd908804447254315d31592f3b102ece441de3f Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Thu, 26 Mar 2026 14:14:35 +0530 Subject: [PATCH 27/42] fix(cli): error when --encryption-algorithm is provided without a key Previously, --encryption-algorithm was silently ignored when no key was provided, which could lead users to believe encryption was enabled. Co-Authored-By: Claude Opus 4.6 (1M context) --- cli/src/main.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/src/main.rs b/cli/src/main.rs index d14592e7..cd37e4b1 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -822,6 +822,12 @@ fn resolve_encryption(args: &cli::EncryptionArgs) -> Result { + return Err(CliError::InvalidEncryptionKey( + "--encryption-algorithm requires --encryption-key or --encryption-key-file" + .to_owned(), + )); + } _ => return Ok(None), }; From 5f917c169ecbaea2524dbf6cc65f8c94e864fa9c Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Thu, 26 Mar 2026 14:14:53 +0530 Subject: [PATCH 28/42] , --- common/src/encryption.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index 582575a9..af0241af 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -166,8 +166,6 @@ pub fn decode_record_plaintext(bytes: Bytes) -> Result<(Vec
, Bytes), Enc .map_err(|e| EncryptionError::EncodingFailed(e.to_string())) } -/// Compute stream_id AAD: BLAKE3 hash of `basin ‖ 0x00 ‖ stream ‖ 0x00`. -/// Matches `StreamId::new(basin, stream)` in lite. pub fn stream_id_aad( basin: &(impl AsRef<[u8]> + ?Sized), stream: &(impl AsRef<[u8]> + ?Sized), From 82dd6b4efcc6a43c710deb362d6864e6b315f49f Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 27 Mar 2026 01:48:32 +0530 Subject: [PATCH 29/42] fix(cli): add encryption support to tail command Co-Authored-By: Claude Opus 4.6 (1M context) --- cli/src/cli.rs | 3 +++ cli/src/main.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 1c2fd90c..7144cd8c 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -564,6 +564,9 @@ pub struct TailArgs { /// Use "-" to write to stdout. #[arg(short = 'o', long, value_parser = parse_records_output_source, default_value = "-")] pub output: RecordsOut, + + #[command(flatten)] + pub encryption: EncryptionArgs, } #[derive(Args, Debug)] diff --git a/cli/src/main.rs b/cli/src/main.rs index cd37e4b1..ebde0df8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -804,6 +804,7 @@ fn resolve_command_encryption(command: &Command) -> Result &a.encryption, Command::Read(a) => &a.encryption, + Command::Tail(a) => &a.encryption, _ => return Ok(None), }; resolve_encryption(args) From 84eadc4b75a7e7d9649e94189e59b8b9f13df9ea Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 27 Mar 2026 02:31:29 +0530 Subject: [PATCH 30/42] refactor(cli): remove --encryption-algorithm from read/tail commands The algorithm is auto-detected from the alg_id byte in the ciphertext envelope during decryption, so specifying it on reads is unnecessary and confusing. Split EncryptionArgs into EncryptionArgs (append, with algorithm) and DecryptionArgs (read/tail, key only). Co-Authored-By: Claude Opus 4.6 (1M context) --- cli/src/cli.rs | 23 +++++++++++++-- cli/src/main.rs | 76 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 75 insertions(+), 24 deletions(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 7144cd8c..0a60b5ea 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -488,6 +488,25 @@ pub struct EncryptionArgs { pub encryption_attest: bool, } +#[derive(Args, Debug, Clone, Default)] +pub struct DecryptionArgs { + /// Hex-encoded 32-byte decryption key. Alternatively, set S2_ENCRYPTION_KEY env var. + #[arg( + long = "encryption-key", + env = "S2_ENCRYPTION_KEY", + hide_env_values = true + )] + pub encryption_key: Option, + + /// Read decryption key from file. + #[arg(long = "encryption-key-file", conflicts_with = "encryption_key")] + pub encryption_key_file: Option, + + /// Attest client-side encryption. + #[arg(long = "encryption-attest", conflicts_with_all = ["encryption_key", "encryption_key_file"])] + pub encryption_attest: bool, +} + #[derive(Args, Debug)] pub struct ReadArgs { /// S2 URI of the format: s2://{basin}/{stream} @@ -539,7 +558,7 @@ pub struct ReadArgs { pub output: RecordsOut, #[command(flatten)] - pub encryption: EncryptionArgs, + pub encryption: DecryptionArgs, } #[derive(Args, Debug)] @@ -566,7 +585,7 @@ pub struct TailArgs { pub output: RecordsOut, #[command(flatten)] - pub encryption: EncryptionArgs, + pub encryption: DecryptionArgs, } #[derive(Args, Debug)] diff --git a/cli/src/main.rs b/cli/src/main.rs index ebde0df8..2d7b4c4b 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -801,42 +801,57 @@ fn print_metrics(metrics: &[Metric]) { } fn resolve_command_encryption(command: &Command) -> Result, CliError> { - let args = match command { - Command::Append(a) => &a.encryption, - Command::Read(a) => &a.encryption, - Command::Tail(a) => &a.encryption, - _ => return Ok(None), - }; - resolve_encryption(args) -} - -fn resolve_encryption(args: &cli::EncryptionArgs) -> Result, CliError> { - if args.encryption_attest { - return Ok(Some(EncryptionConfig::Attest)); + match command { + Command::Append(a) => resolve_encryption(&a.encryption), + Command::Read(a) => resolve_decryption(&a.encryption), + Command::Tail(a) => resolve_decryption(&a.encryption), + _ => Ok(None), } +} - let key = match (&args.encryption_key, &args.encryption_key_file) { - (Some(k), _) => k.clone(), +fn resolve_key( + key: &Option, + key_file: &Option, +) -> Result, CliError> { + match (key, key_file) { + (Some(k), _) => Ok(Some(k.clone())), (_, Some(path)) => { let contents = std::fs::read_to_string(path).map_err(|e| { CliError::InvalidEncryptionKey(format!("cannot read key file: {e}")) })?; - contents.lines().next().unwrap_or("").trim().to_owned() + Ok(Some( + contents.lines().next().unwrap_or("").trim().to_owned(), + )) } - _ if args.encryption_algorithm.is_some() => { + _ => Ok(None), + } +} + +fn validate_hex_key(key: &str) -> Result<(), CliError> { + if key.len() != 64 || !key.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(CliError::InvalidEncryptionKey( + "key must be exactly 64 hex characters (32 bytes)".to_owned(), + )); + } + Ok(()) +} + +fn resolve_encryption(args: &cli::EncryptionArgs) -> Result, CliError> { + if args.encryption_attest { + return Ok(Some(EncryptionConfig::Attest)); + } + + let Some(key) = resolve_key(&args.encryption_key, &args.encryption_key_file)? else { + if args.encryption_algorithm.is_some() { return Err(CliError::InvalidEncryptionKey( "--encryption-algorithm requires --encryption-key or --encryption-key-file" .to_owned(), )); } - _ => return Ok(None), + return Ok(None); }; - if key.len() != 64 || !key.bytes().all(|b| b.is_ascii_hexdigit()) { - return Err(CliError::InvalidEncryptionKey( - "key must be exactly 64 hex characters (32 bytes)".to_owned(), - )); - } + validate_hex_key(&key)?; let alg = args .encryption_algorithm @@ -847,3 +862,20 @@ fn resolve_encryption(args: &cli::EncryptionArgs) -> Result Result, CliError> { + if args.encryption_attest { + return Ok(Some(EncryptionConfig::Attest)); + } + + let Some(key) = resolve_key(&args.encryption_key, &args.encryption_key_file)? else { + return Ok(None); + }; + + validate_hex_key(&key)?; + + Ok(Some(EncryptionConfig::Key { + alg: types::EncryptionAlgorithm::Aegis256.into(), + key: key.into(), + })) +} From 80072b2290bbf5e7ae801e7795c899da7a282c01 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 27 Mar 2026 03:14:20 +0530 Subject: [PATCH 31/42] refactor(cli): enforce --encryption-algorithm requires key via clap group Replace runtime check with clap's `requires = "encryption_key_source"` group constraint, so clap rejects the invalid combination at parse time. Co-Authored-By: Claude Opus 4.6 (1M context) --- cli/src/cli.rs | 16 +++++++++++++--- cli/src/main.rs | 6 ------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 0a60b5ea..91b168f6 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -470,17 +470,27 @@ pub struct AppendArgs { } #[derive(Args, Debug, Clone, Default)] +#[group(multiple = true)] pub struct EncryptionArgs { /// Hex-encoded 32-byte encryption key. Alternatively, set S2_ENCRYPTION_KEY env var. - #[arg(long, env = "S2_ENCRYPTION_KEY", hide_env_values = true)] + #[arg( + long, + env = "S2_ENCRYPTION_KEY", + hide_env_values = true, + group = "encryption_key_source" + )] pub encryption_key: Option, /// Read encryption key from file. - #[arg(long, conflicts_with = "encryption_key")] + #[arg( + long, + conflicts_with = "encryption_key", + group = "encryption_key_source" + )] pub encryption_key_file: Option, /// Encryption algorithm (default: aegis-256). - #[arg(long, value_enum)] + #[arg(long, value_enum, requires = "encryption_key_source")] pub encryption_algorithm: Option, /// Attest client-side encryption. diff --git a/cli/src/main.rs b/cli/src/main.rs index 2d7b4c4b..dfa9deba 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -842,12 +842,6 @@ fn resolve_encryption(args: &cli::EncryptionArgs) -> Result Date: Fri, 27 Mar 2026 10:53:04 +0530 Subject: [PATCH 32/42] better --- common/src/encryption.rs | 143 +++++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 42 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index af0241af..9b47b77c 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -1,25 +1,25 @@ -//! # Ciphertext format +//! # Record encryption format +//! +//! AAD = stream_id bytes //! //! ```text //! [version: 1 byte] [alg_id: 1 byte] [nonce] [ciphertext] [tag] //! ``` //! -//! | version | Description | -//! |---------|--------------------------------------------------| -//! | 0x01 | Initial versioned format. AAD = stream_id bytes | -//! //! | alg_id | Algorithm | Nonce | Tag | //! |--------|-------------|--------|------| //! | 0x01 | AEGIS-256 | 32 B | 32 B | //! | 0x02 | AES-256-GCM | 12 B | 16 B | +use core::str::FromStr; + use aegis::aegis256::Aegis256; use aes_gcm::{ Aes256Gcm, KeyInit, aead::{Aead, Payload}, }; use bytes::{BufMut, Bytes, BytesMut}; -use http::HeaderMap; +use http::{HeaderMap, HeaderValue}; use rand::random; use secrecy::{CloneableSecret, ExposeSecret, SecretBox}; @@ -63,12 +63,17 @@ fn make_key(bytes: [u8; 32]) -> EncryptionKey { SecretBox::new(Box::new(KeyBytes(bytes))) } +/// Parsed `s2-encryption` request directive. #[derive(Clone, Debug)] pub enum EncryptionDirective { + /// Encrypt and decrypt record bodies with the provided AEAD algorithm and key. Key { + /// AEAD algorithm to use. alg: EncryptionAlgorithm, + /// 32-byte symmetric key. key: EncryptionKey, }, + /// Use attestation-based encryption mode instead of a caller-supplied key. Attest, } @@ -84,44 +89,83 @@ pub enum EncryptionError { EncodingFailed(String), } -pub fn parse_s2_encryption_header( - headers: &HeaderMap, -) -> Result, EncryptionError> { - let value = match headers.get(S2_ENCRYPTION_HEADER) { - Some(v) => v, - None => return Ok(None), - }; - - let header_str = value - .to_str() - .map_err(|_| EncryptionError::MalformedHeader("header is not valid UTF-8".to_owned()))?; +impl TryFrom<&HeaderValue> for EncryptionDirective { + type Error = EncryptionError; - if header_str.trim() == "attest" { - return Ok(Some(EncryptionDirective::Attest)); + fn try_from(value: &HeaderValue) -> Result { + value + .to_str() + .map_err(|_| EncryptionError::MalformedHeader("header is not valid UTF-8".to_owned()))? + .parse() } +} - let (alg_part, key_part) = header_str.split_once(';').ok_or_else(|| { - EncryptionError::MalformedHeader("expected 'alg=...; key=...'".to_owned()) - })?; +impl FromStr for EncryptionDirective { + type Err = EncryptionError; - let alg_str = alg_part - .trim() - .strip_prefix("alg=") - .ok_or_else(|| EncryptionError::MalformedHeader("missing 'alg=' prefix".to_owned()))? - .trim(); + fn from_str(s: &str) -> Result { + let s = s.trim(); + if s == "attest" { + return Ok(Self::Attest); + } - let key_hex = key_part - .trim() - .strip_prefix("key=") - .ok_or_else(|| EncryptionError::MalformedHeader("missing 'key=' prefix".to_owned()))? - .trim(); + let mut alg_str = None; + let mut key_hex = None; + for part in s.split(';') { + let (name, value) = part.split_once('=').ok_or_else(|| { + EncryptionError::MalformedHeader("expected 'alg=...; key=...'".to_owned()) + })?; + let name = name.trim(); + let value = value.trim(); + match name { + "alg" => { + if alg_str.replace(value).is_some() { + return Err(EncryptionError::MalformedHeader( + "duplicate 'alg=' parameter".to_owned(), + )); + } + } + "key" => { + if key_hex.replace(value).is_some() { + return Err(EncryptionError::MalformedHeader( + "duplicate 'key=' parameter".to_owned(), + )); + } + } + _ => { + return Err(EncryptionError::MalformedHeader(format!( + "unknown parameter {name:?}; expected 'alg' or 'key'" + ))); + } + } + } - let alg: EncryptionAlgorithm = alg_str.parse().map_err(|_| { - EncryptionError::MalformedHeader(format!( - "unknown algorithm {alg_str:?}; expected 'aegis-256' or 'aes-256-gcm'" - )) - })?; + let alg_str = alg_str.ok_or_else(|| { + EncryptionError::MalformedHeader("missing 'alg=' parameter".to_owned()) + })?; + let key_hex = key_hex.ok_or_else(|| { + EncryptionError::MalformedHeader("missing 'key=' parameter".to_owned()) + })?; + let alg: EncryptionAlgorithm = alg_str.parse().map_err(|_| { + EncryptionError::MalformedHeader(format!( + "unknown algorithm {alg_str:?}; expected 'aegis-256' or 'aes-256-gcm'" + )) + })?; + let key = parse_encryption_key(key_hex)?; + Ok(Self::Key { alg, key }) + } +} +pub fn parse_s2_encryption_header( + headers: &HeaderMap, +) -> Result, EncryptionError> { + headers + .get(S2_ENCRYPTION_HEADER) + .map(EncryptionDirective::try_from) + .transpose() +} + +fn parse_encryption_key(key_hex: &str) -> Result { if key_hex.len() != 64 { return Err(EncryptionError::MalformedHeader(format!( "key must be 64 hex characters (32 bytes), got {} characters", @@ -144,11 +188,7 @@ pub fn parse_s2_encryption_header( )); } }; - - Ok(Some(EncryptionDirective::Key { - alg, - key: make_key(key_array), - })) + Ok(make_key(key_array)) } pub fn encode_record_plaintext( @@ -557,6 +597,25 @@ mod tests { )); } + #[test] + fn parse_header_valid_reordered_params() { + let mut headers = HeaderMap::new(); + headers.insert( + S2_ENCRYPTION_HEADER, + http::HeaderValue::from_static( + "key=0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20; alg=aes-256-gcm", + ), + ); + let directive = parse_s2_encryption_header(&headers).unwrap().unwrap(); + assert!(matches!( + directive, + EncryptionDirective::Key { + alg: EncryptionAlgorithm::Aes256Gcm, + .. + } + )); + } + #[test] fn parse_header_attest() { let mut headers = HeaderMap::new(); From 820efcba6448bdf720148e209ea74f1fad531d17 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 27 Mar 2026 12:58:05 +0530 Subject: [PATCH 33/42] . --- lite/src/handlers/v1/error.rs | 6 ------ lite/src/handlers/v1/records.rs | 6 ++---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/lite/src/handlers/v1/error.rs b/lite/src/handlers/v1/error.rs index 1c97dd45..43f5ae84 100644 --- a/lite/src/handlers/v1/error.rs +++ b/lite/src/handlers/v1/error.rs @@ -64,12 +64,6 @@ pub enum ServiceError { NotImplemented, } -impl From for ServiceError { - fn from(never: std::convert::Infallible) -> Self { - match never {} - } -} - impl From for ServiceError { fn from(value: AppendRequestRejection) -> Self { match value { diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index b43fb4d9..e56b45d2 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -169,7 +169,6 @@ pub async fn check_tail( #[derive(FromRequest)] #[from_request(rejection(ServiceError))] pub struct ReadArgs { - headers: http::HeaderMap, #[from_request(via(Header))] basin: BasinName, #[from_request(via(Path))] @@ -214,8 +213,8 @@ pub struct ReadArgs { ))] pub async fn read( State(backend): State, + headers: http::HeaderMap, ReadArgs { - headers, basin, stream, start, @@ -377,7 +376,6 @@ async fn merge_read_session( #[derive(FromRequest)] #[from_request(rejection(ServiceError))] pub struct AppendArgs { - headers: http::HeaderMap, #[from_request(via(Header))] basin: BasinName, #[from_request(via(Path))] @@ -411,8 +409,8 @@ pub struct AppendArgs { ))] pub async fn append( State(backend): State, + headers: http::HeaderMap, AppendArgs { - headers, basin, stream, request, From dac2431f82e0aefa66d91f2e803c105c37dc6b82 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 27 Mar 2026 20:11:55 +0530 Subject: [PATCH 34/42] refactor: make encryption algorithm optional in directives Algorithm is only needed for encryption (appends). For decryption (reads/tail), the alg_id byte in the ciphertext envelope is authoritative. Making alg Optional removes the need to hardcode a dummy algorithm on the decrypt path. - EncryptionDirective::Key.alg: Option - SDK EncryptionConfig::Key.alg: Option - Header format supports key-only: "key=" (no alg) - Lite handler requires alg on append, ignores on read - CLI resolve_decryption passes alg: None Co-Authored-By: Claude Opus 4.6 (1M context) --- cli/src/main.rs | 4 +-- common/src/encryption.rs | 47 +++++++++++++++++++++++---------- lite/src/handlers/v1/records.rs | 7 ++++- sdk/src/api.rs | 8 +++++- sdk/src/types.rs | 6 ++--- 5 files changed, 51 insertions(+), 21 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index dfa9deba..34c7d50f 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -852,7 +852,7 @@ fn resolve_encryption(args: &cli::EncryptionArgs) -> Result Result EncryptionKey { /// Parsed `s2-encryption` request directive. #[derive(Clone, Debug)] pub enum EncryptionDirective { - /// Encrypt and decrypt record bodies with the provided AEAD algorithm and key. + /// Encrypt/decrypt record bodies with the provided key. + /// Algorithm is required for encryption (appends), optional for decryption + /// (reads auto-detect from the ciphertext envelope). Key { - /// AEAD algorithm to use. - alg: EncryptionAlgorithm, + /// AEAD algorithm. Required for appends, ignored for reads. + alg: Option, /// 32-byte symmetric key. key: EncryptionKey, }, @@ -140,17 +142,18 @@ impl FromStr for EncryptionDirective { } } - let alg_str = alg_str.ok_or_else(|| { - EncryptionError::MalformedHeader("missing 'alg=' parameter".to_owned()) - })?; let key_hex = key_hex.ok_or_else(|| { EncryptionError::MalformedHeader("missing 'key=' parameter".to_owned()) })?; - let alg: EncryptionAlgorithm = alg_str.parse().map_err(|_| { - EncryptionError::MalformedHeader(format!( - "unknown algorithm {alg_str:?}; expected 'aegis-256' or 'aes-256-gcm'" - )) - })?; + let alg = alg_str + .map(|s| { + s.parse::().map_err(|_| { + EncryptionError::MalformedHeader(format!( + "unknown algorithm {s:?}; expected 'aegis-256' or 'aes-256-gcm'" + )) + }) + }) + .transpose()?; let key = parse_encryption_key(key_hex)?; Ok(Self::Key { alg, key }) } @@ -572,7 +575,7 @@ mod tests { assert!(matches!( directive, EncryptionDirective::Key { - alg: EncryptionAlgorithm::Aegis256, + alg: Some(EncryptionAlgorithm::Aegis256), .. } )); @@ -591,7 +594,7 @@ mod tests { assert!(matches!( directive, EncryptionDirective::Key { - alg: EncryptionAlgorithm::Aes256Gcm, + alg: Some(EncryptionAlgorithm::Aes256Gcm), .. } )); @@ -610,12 +613,28 @@ mod tests { assert!(matches!( directive, EncryptionDirective::Key { - alg: EncryptionAlgorithm::Aes256Gcm, + alg: Some(EncryptionAlgorithm::Aes256Gcm), .. } )); } + #[test] + fn parse_header_key_only() { + let mut headers = HeaderMap::new(); + headers.insert( + S2_ENCRYPTION_HEADER, + http::HeaderValue::from_static( + "key=0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", + ), + ); + let directive = parse_s2_encryption_header(&headers).unwrap().unwrap(); + assert!(matches!( + directive, + EncryptionDirective::Key { alg: None, .. } + )); + } + #[test] fn parse_header_attest() { let mut headers = HeaderMap::new(); diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index e56b45d2..21b0fb98 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -68,7 +68,12 @@ impl EncryptionContext { fn encrypt_input(&self, input: AppendInput) -> Result { match &self.directive { Some(EncryptionDirective::Key { alg, key }) => { - encryption::encrypt_append_input(input, *alg, key, self.aad()) + let alg = alg.ok_or_else(|| { + EncryptionError::MalformedHeader( + "encryption algorithm required for append".to_owned(), + ) + })?; + encryption::encrypt_append_input(input, alg, key, self.aad()) } _ => Ok(input), } diff --git a/sdk/src/api.rs b/sdk/src/api.rs index 13d58705..49010de2 100644 --- a/sdk/src/api.rs +++ b/sdk/src/api.rs @@ -842,9 +842,15 @@ impl BaseClient { .map(|enc| { use crate::types::EncryptionConfig; let value = match enc { - EncryptionConfig::Key { alg, key } => { + EncryptionConfig::Key { + alg: Some(alg), + key, + } => { format!("alg={alg}; key={}", key.expose_secret()) } + EncryptionConfig::Key { alg: None, key } => { + format!("key={}", key.expose_secret()) + } EncryptionConfig::Attest => "attest".to_owned(), }; value.try_into() diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 27d0ea72..60207af5 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -388,10 +388,10 @@ impl RetryConfig { /// Encryption configuration. #[derive(Clone)] pub enum EncryptionConfig { - /// Algorithm and key. + /// Algorithm and key for encryption, or key-only for decryption. Key { - /// Encryption algorithm. - alg: EncryptionAlgorithm, + /// Encryption algorithm. Required for appends, ignored for reads + alg: Option, /// Hex-encoded 32-byte key. key: SecretString, }, From ec37e4ea9bb30cb9582a7fe0c83199602333c176 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 27 Mar 2026 20:17:24 +0530 Subject: [PATCH 35/42] . --- common/src/encryption.rs | 10 ++++------ sdk/src/types.rs | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/common/src/encryption.rs b/common/src/encryption.rs index 04b8fdc1..ede19f97 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -66,16 +66,14 @@ fn make_key(bytes: [u8; 32]) -> EncryptionKey { /// Parsed `s2-encryption` request directive. #[derive(Clone, Debug)] pub enum EncryptionDirective { - /// Encrypt/decrypt record bodies with the provided key. - /// Algorithm is required for encryption (appends), optional for decryption - /// (reads auto-detect from the ciphertext envelope). + /// Algorithm and key for encryption, or key-only for decryption. Key { - /// AEAD algorithm. Required for appends, ignored for reads. + /// Encryption algorithm. Required for appends, ignored for reads. alg: Option, - /// 32-byte symmetric key. + /// Hex-encoded 32-byte key. key: EncryptionKey, }, - /// Use attestation-based encryption mode instead of a caller-supplied key. + /// Attest mode. Attest, } diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 60207af5..5a1f30f7 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -390,7 +390,7 @@ impl RetryConfig { pub enum EncryptionConfig { /// Algorithm and key for encryption, or key-only for decryption. Key { - /// Encryption algorithm. Required for appends, ignored for reads + /// Encryption algorithm. Required for appends, ignored for reads. alg: Option, /// Hex-encoded 32-byte key. key: SecretString, From 4f7b9b187beddb679945ac1e8b1e0165f17330ef Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Fri, 27 Mar 2026 20:42:12 +0530 Subject: [PATCH 36/42] chore: deduplicate resolve_encryption/resolve_decryption, rename test helpers - Extract shared resolve_encryption_config to eliminate duplication - Rename make_key_fn/make_wrong_key_fn to test_key/wrong_test_key Co-Authored-By: Claude Opus 4.6 (1M context) --- cli/src/main.rs | 41 ++++++++++++++++++++++------------------ common/src/encryption.rs | 28 +++++++++++++-------------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 34c7d50f..1808d37e 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -837,39 +837,44 @@ fn validate_hex_key(key: &str) -> Result<(), CliError> { } fn resolve_encryption(args: &cli::EncryptionArgs) -> Result, CliError> { - if args.encryption_attest { - return Ok(Some(EncryptionConfig::Attest)); - } - - let Some(key) = resolve_key(&args.encryption_key, &args.encryption_key_file)? else { - return Ok(None); - }; - - validate_hex_key(&key)?; - let alg = args .encryption_algorithm .unwrap_or(types::EncryptionAlgorithm::Aegis256); - - Ok(Some(EncryptionConfig::Key { - alg: Some(alg.into()), - key: key.into(), - })) + resolve_encryption_config( + args.encryption_attest, + &args.encryption_key, + &args.encryption_key_file, + Some(alg.into()), + ) } fn resolve_decryption(args: &cli::DecryptionArgs) -> Result, CliError> { - if args.encryption_attest { + resolve_encryption_config( + args.encryption_attest, + &args.encryption_key, + &args.encryption_key_file, + None, + ) +} + +fn resolve_encryption_config( + attest: bool, + key: &Option, + key_file: &Option, + alg: Option, +) -> Result, CliError> { + if attest { return Ok(Some(EncryptionConfig::Attest)); } - let Some(key) = resolve_key(&args.encryption_key, &args.encryption_key_file)? else { + let Some(key) = resolve_key(key, key_file)? else { return Ok(None); }; validate_hex_key(&key)?; Ok(Some(EncryptionConfig::Key { - alg: None, + alg, key: key.into(), })) } diff --git a/common/src/encryption.rs b/common/src/encryption.rs index ede19f97..2cc6dfd4 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -412,11 +412,11 @@ pub fn decrypt_read_batch( mod tests { use super::*; - fn make_key_fn() -> EncryptionKey { + fn test_key() -> EncryptionKey { make_key([0x42u8; 32]) } - fn make_wrong_key_fn() -> EncryptionKey { + fn wrong_test_key() -> EncryptionKey { make_key([0x99u8; 32]) } @@ -433,7 +433,7 @@ mod tests { let aad = test_aad(); let plaintext = encode_record_plaintext(headers.clone(), body.clone()).unwrap(); - let key = make_key_fn(); + let key = test_key(); let ciphertext = encrypt_record(&plaintext, alg, &key, &aad).unwrap(); let decrypted = decrypt_record(&ciphertext, &key, &aad).unwrap(); let (out_headers, out_body) = decode_record_plaintext(decrypted).unwrap(); @@ -456,10 +456,10 @@ mod tests { fn wrong_key_fails_aegis256() { let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); - let key = make_key_fn(); + let key = test_key(); let ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad).unwrap(); - let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), &aad); + let result = decrypt_record(&ciphertext, &wrong_test_key(), &aad); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } @@ -467,10 +467,10 @@ mod tests { fn wrong_key_fails_aes256gcm() { let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); - let key = make_key_fn(); + let key = test_key(); let ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aes256Gcm, &key, &aad).unwrap(); - let result = decrypt_record(&ciphertext, &make_wrong_key_fn(), &aad); + let result = decrypt_record(&ciphertext, &wrong_test_key(), &aad); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } @@ -478,7 +478,7 @@ mod tests { fn truncated_ciphertext_fails_no_panic() { let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); - let key = make_key_fn(); + let key = test_key(); let ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad).unwrap(); let truncated = &ciphertext[..4]; @@ -489,7 +489,7 @@ mod tests { #[test] fn unsupported_version_fails() { let aad = test_aad(); - let key = make_key_fn(); + let key = test_key(); let body = b"\xFFsome opaque bytes"; let result = decrypt_record(body, &key, &aad); assert!(matches!( @@ -501,7 +501,7 @@ mod tests { #[test] fn empty_body_fails() { let aad = test_aad(); - let key = make_key_fn(); + let key = test_key(); let result = decrypt_record(b"", &key, &aad); assert!(matches!(result, Err(EncryptionError::DecryptionFailed))); } @@ -510,7 +510,7 @@ mod tests { fn version_byte_present() { let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); - let key = make_key_fn(); + let key = test_key(); let ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad).unwrap(); assert_eq!(ciphertext[0], CIPHERTEXT_V1); @@ -521,7 +521,7 @@ mod tests { fn alg_id_flip_detected() { let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); - let key = make_key_fn(); + let key = test_key(); let mut ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad) .unwrap() .to_vec(); @@ -536,7 +536,7 @@ mod tests { fn version_flip_detected() { let aad = test_aad(); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); - let key = make_key_fn(); + let key = test_key(); let mut ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad) .unwrap() .to_vec(); @@ -553,7 +553,7 @@ mod tests { let aad = test_aad(); let other_aad = stream_id_aad("other-basin", "other-stream"); let plaintext = encode_record_plaintext(vec![], Bytes::from_static(b"data")).unwrap(); - let key = make_key_fn(); + let key = test_key(); let ciphertext = encrypt_record(&plaintext, EncryptionAlgorithm::Aegis256, &key, &aad).unwrap(); let result = decrypt_record(&ciphertext, &key, &other_aad); From 2967e1063bf91a4fdf7a0b706bf86ec1a40561b3 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 31 Mar 2026 00:52:18 +0530 Subject: [PATCH 37/42] sbcr --- Cargo.lock | 2 +- cli/src/cli.rs | 4 ++-- cli/src/error.rs | 2 +- cli/src/main.rs | 16 ++++++++----- common/Cargo.toml | 2 +- common/src/encryption.rs | 49 +++++++++++++++++----------------------- sdk/src/types.rs | 2 +- 7 files changed, 37 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa0c02ec..1a1b0e1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4728,13 +4728,13 @@ dependencies = [ "aegis", "aes-gcm", "axum", + "base64ct", "blake3", "bytes", "clap", "compact_str", "enum-ordinalize", "enumset", - "hex", "http 1.4.0", "proptest", "rand 0.10.0", diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 91b168f6..967691a4 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -472,7 +472,7 @@ pub struct AppendArgs { #[derive(Args, Debug, Clone, Default)] #[group(multiple = true)] pub struct EncryptionArgs { - /// Hex-encoded 32-byte encryption key. Alternatively, set S2_ENCRYPTION_KEY env var. + /// Base64-encoded 32-byte encryption key. Alternatively, set S2_ENCRYPTION_KEY env var. #[arg( long, env = "S2_ENCRYPTION_KEY", @@ -500,7 +500,7 @@ pub struct EncryptionArgs { #[derive(Args, Debug, Clone, Default)] pub struct DecryptionArgs { - /// Hex-encoded 32-byte decryption key. Alternatively, set S2_ENCRYPTION_KEY env var. + /// Base64-encoded 32-byte decryption key. Alternatively, set S2_ENCRYPTION_KEY env var. #[arg( long = "encryption-key", env = "S2_ENCRYPTION_KEY", diff --git a/cli/src/error.rs b/cli/src/error.rs index 41c2db9b..c2ca20cd 100644 --- a/cli/src/error.rs +++ b/cli/src/error.rs @@ -67,7 +67,7 @@ pub enum CliError { OperationWithTokenSource(OpKind, #[source] S2Error, TokenSource), #[error("Invalid encryption key: {0}")] - #[diagnostic(help("Key must be exactly 64 hex characters (32 bytes)."))] + #[diagnostic(help("Key must be a base64-encoded 32-byte value (44 characters with padding)."))] InvalidEncryptionKey(String), #[error("S2 Lite server error: {0}")] diff --git a/cli/src/main.rs b/cli/src/main.rs index 1808d37e..cae26429 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -827,11 +827,15 @@ fn resolve_key( } } -fn validate_hex_key(key: &str) -> Result<(), CliError> { - if key.len() != 64 || !key.bytes().all(|b| b.is_ascii_hexdigit()) { - return Err(CliError::InvalidEncryptionKey( - "key must be exactly 64 hex characters (32 bytes)".to_owned(), - )); +fn validate_base64_key(key: &str) -> Result<(), CliError> { + use base64ct::{Base64, Encoding}; + let bytes = Base64::decode_vec(key) + .map_err(|_| CliError::InvalidEncryptionKey("key is not valid base64".to_owned()))?; + if bytes.len() != 32 { + return Err(CliError::InvalidEncryptionKey(format!( + "key must be exactly 32 bytes, got {} bytes", + bytes.len() + ))); } Ok(()) } @@ -871,7 +875,7 @@ fn resolve_encryption_config( return Ok(None); }; - validate_hex_key(&key)?; + validate_base64_key(&key)?; Ok(Some(EncryptionConfig::Key { alg, diff --git a/common/Cargo.toml b/common/Cargo.toml index 70bbf098..7d0ddf8a 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -24,7 +24,7 @@ clap = { workspace = true, optional = true, features = ["derive"] } compact_str = { workspace = true, features = ["serde"] } enum-ordinalize = { workspace = true } enumset = { workspace = true } -hex = { workspace = true } +base64ct = { workspace = true, features = ["alloc"] } http = { workspace = true } rand = { workspace = true } rkyv = { workspace = true, optional = true } diff --git a/common/src/encryption.rs b/common/src/encryption.rs index 2cc6dfd4..57f40855 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -70,7 +70,7 @@ pub enum EncryptionDirective { Key { /// Encryption algorithm. Required for appends, ignored for reads. alg: Option, - /// Hex-encoded 32-byte key. + /// Base64-encoded 32-byte key. key: EncryptionKey, }, /// Attest mode. @@ -110,7 +110,7 @@ impl FromStr for EncryptionDirective { } let mut alg_str = None; - let mut key_hex = None; + let mut key_b64 = None; for part in s.split(';') { let (name, value) = part.split_once('=').ok_or_else(|| { EncryptionError::MalformedHeader("expected 'alg=...; key=...'".to_owned()) @@ -126,7 +126,7 @@ impl FromStr for EncryptionDirective { } } "key" => { - if key_hex.replace(value).is_some() { + if key_b64.replace(value).is_some() { return Err(EncryptionError::MalformedHeader( "duplicate 'key=' parameter".to_owned(), )); @@ -140,7 +140,7 @@ impl FromStr for EncryptionDirective { } } - let key_hex = key_hex.ok_or_else(|| { + let key_b64 = key_b64.ok_or_else(|| { EncryptionError::MalformedHeader("missing 'key=' parameter".to_owned()) })?; let alg = alg_str @@ -152,7 +152,7 @@ impl FromStr for EncryptionDirective { }) }) .transpose()?; - let key = parse_encryption_key(key_hex)?; + let key = parse_encryption_key(key_b64)?; Ok(Self::Key { alg, key }) } } @@ -166,16 +166,11 @@ pub fn parse_s2_encryption_header( .transpose() } -fn parse_encryption_key(key_hex: &str) -> Result { - if key_hex.len() != 64 { - return Err(EncryptionError::MalformedHeader(format!( - "key must be 64 hex characters (32 bytes), got {} characters", - key_hex.len() - ))); - } +fn parse_encryption_key(key_b64: &str) -> Result { + use base64ct::{Base64, Encoding}; - let mut key_bytes: Vec = hex::decode(key_hex) - .map_err(|e| EncryptionError::MalformedHeader(format!("key is not valid hex: {e}")))?; + let mut key_bytes: Vec = Base64::decode_vec(key_b64) + .map_err(|e| EncryptionError::MalformedHeader(format!("key is not valid base64: {e}")))?; let key_array: [u8; 32] = match key_bytes.as_slice().try_into() { Ok(arr) => { @@ -183,10 +178,11 @@ fn parse_encryption_key(key_hex: &str) -> Result arr } Err(_) => { + let len = key_bytes.len(); secrecy::zeroize::Zeroize::zeroize(&mut key_bytes); - return Err(EncryptionError::MalformedHeader( - "key must be exactly 32 bytes".to_owned(), - )); + return Err(EncryptionError::MalformedHeader(format!( + "key must be exactly 32 bytes, got {len} bytes" + ))); } }; Ok(make_key(key_array)) @@ -566,7 +562,7 @@ mod tests { headers.insert( S2_ENCRYPTION_HEADER, http::HeaderValue::from_static( - "alg=aegis-256; key=0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", + "alg=aegis-256; key=AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=", ), ); let directive = parse_s2_encryption_header(&headers).unwrap().unwrap(); @@ -585,7 +581,7 @@ mod tests { headers.insert( S2_ENCRYPTION_HEADER, http::HeaderValue::from_static( - "alg=aes-256-gcm; key=0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", + "alg=aes-256-gcm; key=AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=", ), ); let directive = parse_s2_encryption_header(&headers).unwrap().unwrap(); @@ -604,7 +600,7 @@ mod tests { headers.insert( S2_ENCRYPTION_HEADER, http::HeaderValue::from_static( - "key=0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20; alg=aes-256-gcm", + "key=AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=; alg=aes-256-gcm", ), ); let directive = parse_s2_encryption_header(&headers).unwrap().unwrap(); @@ -622,9 +618,7 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert( S2_ENCRYPTION_HEADER, - http::HeaderValue::from_static( - "key=0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", - ), + http::HeaderValue::from_static("key=AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="), ); let directive = parse_s2_encryption_header(&headers).unwrap().unwrap(); assert!(matches!( @@ -665,22 +659,21 @@ mod tests { #[test] fn parse_header_wrong_key_length() { let mut headers = HeaderMap::new(); + // "deadbeef" decodes to 4 bytes, not 32 headers.insert( S2_ENCRYPTION_HEADER, - http::HeaderValue::from_static("alg=aegis-256; key=deadbeef"), + http::HeaderValue::from_static("alg=aegis-256; key=3q2+7w=="), ); let result = parse_s2_encryption_header(&headers); assert!(matches!(result, Err(EncryptionError::MalformedHeader(_)))); } #[test] - fn parse_header_invalid_hex() { + fn parse_header_invalid_base64() { let mut headers = HeaderMap::new(); headers.insert( S2_ENCRYPTION_HEADER, - http::HeaderValue::from_static( - "alg=aegis-256; key=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", - ), + http::HeaderValue::from_static("alg=aegis-256; key=not-valid-base64!!!"), ); let result = parse_s2_encryption_header(&headers); assert!(matches!(result, Err(EncryptionError::MalformedHeader(_)))); diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 5a1f30f7..aad45b69 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -392,7 +392,7 @@ pub enum EncryptionConfig { Key { /// Encryption algorithm. Required for appends, ignored for reads. alg: Option, - /// Hex-encoded 32-byte key. + /// Base64-encoded 32-byte key. key: SecretString, }, /// Attest mode. From ef57351a3cc91760919521b53dc1f6c324dac54a Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 31 Mar 2026 01:10:33 +0530 Subject: [PATCH 38/42] . --- cli/src/cli.rs | 6 +----- cli/src/main.rs | 18 ++++++++---------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 967691a4..46fe8626 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -489,7 +489,7 @@ pub struct EncryptionArgs { )] pub encryption_key_file: Option, - /// Encryption algorithm (default: aegis-256). + /// Encryption algorithm. #[arg(long, value_enum, requires = "encryption_key_source")] pub encryption_algorithm: Option, @@ -511,10 +511,6 @@ pub struct DecryptionArgs { /// Read decryption key from file. #[arg(long = "encryption-key-file", conflicts_with = "encryption_key")] pub encryption_key_file: Option, - - /// Attest client-side encryption. - #[arg(long = "encryption-attest", conflicts_with_all = ["encryption_key", "encryption_key_file"])] - pub encryption_attest: bool, } #[derive(Args, Debug)] diff --git a/cli/src/main.rs b/cli/src/main.rs index cae26429..1fb69d2d 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -841,24 +841,22 @@ fn validate_base64_key(key: &str) -> Result<(), CliError> { } fn resolve_encryption(args: &cli::EncryptionArgs) -> Result, CliError> { - let alg = args - .encryption_algorithm - .unwrap_or(types::EncryptionAlgorithm::Aegis256); + let has_key = args.encryption_key.is_some() || args.encryption_key_file.is_some(); + if has_key && args.encryption_algorithm.is_none() { + return Err(CliError::InvalidEncryptionKey( + "--encryption-algorithm is required when encrypting".to_owned(), + )); + } resolve_encryption_config( args.encryption_attest, &args.encryption_key, &args.encryption_key_file, - Some(alg.into()), + args.encryption_algorithm.map(Into::into), ) } fn resolve_decryption(args: &cli::DecryptionArgs) -> Result, CliError> { - resolve_encryption_config( - args.encryption_attest, - &args.encryption_key, - &args.encryption_key_file, - None, - ) + resolve_encryption_config(false, &args.encryption_key, &args.encryption_key_file, None) } fn resolve_encryption_config( From 961cd752197f98aa4a8f8c34768835d6a70ca333 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 31 Mar 2026 02:20:03 +0530 Subject: [PATCH 39/42] . --- cli/src/cli.rs | 6 ++++-- cli/src/main.rs | 6 ------ common/src/encryption.rs | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 46fe8626..c5584cbe 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -477,7 +477,8 @@ pub struct EncryptionArgs { long, env = "S2_ENCRYPTION_KEY", hide_env_values = true, - group = "encryption_key_source" + group = "encryption_key_source", + requires = "encryption_algorithm" )] pub encryption_key: Option, @@ -485,7 +486,8 @@ pub struct EncryptionArgs { #[arg( long, conflicts_with = "encryption_key", - group = "encryption_key_source" + group = "encryption_key_source", + requires = "encryption_algorithm" )] pub encryption_key_file: Option, diff --git a/cli/src/main.rs b/cli/src/main.rs index 1fb69d2d..8f0bcd16 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -841,12 +841,6 @@ fn validate_base64_key(key: &str) -> Result<(), CliError> { } fn resolve_encryption(args: &cli::EncryptionArgs) -> Result, CliError> { - let has_key = args.encryption_key.is_some() || args.encryption_key_file.is_some(); - if has_key && args.encryption_algorithm.is_none() { - return Err(CliError::InvalidEncryptionKey( - "--encryption-algorithm is required when encrypting".to_owned(), - )); - } resolve_encryption_config( args.encryption_attest, &args.encryption_key, diff --git a/common/src/encryption.rs b/common/src/encryption.rs index 57f40855..812bf5b5 100644 --- a/common/src/encryption.rs +++ b/common/src/encryption.rs @@ -70,7 +70,7 @@ pub enum EncryptionDirective { Key { /// Encryption algorithm. Required for appends, ignored for reads. alg: Option, - /// Base64-encoded 32-byte key. + /// Decoded 32-byte encryption key. key: EncryptionKey, }, /// Attest mode. From 36d8109e28e335e52c6a18307937f78dd9707582 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 31 Mar 2026 19:34:18 +0530 Subject: [PATCH 40/42] sort --- common/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/Cargo.toml b/common/Cargo.toml index 7d0ddf8a..e24e5417 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -18,13 +18,13 @@ utoipa = ["dep:utoipa"] aegis = { workspace = true } aes-gcm = { workspace = true } axum = { workspace = true, optional = true } +base64ct = { workspace = true, features = ["alloc"] } blake3 = { workspace = true } bytes = { workspace = true } clap = { workspace = true, optional = true, features = ["derive"] } compact_str = { workspace = true, features = ["serde"] } enum-ordinalize = { workspace = true } enumset = { workspace = true } -base64ct = { workspace = true, features = ["alloc"] } http = { workspace = true } rand = { workspace = true } rkyv = { workspace = true, optional = true } From 9fdaf223bc4bc9e593ec7a3d2683599a1b79f445 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 31 Mar 2026 19:37:37 +0530 Subject: [PATCH 41/42] . --- cli/src/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/error.rs b/cli/src/error.rs index c2ca20cd..d4e8f7f6 100644 --- a/cli/src/error.rs +++ b/cli/src/error.rs @@ -67,7 +67,7 @@ pub enum CliError { OperationWithTokenSource(OpKind, #[source] S2Error, TokenSource), #[error("Invalid encryption key: {0}")] - #[diagnostic(help("Key must be a base64-encoded 32-byte value (44 characters with padding)."))] + #[diagnostic(help("Key must be a base64-encoded 32-byte value."))] InvalidEncryptionKey(String), #[error("S2 Lite server error: {0}")] From 4f89d789b15aa6eeacdbe6bc6f3216cb736c2187 Mon Sep 17 00:00:00 2001 From: Mehul Arora Date: Tue, 31 Mar 2026 20:29:45 +0530 Subject: [PATCH 42/42] . --- sdk/src/types.rs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/sdk/src/types.rs b/sdk/src/types.rs index aad45b69..7f75f271 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -386,7 +386,7 @@ impl RetryConfig { } /// Encryption configuration. -#[derive(Clone)] +#[derive(Debug, Clone)] pub enum EncryptionConfig { /// Algorithm and key for encryption, or key-only for decryption. Key { @@ -399,19 +399,6 @@ pub enum EncryptionConfig { Attest, } -impl fmt::Debug for EncryptionConfig { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Key { alg, .. } => f - .debug_struct("Key") - .field("alg", alg) - .field("key", &"[REDACTED]") - .finish(), - Self::Attest => write!(f, "Attest"), - } - } -} - #[derive(Debug, Clone)] #[non_exhaustive] /// Configuration for [`S2`](crate::S2).