diff --git a/Cargo.lock b/Cargo.lock index ce36370b..6450559c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -797,6 +797,7 @@ dependencies = [ "rustls", "serde", "serde_json", + "tap", "temp-env", "tokio", "tokio-postgres", diff --git a/mise.toml b/mise.toml index 1cd04499..d30f4512 100644 --- a/mise.toml +++ b/mise.toml @@ -31,7 +31,7 @@ CS_PROXY__HOST = "proxy" # Misc DOCKER_CLI_HINTS = "false" # Please don't show us What's Next. -CS_EQL_VERSION = "eql-2.0.1" +CS_EQL_VERSION = "eql-2.0.4" [tools] "cargo:cargo-binstall" = "latest" diff --git a/packages/cipherstash-proxy-integration/Cargo.toml b/packages/cipherstash-proxy-integration/Cargo.toml index 90162eb9..c923ba97 100644 --- a/packages/cipherstash-proxy-integration/Cargo.toml +++ b/packages/cipherstash-proxy-integration/Cargo.toml @@ -29,4 +29,5 @@ cipherstash-config = "0.2.3" clap = "4.5.32" fake = { version = "4", features = ["chrono", "derive"] } hex = "0.4.3" +tap = "1.0.1" uuid = { version = "1.11.0", features = ["serde", "v4"] } diff --git a/packages/cipherstash-proxy-integration/src/common.rs b/packages/cipherstash-proxy-integration/src/common.rs index 4afabc79..bfffa3d3 100644 --- a/packages/cipherstash-proxy-integration/src/common.rs +++ b/packages/cipherstash-proxy-integration/src/common.rs @@ -6,7 +6,7 @@ use rustls::{ pki_types::CertificateDer, ClientConfig, }; use std::sync::{Arc, Once}; -use tokio_postgres::{Client, NoTls}; +use tokio_postgres::{types::ToSql, Client, NoTls}; use tracing_subscriber::{filter::Directive, EnvFilter, FmtSubscriber}; pub const PROXY: u16 = 6432; @@ -17,7 +17,7 @@ pub const TEST_SCHEMA_SQL: &str = include_str!(concat!("../../../tests/sql/schem static INIT: Once = Once::new(); -pub fn id() -> i64 { +pub fn random_id() -> i64 { use rand::Rng; let mut rng = rand::rng(); rng.random_range(1..=i64::MAX) @@ -113,6 +113,52 @@ pub async fn connect(port: u16) -> Client { client } +pub async fn insert(sql: &str, params: &[&(dyn ToSql + Sync)]) { + let client = connect_with_tls(PROXY).await; + client.query(sql, params).await.unwrap(); +} + +pub async fn query tokio_postgres::types::FromSql<'a> + Send + Sync>( + sql: &str, +) -> Vec { + let client = connect_with_tls(PROXY).await; + let rows = client.query(sql, &[]).await.unwrap(); + rows.iter().map(|row| row.get(0)).collect::>() +} + +pub async fn simple_query(sql: &str) -> Vec +where + ::Err: std::fmt::Debug, +{ + let client = connect_with_tls(PROXY).await; + let rows = client.simple_query(sql).await.unwrap(); + rows.iter() + .filter_map(|row| { + if let tokio_postgres::SimpleQueryMessage::Row(r) = row { + r.get(0).and_then(|val| val.parse::().ok()) + } else { + None + } + }) + .collect() +} + +// Returns a vector of `Option` for each row in the result set. +// Nulls are represented as `None`, and non-null values are converted to `Some(String)`. +pub async fn simple_query_with_null(sql: &str) -> Vec> { + let client = connect_with_tls(PROXY).await; + let rows = client.simple_query(sql).await.unwrap(); + rows.iter() + .filter_map(|row| { + if let tokio_postgres::SimpleQueryMessage::Row(r) = row { + Some(r.get(0).map(|val| val.to_string())) + } else { + None + } + }) + .collect() +} + /// /// Configure the client TLS settings. /// These are the settings for connecting to the database with TLS. diff --git a/packages/cipherstash-proxy-integration/src/decrypt/insert_returning.rs b/packages/cipherstash-proxy-integration/src/decrypt/insert_returning.rs index d507aa46..93372981 100644 --- a/packages/cipherstash-proxy-integration/src/decrypt/insert_returning.rs +++ b/packages/cipherstash-proxy-integration/src/decrypt/insert_returning.rs @@ -2,7 +2,7 @@ mod tests { use chrono::NaiveDate; - use crate::common::{clear, connect_with_tls, id, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; #[tokio::test] async fn decrypt_insert_returning_with_different_column_order() { @@ -12,7 +12,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let plaintext = "plaintext"; let plaintext_date: Option = None; let encrypted_text = "hello@cipherstash.com"; @@ -61,7 +61,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let plaintext = "plaintext"; let encrypted_text = "hello@cipherstash.com"; @@ -103,7 +103,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let plaintext = "plaintext"; let encrypted_text = "hello@cipherstash.com"; diff --git a/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs b/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs index ae868aea..354c0ad5 100644 --- a/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs +++ b/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs @@ -2,7 +2,7 @@ mod tests { use tracing::{debug, info}; - use crate::common::{clear, connect_with_tls, id, reset_schema, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, reset_schema, trace, PROXY}; struct Reset; @@ -24,7 +24,7 @@ mod tests { let _reset = Reset; - let id = id(); + let id = random_id(); let client = connect_with_tls(PROXY).await; @@ -55,7 +55,7 @@ mod tests { // Create a record // If select returns no results, no configuration is required - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = "INSERT INTO unconfigured (id, encrypted_unconfigured) VALUES ($1, $2)"; @@ -82,7 +82,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); // let encrypted_date = NaiveDate::parse_from_str("2025-01-01", "%Y-%m-%d").unwrap(); let encrypted_date: i32 = 2025; @@ -111,7 +111,7 @@ mod tests { // Create a record // If select returns no results, no configuration is required - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = "INSERT INTO encrypted id, encrypted_text VALUES ($1, $2)"; diff --git a/packages/cipherstash-proxy-integration/src/lib.rs b/packages/cipherstash-proxy-integration/src/lib.rs index 86a0ec77..6f2f6fd2 100644 --- a/packages/cipherstash-proxy-integration/src/lib.rs +++ b/packages/cipherstash-proxy-integration/src/lib.rs @@ -14,4 +14,5 @@ mod migrate; mod passthrough; mod pipeline; mod schema_change; +mod select; mod simple_protocol; diff --git a/packages/cipherstash-proxy-integration/src/map_concat.rs b/packages/cipherstash-proxy-integration/src/map_concat.rs index 90abd9e4..21e529c8 100644 --- a/packages/cipherstash-proxy-integration/src/map_concat.rs +++ b/packages/cipherstash-proxy-integration/src/map_concat.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, PROXY}; #[tokio::test] async fn map_concat_regression() { @@ -8,7 +8,7 @@ mod tests { clear().await; - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; diff --git a/packages/cipherstash-proxy-integration/src/map_literals.rs b/packages/cipherstash-proxy-integration/src/map_literals.rs index 83c5193e..53610a75 100644 --- a/packages/cipherstash-proxy-integration/src/map_literals.rs +++ b/packages/cipherstash-proxy-integration/src/map_literals.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, PROXY}; #[tokio::test] async fn map_literal() { @@ -8,7 +8,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = @@ -28,7 +28,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let int2: i16 = 1; @@ -51,7 +51,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_jsonb = serde_json::json!({"key": "value"}); let sql = format!( @@ -80,7 +80,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let sql = format!("INSERT INTO encrypted (id, encrypted_int8) VALUES ({id}, {id}) RETURNING id, encrypted_int8"); @@ -103,7 +103,7 @@ mod tests { let client = connect_with_tls(PROXY).await; let sql = - format!("INSERT INTO encrypted (id, encrypted_text) VALUES ({}, 'a'), ({}, 'a') RETURNING encrypted_text", id(), id()); + format!("INSERT INTO encrypted (id, encrypted_text) VALUES ({}, 'a'), ({}, 'a') RETURNING encrypted_text", random_id(), random_id()); let rows = client.query(&sql, &[]).await.unwrap(); let actual = rows.iter().map(|row| row.get(0)).collect::>(); diff --git a/packages/cipherstash-proxy-integration/src/map_match_index.rs b/packages/cipherstash-proxy-integration/src/map_match_index.rs index 1bef2993..efc5711e 100644 --- a/packages/cipherstash-proxy-integration/src/map_match_index.rs +++ b/packages/cipherstash-proxy-integration/src/map_match_index.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; #[tokio::test] async fn map_match_index_text() { @@ -10,7 +10,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; diff --git a/packages/cipherstash-proxy-integration/src/map_nulls.rs b/packages/cipherstash-proxy-integration/src/map_nulls.rs index 069a371c..46f63548 100644 --- a/packages/cipherstash-proxy-integration/src/map_nulls.rs +++ b/packages/cipherstash-proxy-integration/src/map_nulls.rs @@ -1,16 +1,15 @@ #[cfg(test)] mod tests { + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; use chrono::NaiveDate; - use crate::common::{clear, connect_with_tls, id, trace, PROXY}; - #[tokio::test] async fn map_insert_null_param() { trace(); let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_text: Option = None; let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; @@ -33,7 +32,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; @@ -71,7 +70,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, NULL)"; client.query(sql, &[&id]).await.unwrap(); @@ -96,7 +95,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_int2: i16 = 42; let sql = @@ -128,7 +127,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let plaintext: Option = None; let plaintext_date: Option = None; let encrypted_text: Option = None; diff --git a/packages/cipherstash-proxy-integration/src/map_ore_index_order.rs b/packages/cipherstash-proxy-integration/src/map_ore_index_order.rs index 6286202e..c72e9c06 100644 --- a/packages/cipherstash-proxy-integration/src/map_ore_index_order.rs +++ b/packages/cipherstash-proxy-integration/src/map_ore_index_order.rs @@ -2,7 +2,7 @@ mod tests { use tokio_postgres::SimpleQueryMessage; - use crate::common::{clear, connect_with_tls, id, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; #[tokio::test] async fn map_ore_order_text() { @@ -22,7 +22,17 @@ mod tests { "; client - .query(sql, &[&id(), &s_two, &id(), &s_one, &id(), &s_three]) + .query( + sql, + &[ + &random_id(), + &s_two, + &random_id(), + &s_one, + &random_id(), + &s_three, + ], + ) .await .unwrap(); @@ -53,7 +63,17 @@ mod tests { "; client - .query(sql, &[&id(), &s_two, &id(), &s_one, &id(), &s_three]) + .query( + sql, + &[ + &random_id(), + &s_two, + &random_id(), + &s_one, + &random_id(), + &s_three, + ], + ) .await .unwrap(); @@ -78,7 +98,7 @@ mod tests { let s_two = "b"; client - .query("INSERT INTO encrypted (id) values ($1)", &[&id()]) + .query("INSERT INTO encrypted (id) values ($1)", &[&random_id()]) .await .unwrap(); @@ -88,7 +108,7 @@ mod tests { "; client - .query(sql, &[&id(), &s_one, &id(), &s_two]) + .query(sql, &[&random_id(), &s_one, &random_id(), &s_two]) .await .unwrap(); @@ -121,12 +141,12 @@ mod tests { "; client - .query(sql, &[&id(), &s_one, &id(), &s_two]) + .query(sql, &[&random_id(), &s_one, &random_id(), &s_two]) .await .unwrap(); client - .query("INSERT INTO encrypted (id) values ($1)", &[&id()]) + .query("INSERT INTO encrypted (id) values ($1)", &[&random_id()]) .await .unwrap(); @@ -160,7 +180,17 @@ mod tests { "; client - .query(sql, &[&id(), &s_two, &id(), &s_one, &id(), &s_three]) + .query( + sql, + &[ + &random_id(), + &s_two, + &random_id(), + &s_one, + &random_id(), + &s_three, + ], + ) .await .unwrap(); @@ -191,7 +221,17 @@ mod tests { "; client - .query(sql, &[&id(), &s_two, &id(), &s_one, &id(), &s_three]) + .query( + sql, + &[ + &random_id(), + &s_two, + &random_id(), + &s_one, + &random_id(), + &s_three, + ], + ) .await .unwrap(); @@ -212,11 +252,11 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id_one = id(); + let id_one = random_id(); let s_one = "a"; - let id_two = id(); + let id_two = random_id(); let s_two = "b"; - let id_three = id(); + let id_three = random_id(); let s_three = "c"; let sql = " @@ -259,7 +299,17 @@ mod tests { "; client - .query(sql, &[&id(), &s_two, &id(), &s_one, &id(), &s_three]) + .query( + sql, + &[ + &random_id(), + &s_two, + &random_id(), + &s_one, + &random_id(), + &s_three, + ], + ) .await .unwrap(); @@ -297,13 +347,13 @@ mod tests { .query( sql, &[ - &id(), + &random_id(), &s_plaintext_two, &s_encrypted_two, - &id(), + &random_id(), &s_plaintext_one, &s_enctrypted_one, - &id(), + &random_id(), &s_plaintext_three, &s_encrypted_three, ], @@ -339,9 +389,9 @@ mod tests { let sql = format!( "INSERT INTO encrypted (id, encrypted_text) VALUES ({}, 'y'), ({}, 'x'), ({}, 'z')", - id(), - id(), - id() + random_id(), + random_id(), + random_id() ); client.simple_query(&sql).await.unwrap(); @@ -383,7 +433,17 @@ mod tests { "; client - .query(sql, &[&id(), &n_two, &id(), &n_one, &id(), &n_three]) + .query( + sql, + &[ + &random_id(), + &n_two, + &random_id(), + &n_one, + &random_id(), + &n_three, + ], + ) .await .unwrap(); @@ -414,7 +474,17 @@ mod tests { "; client - .query(sql, &[&id(), &n_two, &id(), &n_one, &id(), &n_three]) + .query( + sql, + &[ + &random_id(), + &n_two, + &random_id(), + &n_one, + &random_id(), + &n_three, + ], + ) .await .unwrap(); @@ -445,7 +515,17 @@ mod tests { "; client - .query(sql, &[&id(), &n_two, &id(), &n_one, &id(), &n_three]) + .query( + sql, + &[ + &random_id(), + &n_two, + &random_id(), + &n_one, + &random_id(), + &n_three, + ], + ) .await .unwrap(); @@ -476,7 +556,17 @@ mod tests { "; client - .query(sql, &[&id(), &n_two, &id(), &n_one, &id(), &n_three]) + .query( + sql, + &[ + &random_id(), + &n_two, + &random_id(), + &n_one, + &random_id(), + &n_three, + ], + ) .await .unwrap(); diff --git a/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs b/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs index 7c82897d..2563140f 100644 --- a/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs +++ b/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; use chrono::NaiveDate; use tokio_postgres::types::{FromSql, ToSql}; use tokio_postgres::Client; @@ -59,14 +59,17 @@ mod tests { let sql = format!("INSERT INTO encrypted (id, {col_name}) VALUES ($1, $2)"); for val in [low.clone(), high.clone()] { client - .query(&sql, &[&id(), &val]) + .query(&sql, &[&random_id(), &val]) .await .expect("insert failed"); } // NULL record let sql = format!("INSERT INTO encrypted (id, {col_name}) VALUES ($1, null)"); - client.query(&sql, &[&id()]).await.expect("insert failed"); + client + .query(&sql, &[&random_id()]) + .await + .expect("insert failed"); // GT: given [1, 3], `> 1` returns [3] let sql = format!("SELECT {col_name} FROM encrypted WHERE {col_name} > $1"); diff --git a/packages/cipherstash-proxy-integration/src/map_params.rs b/packages/cipherstash-proxy-integration/src/map_params.rs index 5fbbf11c..db33e154 100644 --- a/packages/cipherstash-proxy-integration/src/map_params.rs +++ b/packages/cipherstash-proxy-integration/src/map_params.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{connect_with_tls, id, reset_schema, trace, PROXY}; + use crate::common::{connect_with_tls, random_id, reset_schema, trace, PROXY}; use chrono::NaiveDate; #[tokio::test] @@ -9,7 +9,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; @@ -32,7 +32,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_bool: bool = true; let sql = "INSERT INTO encrypted (id, encrypted_bool) VALUES ($1, $2)"; @@ -58,7 +58,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_int2: i16 = 42; let sql = "INSERT INTO encrypted (id, encrypted_int2) VALUES ($1, $2)"; @@ -84,7 +84,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_int4: i32 = 42; let sql = "INSERT INTO encrypted (id, encrypted_int4) VALUES ($1, $2)"; @@ -110,7 +110,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_int8: i64 = 42; let sql = "INSERT INTO encrypted (id, encrypted_int8) VALUES ($1, $2)"; @@ -136,7 +136,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_float8: f64 = 42.00; let sql = "INSERT INTO encrypted (id, encrypted_float8) VALUES ($1, $2)"; @@ -162,7 +162,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_date = NaiveDate::parse_from_str("2025-01-01", "%Y-%m-%d").unwrap(); let sql = "INSERT INTO encrypted (id, encrypted_date) VALUES ($1, $2)"; @@ -188,7 +188,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_jsonb = serde_json::json!({"key": "value"}); let sql = "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)"; @@ -214,7 +214,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_jsonb = serde_json::json!({"key": 42}); let sql = "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)"; @@ -240,7 +240,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let plaintext = "hello@cipherstash.com"; let sql = "INSERT INTO encrypted (id, plaintext) VALUES ($1, $2)"; @@ -265,7 +265,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let plaintext = "hello@cipherstash.com"; let encrypted_text = "hello@cipherstash.com"; let encrypted_bool = false; diff --git a/packages/cipherstash-proxy-integration/src/map_unique_index.rs b/packages/cipherstash-proxy-integration/src/map_unique_index.rs index a72007d2..0d9e5589 100644 --- a/packages/cipherstash-proxy-integration/src/map_unique_index.rs +++ b/packages/cipherstash-proxy-integration/src/map_unique_index.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, reset_schema, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, reset_schema, trace, PROXY}; use chrono::NaiveDate; #[tokio::test] @@ -11,7 +11,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; @@ -36,7 +36,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_bool: bool = true; let sql = "INSERT INTO encrypted (id, encrypted_bool) VALUES ($1, $2)"; @@ -64,7 +64,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_int2: i16 = 42; let sql = "INSERT INTO encrypted (id, encrypted_int2) VALUES ($1, $2)"; @@ -92,7 +92,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_int4: i32 = 42; let sql = "INSERT INTO encrypted (id, encrypted_int4) VALUES ($1, $2)"; @@ -120,7 +120,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_int8: i64 = 42; let sql = "INSERT INTO encrypted (id, encrypted_int8) VALUES ($1, $2)"; @@ -148,7 +148,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_float8: f64 = 42.00; let sql = "INSERT INTO encrypted (id, encrypted_float8) VALUES ($1, $2)"; @@ -176,7 +176,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_date = NaiveDate::parse_from_str("2025-01-01", "%Y-%m-%d").unwrap(); let sql = "INSERT INTO encrypted (id, encrypted_date) VALUES ($1, $2)"; @@ -204,7 +204,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let plaintext = "hello@cipherstash.com"; let sql = "INSERT INTO encrypted (id, plaintext) VALUES ($1, $2)"; @@ -229,7 +229,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let plaintext = "hello@cipherstash.com"; let encrypted_text = "hello@cipherstash.com"; let encrypted_bool = false; diff --git a/packages/cipherstash-proxy-integration/src/migrate/mod.rs b/packages/cipherstash-proxy-integration/src/migrate/mod.rs index f8188572..41fdcdc9 100644 --- a/packages/cipherstash-proxy-integration/src/migrate/mod.rs +++ b/packages/cipherstash-proxy-integration/src/migrate/mod.rs @@ -2,7 +2,7 @@ mod tests { use std::ops::Deref; - use crate::common::{clear, connect_with_tls, id, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; use cipherstash_proxy::{ config::{LogFormat, LogLevel}, Args, Migrate, TandemConfig, @@ -55,7 +55,7 @@ mod tests { let client = connect_with_tls(PROXY).await; for _ in 1..10 { - let id = id(); + let id = random_id(); let plaintext = Faker.fake::(); let sql = "INSERT INTO encrypted (id, plaintext) VALUES ($1, $2)"; diff --git a/packages/cipherstash-proxy-integration/src/passthrough.rs b/packages/cipherstash-proxy-integration/src/passthrough.rs index 4f09cd4b..676f94e5 100644 --- a/packages/cipherstash-proxy-integration/src/passthrough.rs +++ b/packages/cipherstash-proxy-integration/src/passthrough.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, random_string, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, random_string, PROXY}; use rand::Rng; use std::error::Error; @@ -10,7 +10,7 @@ mod tests { clear().await; - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = "INSERT INTO plaintext (id, plaintext) VALUES ($1, $2)"; @@ -59,7 +59,7 @@ mod tests { let client = connect_with_tls(PROXY).await; for _x in 1..10 { - let id = id(); + let id = random_id(); let encrypted_text = random_string(); let sql = "INSERT INTO plaintext (id, plaintext) VALUES ($1, $2)"; @@ -91,7 +91,7 @@ mod tests { clear().await; // Setup data - let id_1 = id(); + let id_1 = random_id(); let plaintext = "hello@cipherstash.com"; let sql = "INSERT INTO plaintext (id, plaintext) VALUES ($1, $2)"; @@ -123,13 +123,13 @@ mod tests { clear().await; // Setup data - let id_1 = id(); + let id_1 = random_id(); let plaintext = "hello@cipherstash.com"; let sql = "INSERT INTO plaintext (id, plaintext) VALUES ($1, $2)"; client.query(sql, &[&id_1, &plaintext]).await.unwrap(); - let id_2 = id(); + let id_2 = random_id(); // Insert value is selected from the record we just created let select = "SELECT plaintext FROM plaintext WHERE id = $2"; @@ -154,7 +154,7 @@ mod tests { clear().await; // Setup data - let id = id(); + let id = random_id(); let plaintext = "hello@cipherstash.com"; let sql = "INSERT INTO plaintext (id, plaintext) VALUES ($1, $2) RETURNING *"; @@ -175,13 +175,13 @@ mod tests { clear().await; // Setup data - let id_1 = id(); + let id_1 = random_id(); let plaintext = "hello@cipherstash.com"; let sql = "INSERT INTO plaintext (id, plaintext) VALUES ($1, $2)"; client.query(sql, &[&id_1, &plaintext]).await.unwrap(); - let id_2 = id(); + let id_2 = random_id(); let sql = "INSERT INTO plaintext (id) VALUES ($1)"; client.query(sql, &[&id_2]).await.unwrap(); @@ -203,13 +203,13 @@ mod tests { clear().await; // Setup data - let id_1 = id(); + let id_1 = random_id(); let plaintext = "one@cipherstash.com"; let sql = "INSERT INTO plaintext (id, plaintext) VALUES ($1, $2)"; client.query(sql, &[&id_1, &plaintext]).await.unwrap(); - let id_2 = id(); + let id_2 = random_id(); let plaintext = "two@cipherstash.com"; let sql = "INSERT INTO plaintext (id, plaintext) VALUES ($1, $2)"; diff --git a/packages/cipherstash-proxy-integration/src/pipeline.rs b/packages/cipherstash-proxy-integration/src/pipeline.rs index 2e1017ae..d1b1ea33 100644 --- a/packages/cipherstash-proxy-integration/src/pipeline.rs +++ b/packages/cipherstash-proxy-integration/src/pipeline.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; use std::sync::atomic::{AtomicUsize, Ordering}; /// @@ -20,7 +20,7 @@ mod tests { let counter = AtomicUsize::new(0); - let text_id = id(); + let text_id = random_id(); let encrypted_text = "hello@cipherstash.com"; let text = async { @@ -33,7 +33,7 @@ mod tests { let _ = counter.fetch_add(1, Ordering::SeqCst); }; - let int2_id = id(); + let int2_id = random_id(); let encrypted_int2: i16 = 16; let int2 = async { @@ -46,7 +46,7 @@ mod tests { let _ = counter.fetch_add(1, Ordering::SeqCst); }; - let int4_id = id(); + let int4_id = random_id(); let encrypted_int4: i32 = 32; let int4 = async { @@ -59,7 +59,7 @@ mod tests { let _ = counter.fetch_add(1, Ordering::SeqCst); }; - let plaintext_id = id(); + let plaintext_id = random_id(); let plaintext_text = "blahvtha"; let plaintext = async { diff --git a/packages/cipherstash-proxy-integration/src/schema_change.rs b/packages/cipherstash-proxy-integration/src/schema_change.rs index ab9a52ef..7c7e2f82 100644 --- a/packages/cipherstash-proxy-integration/src/schema_change.rs +++ b/packages/cipherstash-proxy-integration/src/schema_change.rs @@ -1,12 +1,12 @@ #[cfg(test)] mod tests { - use crate::common::{connect_with_tls, id, PROXY}; + use crate::common::{connect_with_tls, random_id, PROXY}; #[tokio::test] async fn schema_change_reloads_schema() { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let sql = format!( "CREATE TABLE table_{id} ( diff --git a/packages/cipherstash-proxy-integration/src/select/group_by.rs b/packages/cipherstash-proxy-integration/src/select/group_by.rs new file mode 100644 index 00000000..87bd3fea --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/select/group_by.rs @@ -0,0 +1,55 @@ +#[cfg(test)] +mod tests { + use crate::common::{clear, insert, query, random_id, simple_query, trace}; + use chrono::NaiveDate; + + macro_rules! test_group_by { + ($name: ident, $type: ident, $pg_type: ident) => { + #[tokio::test] + pub async fn $name() { + trace(); + + clear().await; + + let encrypted_col = format!("encrypted_{}", stringify!($pg_type)); + + for i in 1..=10 { + let encrypted_val = crate::value_for_type!($type, i); + + // Create two records with the same encrypted_int4 value + for _ in 1..=2 { + let id = random_id(); + let sql = + format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ($1, $2)"); + insert(&sql, &[&id, &encrypted_val]).await; + } + } + + // Validate that there are 20 records in the encrypted table + let sql = format!("SELECT {encrypted_col} FROM encrypted"); + + let rows = query::<$type>(&sql).await; + assert_eq!(rows.len(), 20); + + let rows = simple_query::<$type>(&sql).await; + assert_eq!(rows.len(), 20); + + // GROUP BY should return 10 records, each representing two records with the same encrypted_int4 value + let sql = format!("SELECT COUNT(*) FROM encrypted GROUP BY {encrypted_col}"); + + let rows = query::(&sql).await; + assert_eq!(rows.len(), 10); + + let rows = simple_query::(&sql).await; + assert_eq!(rows.len(), 10); + } + }; + } + + test_group_by!(group_by_int2, i16, int2); + test_group_by!(group_by_int4, i32, int4); + test_group_by!(group_by_int8, i64, int8); + test_group_by!(group_by_float8, f64, float8); + test_group_by!(group_by_text, String, text); + test_group_by!(group_by_date, NaiveDate, date); +} diff --git a/packages/cipherstash-proxy-integration/src/select/mod.rs b/packages/cipherstash-proxy-integration/src/select/mod.rs new file mode 100644 index 00000000..d50dc665 --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/select/mod.rs @@ -0,0 +1,18 @@ +mod group_by; +mod order_by; +mod order_by_with_null; + +#[macro_export] +macro_rules! value_for_type { + (String, $i:expr) => { + ((b'A' + ($i - 1) as u8) as char).to_string() + }; + + (NaiveDate, $i:expr) => { + NaiveDate::parse_from_str(&format!("2023-01-{}", $i), "%Y-%m-%d").unwrap() + }; + + ($type:ident, $i:expr) => { + $i as $type + }; +} diff --git a/packages/cipherstash-proxy-integration/src/select/order_by.rs b/packages/cipherstash-proxy-integration/src/select/order_by.rs new file mode 100644 index 00000000..f6b498ab --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/select/order_by.rs @@ -0,0 +1,56 @@ +#[cfg(test)] +mod tests { + use crate::common::{clear, insert, query, random_id, simple_query, trace}; + use chrono::NaiveDate; + + macro_rules! test_order_by { + ($name: ident, $type: ident, $pg_type: ident) => { + #[tokio::test] + pub async fn $name() { + trace(); + + clear().await; + let encrypted_col = format!("encrypted_{}", stringify!($pg_type)); + + let mut expected = vec![]; + for i in 1..=10 { + let encrypted_val = crate::value_for_type!($type, i); + + let id = random_id(); + let sql = + format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ($1, $2)"); + insert(&sql, &[&id, &encrypted_val]).await; + + expected.push(encrypted_val); + } + + let sql = + format!("SELECT {encrypted_col} FROM encrypted ORDER BY {encrypted_col} ASC"); + + let actual = query::<$type>(&sql).await; + assert_eq!(expected, actual); + + let actual = simple_query::<$type>(&sql).await; + assert_eq!(expected, actual); + + let sql = + format!("SELECT {encrypted_col} FROM encrypted ORDER BY {encrypted_col} DESC"); + + expected.reverse(); + + let actual = query::<$type>(&sql).await; + assert_eq!(expected, actual); + + let actual = simple_query::<$type>(&sql).await; + assert_eq!(expected, actual); + } + }; + } + + test_order_by!(order_by_int2, i16, int2); + test_order_by!(order_by_int4, i32, int4); + test_order_by!(order_by_int8, i64, int8); + test_order_by!(order_by_float8, f64, float8); + test_order_by!(order_by_text, String, text); + test_order_by!(order_by_date, NaiveDate, date); +} diff --git a/packages/cipherstash-proxy-integration/src/select/order_by_with_null.rs b/packages/cipherstash-proxy-integration/src/select/order_by_with_null.rs new file mode 100644 index 00000000..37d71382 --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/select/order_by_with_null.rs @@ -0,0 +1,364 @@ +#[cfg(test)] +mod tests { + use crate::common::{clear, insert, query, random_id, simple_query_with_null, trace}; + use chrono::NaiveDate; + use tap::prelude::*; + use tokio_postgres::types::ToSql; + + async fn insert_encrypted_value(col: &str, val: &T) + where + T: ToSql + Sync + Send + 'static, + { + let id = random_id(); + let sql = format!("INSERT INTO encrypted (id, {}) VALUES ($1, $2)", col); + insert(&sql, &[&id, &val]).await; + } + + fn assert_expected(expected: &[Option], actual: &[Option]) + where + T: std::fmt::Display + PartialEq + std::fmt::Debug, + { + assert_eq!(expected.len(), actual.len()); + for (e, r) in expected.iter().zip(actual) { + // info!("Expected: {:?}, Actual: {:?}", e, r); + assert_eq!(e, r); + } + } + + fn assert_expected_as_string(expected: &[Option], actual: &[Option]) + where + T: std::fmt::Display + PartialEq + std::fmt::Debug, + { + assert_eq!(expected.len(), actual.len()); + + for (e, r) in expected.iter().zip(actual) { + let e_str = e.as_ref().map(|v| v.to_string()); + assert_eq!(e_str.as_ref(), r.as_ref()); + } + } + + macro_rules! insert_encrypted_values { + ($type:ident, $encrypted_col:expr) => {{ + let mut expected = vec![]; + for i in 1..=5 { + let value = crate::value_for_type!($type, i); + insert_encrypted_value($encrypted_col, &value).await; + expected.push(value); + } + expected + }}; + } + + // ------------------------------------------------------------------------ + // ASC + // Default if unspecified is NULLS LAST + // [Some("A"), Some("B"), Some("C"), Some("D"), Some("E"),None, ] + macro_rules! test_order_by_with_null_asc { + ($name: ident, $type: ident, $pg_type: ident) => { + #[tokio::test] + pub async fn $name() { + trace(); + clear().await; + let encrypted_col = format!("encrypted_{}", stringify!($pg_type)); + + // insert test values + let expected = insert_encrypted_values!($type, &encrypted_col); + + // insert a null value (after the test values to avoid any natural ordering issues) + let encrypted_value: Option<$type> = None; + insert_encrypted_value(&encrypted_col, &encrypted_value).await; + + // NULL values are Option in tokio_postgres + // Wrap expected values in Some + // [Some("A"), Some("B"), Some("C"), Some("D"), Some("E"), None, ] + let expected = expected + .into_iter() + .map(|v| Some(v)) + .collect::>() + .tap_mut(|e| { + e.push(None); + }); + + // [Some("A"), Some("B"), Some("C"), Some("D"), Some("E"), None,] + let sql = + format!("SELECT {encrypted_col} FROM encrypted ORDER BY {encrypted_col} ASC"); + + let result = query::>(&sql).await; + assert_expected(&expected, &result); + + let result = simple_query_with_null(&sql).await; + assert_expected_as_string(&expected, &result); + } + }; + } + + // ------------------------------------------------------------------------ + // ASC NULLS FIRST + // [None, Some("A"), Some("B"), Some("C"), Some("D"), Some("E")] + macro_rules! test_order_by_with_null_asc_nulls_first { + ($name: ident, $type: ident, $pg_type: ident) => { + #[tokio::test] + pub async fn $name() { + clear().await; + let encrypted_col = format!("encrypted_{}", stringify!($pg_type)); + + // insert test values + let expected = insert_encrypted_values!($type, &encrypted_col); + + // insert a null value (after the test values to avoid any natural ordering issues) + let encrypted_value: Option<$type> = None; + insert_encrypted_value(&encrypted_col, &encrypted_value).await; + + // NULL values are Option in tokio_postgres + // Wrap expected values in Some + // [None, Some("A"), Some("B"), Some("C"), Some("D"), Some("E")] + let expected = expected.into_iter().map(|v| Some(v)).collect::>().tap_mut(|e| { + e.insert(0, None); + }); + + // ------------------------------------------------------------------------ + // ASC NULLS FIRST + // [None, Some("A"), Some("B"), Some("C"), Some("D"), Some("E")] + let sql = + format!("SELECT {encrypted_col} FROM encrypted ORDER BY {encrypted_col} ASC NULLS FIRST"); + + let result = query::>(&sql).await; + assert_expected(&expected, &result); + + let result = simple_query_with_null(&sql).await; + assert_expected_as_string(&expected, &result); + + } + }; + } + + // ------------------------------------------------------------------------ + // ASC NULLS LAST + // [Some("A"), Some("B"), Some("C"), Some("D"), Some("E"), None, ] + macro_rules! test_order_by_with_null_asc_nulls_last { + ($name: ident, $type: ident, $pg_type: ident) => { + #[tokio::test] + pub async fn $name() { + clear().await; + let encrypted_col = format!("encrypted_{}", stringify!($pg_type)); + + // insert a null value (before the test values to avoid any natural ordering issues) + let encrypted_value: Option<$type> = None; + insert_encrypted_value(&encrypted_col, &encrypted_value).await; + + // insert test values + let expected = insert_encrypted_values!($type, &encrypted_col); + + // NULL values are Option in tokio_postgres + // Wrap expected values in Some + // [Some("A"), Some("B"), Some("C"), Some("D"), Some("E"), None] + let expected = expected.into_iter().map(|v| Some(v)).collect::>().tap_mut(|e| { + e.push(None); + }); + + // ------------------------------------------------------------------------ + // ASC NULLS LAST + // [None, Some("A"), Some("B"), Some("C"), Some("D"), Some("E")] + let sql = + format!("SELECT {encrypted_col} FROM encrypted ORDER BY {encrypted_col} ASC NULLS LAST"); + + let result = query::>(&sql).await; + assert_expected(&expected, &result); + + let result = simple_query_with_null(&sql).await; + assert_expected_as_string(&expected, &result); + + } + }; + } + + // ------------------------------------------------------------------------ + // DESC + // Default if unspecified is NULLS FIRST + // [None, Some("E"), Some("D"), Some("C"), Some("B"), Some("A")] + macro_rules! test_order_by_with_null_desc { + ($name: ident, $type: ident, $pg_type: ident) => { + #[tokio::test] + pub async fn $name() { + trace(); + clear().await; + let encrypted_col = format!("encrypted_{}", stringify!($pg_type)); + + // insert a null value (before the test values to avoid any natural ordering issues) + let encrypted_value: Option<$type> = None; + insert_encrypted_value(&encrypted_col, &encrypted_value).await; + + // insert test values + let expected = insert_encrypted_values!($type, &encrypted_col); + + // NULL values are Option in tokio_postgres + // Wrap expected values in Some + // [None, Some("E"), Some("D"), Some("C"), Some("B"), Some("A")] + let expected = expected + .tap_mut(|e| e.reverse()) + .into_iter() + .map(|v| Some(v)) + .collect::>() + .tap_mut(|e| { + e.insert(0, None); + }); + + let sql = + format!("SELECT {encrypted_col} FROM encrypted ORDER BY {encrypted_col} DESC"); + + let result = query::>(&sql).await; + assert_expected(&expected, &result); + + let result = simple_query_with_null(&sql).await; + assert_expected_as_string(&expected, &result); + } + }; + } + + // ------------------------------------------------------------------------ + // DESC NULLS FIRST + // [None, Some("E"), Some("D"), Some("C"), Some("B"), Some("A")] + macro_rules! test_order_by_with_null_desc_nulls_first { + ($name: ident, $type: ident, $pg_type: ident) => { + #[tokio::test] + pub async fn $name() { + trace(); + clear().await; + let encrypted_col = format!("encrypted_{}", stringify!($pg_type)); + + // insert a null value (before the test values to avoid any natural ordering issues) + let encrypted_value: Option<$type> = None; + insert_encrypted_value(&encrypted_col, &encrypted_value).await; + + // insert test values + let expected = insert_encrypted_values!($type, &encrypted_col); + + // NULL values are Option in tokio_postgres + // Wrap expected values in Some + // [None, Some("E"), Some("D"), Some("C"), Some("B"), Some("A")] + let expected = expected.tap_mut(|e| e.reverse()).into_iter().map(|v| Some(v)).collect::>() + .tap_mut(|e| { + e.insert(0, None); + }); + + let sql = + format!("SELECT {encrypted_col} FROM encrypted ORDER BY {encrypted_col} DESC NULLS FIRST"); + + let result = query::>(&sql).await; + assert_expected(&expected, &result); + + let result = simple_query_with_null(&sql).await; + assert_expected_as_string(&expected, &result); + + } + }; + } + + // ------------------------------------------------------------------------ + // DESC NULLS LAST + // [Some("E"), Some("D"), Some("C"), Some("B"), Some("A"), None] + macro_rules! test_order_by_with_null_desc_nulls_last { + ($name: ident, $type: ident, $pg_type: ident) => { + #[tokio::test] + pub async fn $name() { + trace(); + clear().await; + let encrypted_col = format!("encrypted_{}", stringify!($pg_type)); + + // insert a null value (before the test values to avoid any natural ordering issues) + let encrypted_value: Option<$type> = None; + insert_encrypted_value(&encrypted_col, &encrypted_value).await; + + // insert test values + let expected = insert_encrypted_values!($type, &encrypted_col); + + // NULL values are Option in tokio_postgres + // Wrap expected values in Some + // [Some("E"), Some("D"), Some("C"), Some("B"), Some("A"), None] + let expected = expected.tap_mut(|e| e.reverse()).into_iter().map(|v| Some(v)).collect::>() + .tap_mut(|e| { + e.push(None); + }); + + let sql = + format!("SELECT {encrypted_col} FROM encrypted ORDER BY {encrypted_col} DESC NULLS LAST"); + + + let result = query::>(&sql).await; + assert_expected(&expected, &result); + + let result = simple_query_with_null(&sql).await; + assert_expected_as_string(&expected, &result); + + } + }; + } + + // ============================================================ + + test_order_by_with_null_asc!(order_by_int2_with_null_asc, i16, int2); + test_order_by_with_null_asc!(order_by_int4_with_null_asc, i32, int4); + test_order_by_with_null_asc!(order_by_int8_with_null_asc, i64, int8); + test_order_by_with_null_asc!(order_by_floatwith_null_asc, f64, float8); + test_order_by_with_null_asc!(order_by_text_with_null_asc, String, text); + test_order_by_with_null_asc!(order_by_date_with_null_asc, NaiveDate, date); + + test_order_by_with_null_asc_nulls_last!(order_by_int2_with_null_asc_nulls_last, i16, int2); + test_order_by_with_null_asc_nulls_last!(order_by_int4_with_null_asc_nulls_last, i32, int4); + test_order_by_with_null_asc_nulls_last!(order_by_int8_with_null_asc_nulls_last, i64, int8); + test_order_by_with_null_asc_nulls_last!(order_by_floatwith_null_asc_nulls_last, f64, float8); + test_order_by_with_null_asc_nulls_last!(order_by_text_with_null_asc_nulls_last, String, text); + test_order_by_with_null_asc_nulls_last!( + order_by_date_with_null_asc_nulls_last, + NaiveDate, + date + ); + + test_order_by_with_null_asc_nulls_first!(order_by_int2_with_null_asc_nulls_first, i16, int2); + test_order_by_with_null_asc_nulls_first!(order_by_int4_with_null_asc_nulls_first, i32, int4); + test_order_by_with_null_asc_nulls_first!(order_by_int8_with_null_asc_nulls_first, i64, int8); + test_order_by_with_null_asc_nulls_first!(order_by_floatwith_null_asc_nulls_first, f64, float8); + test_order_by_with_null_asc_nulls_first!(order_by_text_with_null_asc_nulls_first, String, text); + test_order_by_with_null_asc_nulls_first!( + order_by_date_with_null_asc_nulls_first, + NaiveDate, + date + ); + + test_order_by_with_null_desc!(order_by_int2_with_null_desc, i16, int2); + test_order_by_with_null_desc!(order_by_int4_with_null_desc, i32, int4); + test_order_by_with_null_desc!(order_by_int8_with_null_desc, i64, int8); + test_order_by_with_null_desc!(order_by_floatwith_null_desc, f64, float8); + test_order_by_with_null_desc!(order_by_text_with_null_desc, String, text); + test_order_by_with_null_desc!(order_by_date_with_null_desc, NaiveDate, date); + + test_order_by_with_null_desc_nulls_first!(order_by_int2_with_null_desc_nulls_first, i16, int2); + test_order_by_with_null_desc_nulls_first!(order_by_int4_with_null_desc_nulls_first, i32, int4); + test_order_by_with_null_desc_nulls_first!(order_by_int8_with_null_desc_nulls_first, i64, int8); + test_order_by_with_null_desc_nulls_first!( + order_by_floatwith_null_desc_nulls_first, + f64, + float8 + ); + test_order_by_with_null_desc_nulls_first!( + order_by_text_with_null_desc_nulls_first, + String, + text + ); + test_order_by_with_null_desc_nulls_first!( + order_by_date_with_null_desc_nulls_first, + NaiveDate, + date + ); + + test_order_by_with_null_desc_nulls_last!(order_by_int2_with_null_desc_nulls_last, i16, int2); + test_order_by_with_null_desc_nulls_last!(order_by_int4_with_null_desc_nulls_last, i32, int4); + test_order_by_with_null_desc_nulls_last!(order_by_int8_with_null_desc_nulls_last, i64, int8); + test_order_by_with_null_desc_nulls_last!(order_by_floatwith_null_desc_nulls_last, f64, float8); + test_order_by_with_null_desc_nulls_last!(order_by_text_with_null_desc_nulls_last, String, text); + test_order_by_with_null_desc_nulls_last!( + order_by_date_with_null_desc_nulls_last, + NaiveDate, + date + ); +} diff --git a/packages/cipherstash-proxy-integration/src/simple_protocol/error_handling.rs b/packages/cipherstash-proxy-integration/src/simple_protocol/error_handling.rs index 2d51912b..5871c8c0 100644 --- a/packages/cipherstash-proxy-integration/src/simple_protocol/error_handling.rs +++ b/packages/cipherstash-proxy-integration/src/simple_protocol/error_handling.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{connect_with_tls, id, PROXY}; + use crate::common::{connect_with_tls, random_id, PROXY}; #[tokio::test] async fn frontend_error_does_not_crash_connection() { @@ -9,7 +9,7 @@ mod tests { // Statement has the wrong column name let sql = format!( "INSERT INTO encrypted (id, encrypted) VALUES ({}, 'foo@example.net')", - id() + random_id() ); let result = client.simple_query(&sql).await; @@ -23,7 +23,7 @@ mod tests { // And we can still use the connection let sql = format!( "INSERT INTO encrypted (id, encrypted_text) VALUES ({}, 'foo@example.net')", - id() + random_id() ); let result = client.simple_query(&sql).await; diff --git a/packages/cipherstash-proxy-integration/src/simple_protocol/map_literals.rs b/packages/cipherstash-proxy-integration/src/simple_protocol/map_literals.rs index f114b757..aee5d0ec 100644 --- a/packages/cipherstash-proxy-integration/src/simple_protocol/map_literals.rs +++ b/packages/cipherstash-proxy-integration/src/simple_protocol/map_literals.rs @@ -1,13 +1,13 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, PROXY}; use chrono::NaiveDate; use tokio_postgres::SimpleQueryMessage::{CommandComplete, Row}; #[tokio::test] async fn simple_protocol_without_encryption() { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let sql = format!("INSERT INTO encrypted (id, plaintext) VALUES ({id}, 'plain')"); client .simple_query(&sql) @@ -30,7 +30,7 @@ mod tests { async fn simple_protocol_text() { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_text = "hello@cipherstash.com"; let sql = @@ -70,7 +70,7 @@ mod tests { async fn simple_protocol_int2() { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_int2: i16 = 42; let sql = @@ -110,7 +110,7 @@ mod tests { async fn simple_protocol_date() { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_date = NaiveDate::parse_from_str("2025-01-01", "%Y-%m-%d").unwrap(); let sql = @@ -151,7 +151,7 @@ mod tests { async fn simple_protocol_date_with_iso() { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_date = NaiveDate::parse_from_str("2025-01-01T13:00:00+10:00", "%Y-%m-%dT%H:%M:%S%z").unwrap(); @@ -193,7 +193,7 @@ mod tests { async fn simple_protocol_int4() { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_int4: i32 = 42; let statements = vec![ @@ -248,7 +248,7 @@ mod tests { // Statement has the wrong column name let sql = format!( "INSERT INTO encrypted (id, encrypted) VALUES ({}, 'foo@example.net')", - id() + random_id() ); let result = client.simple_query(&sql).await; @@ -262,7 +262,7 @@ mod tests { // And we can still use the connection let sql = format!( "INSERT INTO encrypted (id, encrypted_text) VALUES ({}, 'foo@example.net')", - id() + random_id() ); let result = client.simple_query(&sql).await; diff --git a/packages/cipherstash-proxy-integration/src/simple_protocol/map_nulls.rs b/packages/cipherstash-proxy-integration/src/simple_protocol/map_nulls.rs index f8443925..effd7bcf 100644 --- a/packages/cipherstash-proxy-integration/src/simple_protocol/map_nulls.rs +++ b/packages/cipherstash-proxy-integration/src/simple_protocol/map_nulls.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; use tokio_postgres::SimpleQueryMessage::Row; #[tokio::test] @@ -11,7 +11,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let encrypted_text: Option<&str> = None; let sql = format!("INSERT INTO encrypted (id, encrypted_text) VALUES ({id}, NULL)"); @@ -53,7 +53,7 @@ mod tests { let client = connect_with_tls(PROXY).await; - let id = id(); + let id = random_id(); let sql = format!("INSERT INTO encrypted (id, encrypted_text, encrypted_int2, encrypted_int4, encrypted_int8) VALUES ({id}, NULL, NULL, NULL, NULL)"); client.simple_query(&sql).await.unwrap(); diff --git a/packages/cipherstash-proxy-integration/src/simple_protocol/multiple_statements.rs b/packages/cipherstash-proxy-integration/src/simple_protocol/multiple_statements.rs index 3b64dc2e..3b559bdf 100644 --- a/packages/cipherstash-proxy-integration/src/simple_protocol/multiple_statements.rs +++ b/packages/cipherstash-proxy-integration/src/simple_protocol/multiple_statements.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, id, trace, PROXY}; + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; use fake::{Fake, Faker}; use tokio_postgres::SimpleQueryMessage::CommandComplete; @@ -12,7 +12,7 @@ mod tests { let client = connect_with_tls(PROXY).await; let data = (0..5) - .map(|_| (id(), Faker.fake::())) + .map(|_| (random_id(), Faker.fake::())) .collect::>(); // Build SQL string containing multiple statements; @@ -56,7 +56,7 @@ mod tests { let client = connect_with_tls(PROXY).await; let data = (0..5) - .map(|_| (id(), Faker.fake::(), Faker.fake::())) + .map(|_| (random_id(), Faker.fake::(), Faker.fake::())) .collect::>(); // Build SQL string containing multiple statements; diff --git a/packages/cipherstash-proxy/src/encrypt/mod.rs b/packages/cipherstash-proxy/src/encrypt/mod.rs index 83896c23..fab749dc 100644 --- a/packages/cipherstash-proxy/src/encrypt/mod.rs +++ b/packages/cipherstash-proxy/src/encrypt/mod.rs @@ -8,7 +8,7 @@ use crate::{ error::{EncryptError, Error}, log::ENCRYPT, postgresql::Column, - Identifier, + Identifier, EQL_SCHEMA_VERSION, }; use cipherstash_client::{ config::EnvSource, @@ -287,7 +287,7 @@ fn to_eql_encrypted( Ok(eql::EqlEncrypted { identifier: identifier.to_owned(), - version: 1, + version: EQL_SCHEMA_VERSION, body: EqlEncryptedBody { ciphertext, indexes: EqlEncryptedIndexes { @@ -347,7 +347,7 @@ fn to_eql_encrypted( // The way it's implemented right now is that it will be repeated one in the ste_vec_index. Ok(eql::EqlEncrypted { identifier: identifier.to_owned(), - version: 1, + version: EQL_SCHEMA_VERSION, body: EqlEncryptedBody { ciphertext: ciphertext.clone(), indexes: EqlEncryptedIndexes { diff --git a/packages/cipherstash-proxy/src/lib.rs b/packages/cipherstash-proxy/src/lib.rs index 2e81ca9e..62d1d2db 100644 --- a/packages/cipherstash-proxy/src/lib.rs +++ b/packages/cipherstash-proxy/src/lib.rs @@ -22,6 +22,8 @@ use std::mem; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +pub const EQL_SCHEMA_VERSION: u16 = 2; + pub const SIZE_U8: usize = mem::size_of::(); pub const SIZE_I16: usize = mem::size_of::(); pub const SIZE_I32: usize = mem::size_of::(); diff --git a/packages/eql-mapper/src/transformation_rules/group_by_eql_col.rs b/packages/eql-mapper/src/transformation_rules/group_by_eql_col.rs deleted file mode 100644 index 717b00d1..00000000 --- a/packages/eql-mapper/src/transformation_rules/group_by_eql_col.rs +++ /dev/null @@ -1,66 +0,0 @@ -use std::{collections::HashMap, mem, sync::Arc}; - -use sqltk::parser::ast::{ - helpers::attached_token::AttachedToken, Expr, GroupByExpr, Ident, ObjectName, -}; -use sqltk::parser::tokenizer::{Span, Token, TokenWithSpan}; -use sqltk::{NodeKey, NodePath, Visitable}; - -use crate::{EqlMapperError, Type, Value}; - -use super::{helpers, TransformationRule}; - -#[derive(Debug)] -pub struct GroupByEqlCol<'ast> { - node_types: Arc, Type>>, -} - -impl<'ast> GroupByEqlCol<'ast> { - pub fn new(node_types: Arc, Type>>) -> Self { - Self { node_types } - } -} - -impl<'ast> TransformationRule<'ast> for GroupByEqlCol<'ast> { - fn apply( - &mut self, - node_path: &NodePath<'ast>, - target_node: &mut N, - ) -> Result { - if self.would_edit(node_path, &*target_node) { - let target_node = target_node.downcast_mut::().unwrap(); - - // Nodes are modified starting from the leaf nodes, to target_node *is* what we want to be wrapping. - // So we steal the existing value and replace the original with a cheap placeholder (Expr::Wildcard). - // Stealing the original subtree means we can avoid cloning it. - let transformed_expr = mem::replace( - target_node, - Expr::Wildcard(AttachedToken(TokenWithSpan::new(Token::EOF, Span::empty()))), - ); - - *target_node = helpers::wrap_in_1_arg_function( - transformed_expr, - ObjectName(vec![ - Ident::new("eql_v2"), - Ident::new("ore_block_u64_8_256"), - ]), - ); - - return Ok(true); - } - - Ok(false) - } - - fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { - if let Some((_group_by_expr, _exprs, expr)) = - node_path.last_3_as::, Expr>() - { - if let Some(Type::Value(Value::Eql(_))) = self.node_types.get(&NodeKey::new(expr)) { - return true; - } - } - - false - } -} diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index ccf0117d..1c35aec2 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -1,24 +1,4 @@ -use sqltk::parser::ast::{ - CastKind, DataType, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, - FunctionArguments, Ident, ObjectName, -}; - -pub(crate) fn wrap_in_1_arg_function(expr: Expr, name: ObjectName) -> Expr { - Expr::Function(Function { - name, - parameters: FunctionArguments::None, - args: FunctionArguments::List(FunctionArgumentList { - args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(expr))], - duplicate_treatment: None, - clauses: vec![], - }), - filter: None, - null_treatment: None, - over: None, - within_group: vec![], - uses_odbc_syntax: false, - }) -} +use sqltk::parser::ast::{CastKind, DataType, Expr, Ident, ObjectName}; pub(crate) fn cast_as_encrypted(wrapped: sqltk::parser::ast::Value) -> Expr { let cast_jsonb = Expr::Cast { diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index e89b2c80..cf9f4346 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -14,7 +14,6 @@ mod helpers; mod cast_literals_as_encrypted; mod cast_params_as_encrypted; mod fail_on_placeholder_change; -mod group_by_eql_col; mod preserve_effective_aliases; mod rewrite_standard_sql_fns_on_eql_types; @@ -23,7 +22,6 @@ use std::marker::PhantomData; pub(crate) use cast_literals_as_encrypted::*; pub(crate) use cast_params_as_encrypted::*; pub(crate) use fail_on_placeholder_change::*; -pub(crate) use group_by_eql_col::*; pub(crate) use preserve_effective_aliases::*; pub(crate) use rewrite_standard_sql_fns_on_eql_types::*; diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index 7475982a..f36a6887 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -5,7 +5,7 @@ use sqltk::{AsNodeKey, NodeKey, Transformable}; use crate::{ CastLiteralsAsEncrypted, CastParamsAsEncrypted, DryRunnable, EqlMapperError, EqlValue, - FailOnPlaceholderChange, GroupByEqlCol, Param, PreserveEffectiveAliases, Projection, + FailOnPlaceholderChange, Param, PreserveEffectiveAliases, Projection, RewriteStandardSqlFnsOnEqlTypes, TransformationRule, Type, Value, }; @@ -140,9 +140,6 @@ impl<'ast> TypeCheckedStatement<'ast> { ) -> DryRunnable> { DryRunnable::new(( RewriteStandardSqlFnsOnEqlTypes::new(Arc::clone(&self.node_types)), - // WrapGroupedEqlColInAggregateFn::new(Arc::clone(&self.node_types)), - GroupByEqlCol::new(Arc::clone(&self.node_types)), - // WrapEqlColsInOrderByWithOreFn::new(Arc::clone(&self.node_types)), PreserveEffectiveAliases, CastLiteralsAsEncrypted::new(encrypted_literals), FailOnPlaceholderChange::new(),