From 38f006dd345ac7c8e9fc779261c8a88dc7b579ac Mon Sep 17 00:00:00 2001 From: Michelle Dhanani Date: Tue, 5 Nov 2024 21:47:18 -0500 Subject: [PATCH 1/2] spin 3 postgres support Signed-off-by: Michelle Dhanani Co-authored-by: Brian Hardock --- Cargo.lock | 39 +++ Cargo.toml | 2 + examples/postgres-v3/.cargo/config.toml | 2 + examples/postgres-v3/Cargo.toml | 16 ++ examples/postgres-v3/README.md | 57 ++++ examples/postgres-v3/db/testdata.sql | 22 ++ examples/postgres-v3/spin.toml | 17 ++ examples/postgres-v3/src/lib.rs | 132 +++++++++ src/lib.rs | 4 + src/pg3.rs | 329 ++++++++++++++++++++++ wit/deps/keyvalue-2024-10-17/atomic.wit | 46 +++ wit/deps/keyvalue-2024-10-17/batch.wit | 63 +++++ wit/deps/keyvalue-2024-10-17/store.wit | 122 ++++++++ wit/deps/keyvalue-2024-10-17/watch.wit | 16 ++ wit/deps/keyvalue-2024-10-17/world.wit | 26 ++ wit/deps/spin-postgres@3.0.0/postgres.wit | 100 +++++++ wit/world.wit | 2 + 17 files changed, 995 insertions(+) create mode 100644 examples/postgres-v3/.cargo/config.toml create mode 100644 examples/postgres-v3/Cargo.toml create mode 100644 examples/postgres-v3/README.md create mode 100644 examples/postgres-v3/db/testdata.sql create mode 100644 examples/postgres-v3/spin.toml create mode 100644 examples/postgres-v3/src/lib.rs create mode 100644 src/pg3.rs create mode 100644 wit/deps/keyvalue-2024-10-17/atomic.wit create mode 100644 wit/deps/keyvalue-2024-10-17/batch.wit create mode 100644 wit/deps/keyvalue-2024-10-17/store.wit create mode 100644 wit/deps/keyvalue-2024-10-17/watch.wit create mode 100644 wit/deps/keyvalue-2024-10-17/world.wit create mode 100644 wit/deps/spin-postgres@3.0.0/postgres.wit diff --git a/Cargo.lock b/Cargo.lock index 73fb8ea..89edf68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,6 +35,12 @@ version = "0.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -243,6 +249,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.52.3", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1224,6 +1244,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -1552,6 +1581,15 @@ dependencies = [ "spin-sdk", ] +[[package]] +name = "rust-outbound-pg-v3" +version = "0.1.0" +dependencies = [ + "anyhow", + "http 1.0.0", + "spin-sdk", +] + [[package]] name = "rust-outbound-redis" version = "0.1.0" @@ -1855,6 +1893,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", + "chrono", "form_urlencoded", "futures", "http 1.0.0", diff --git a/Cargo.toml b/Cargo.toml index d84aa71..1e39450 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ name = "spin_sdk" [dependencies] anyhow = "1" async-trait = "0.1.74" +chrono = "0.4.38" form_urlencoded = "1.0" spin-executor = { version = "3.0.1", path = "crates/executor" } spin-macro = { version = "3.0.1", path = "crates/macro" } @@ -52,6 +53,7 @@ members = [ "examples/key-value", "examples/mysql", "examples/postgres", + "examples/postgres-v3", "examples/redis-outbound", "examples/mqtt-outbound", "examples/variables", diff --git a/examples/postgres-v3/.cargo/config.toml b/examples/postgres-v3/.cargo/config.toml new file mode 100644 index 0000000..6b77899 --- /dev/null +++ b/examples/postgres-v3/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "wasm32-wasi" diff --git a/examples/postgres-v3/Cargo.toml b/examples/postgres-v3/Cargo.toml new file mode 100644 index 0000000..df486c9 --- /dev/null +++ b/examples/postgres-v3/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "rust-outbound-pg-v3" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +# Useful crate to handle errors. +anyhow = "1" +# General-purpose crate with common HTTP types. +http = "1.0.0" +# The Spin SDK. +spin-sdk = { path = "../.." } + diff --git a/examples/postgres-v3/README.md b/examples/postgres-v3/README.md new file mode 100644 index 0000000..cca7179 --- /dev/null +++ b/examples/postgres-v3/README.md @@ -0,0 +1,57 @@ +# Spin Outbound PostgreSQL example + +This example shows how to access a PostgreSQL database from Spin component. + +## Spin up + +From example root: + +``` +createdb spin_dev +psql -d spin_dev -f db/testdata.sql +RUST_LOG=spin=trace spin build --up +``` + +Curl the read route: + +``` +$ curl -i localhost:3000/read +HTTP/1.1 200 OK +transfer-encoding: chunked +date: Wed, 06 Nov 2024 20:17:03 GMT + +Found 2 article(s) as follows: +article: Article { + id: 1, + title: "My Life as a Goat", + content: "I went to Nepal to live as a goat, and it was much better than being a butler.", + authorname: "E. Blackadder", + published: Date( + 2024-11-05, + ), + coauthor: None, +} +article: Article { + id: 2, + title: "Magnificent Octopus", + content: "Once upon a time there was a lovely little sausage.", + authorname: "S. Baldrick", + published: Date( + 2024-11-06, + ), + coauthor: None, +} + +(Column info: id:DbDataType::Int32, title:DbDataType::Str, content:DbDataType::Str, authorname:DbDataType::Str, published:DbDataType::Date, coauthor:DbDataType::Str) +``` + +Curl the write route: + +``` +$ curl -i localhost:3000/write +HTTP/1.1 200 OK +content-length: 9 +date: Sun, 25 Sep 2022 15:46:22 GMT + +Count: 3 +``` diff --git a/examples/postgres-v3/db/testdata.sql b/examples/postgres-v3/db/testdata.sql new file mode 100644 index 0000000..2c8b667 --- /dev/null +++ b/examples/postgres-v3/db/testdata.sql @@ -0,0 +1,22 @@ +CREATE TABLE articletest ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + title varchar(40) NOT NULL, + content text NOT NULL, + authorname varchar(40) NOT NULL , + published date NOT NULL, + coauthor text +); + +INSERT INTO articletest (title, content, authorname, published) VALUES +( + 'My Life as a Goat', + 'I went to Nepal to live as a goat, and it was much better than being a butler.', + 'E. Blackadder', + '2024-11-05' +), +( + 'Magnificent Octopus', + 'Once upon a time there was a lovely little sausage.', + 'S. Baldrick', + '2024-11-06' +); diff --git a/examples/postgres-v3/spin.toml b/examples/postgres-v3/spin.toml new file mode 100644 index 0000000..04ca806 --- /dev/null +++ b/examples/postgres-v3/spin.toml @@ -0,0 +1,17 @@ +spin_manifest_version = 2 + +[application] +authors = ["Fermyon Engineering "] +name = "rust-outbound-pg-v3-example" +version = "0.1.0" + +[[trigger.http]] +route = "/..." +component = "outbound-pg" + +[component.outbound-pg] +environment = { DB_URL = "host=localhost user=postgres dbname=spin_dev" } +source = "../../target/wasm32-wasi/release/rust_outbound_pg_v3.wasm" +allowed_outbound_hosts = ["postgres://localhost"] +[component.outbound-pg.build] +command = "cargo build --target wasm32-wasi --release" diff --git a/examples/postgres-v3/src/lib.rs b/examples/postgres-v3/src/lib.rs new file mode 100644 index 0000000..0f9cb87 --- /dev/null +++ b/examples/postgres-v3/src/lib.rs @@ -0,0 +1,132 @@ +#![allow(dead_code)] +use anyhow::Result; +use http::{Request, Response}; +use spin_sdk::{ + http_component, pg3, + pg3::{Date, Decode}, +}; + +// The environment variable set in `spin.toml` that points to the +// address of the Pg server that the component will write to +const DB_URL_ENV: &str = "DB_URL"; + +#[derive(Debug, Clone)] +struct Article { + id: i32, + title: String, + content: String, + authorname: String, + published: Date, + coauthor: Option, +} + +impl TryFrom<&pg3::Row> for Article { + type Error = anyhow::Error; + + fn try_from(row: &pg3::Row) -> Result { + let id = i32::decode(&row[0])?; + let title = String::decode(&row[1])?; + let content = String::decode(&row[2])?; + let authorname = String::decode(&row[3])?; + let published = Date::decode(&row[4])?; + let coauthor = Option::::decode(&row[5])?; + + Ok(Self { + id, + title, + content, + authorname, + published, + coauthor, + }) + } +} + +#[http_component] +fn process(req: Request<()>) -> Result> { + match req.uri().path() { + "/read" => read(req), + "/write" => write(req), + "/pg_backend_pid" => pg_backend_pid(req), + _ => Ok(http::Response::builder() + .status(404) + .body("Not found".into())?), + } +} + +fn read(_req: Request<()>) -> Result> { + let address = std::env::var(DB_URL_ENV)?; + let conn = pg3::Connection::open(&address)?; + + let sql = "SELECT id, title, content, authorname, published, coauthor FROM articletest"; + let rowset = conn.query(sql, &[])?; + + let column_summary = rowset + .columns + .iter() + .map(format_col) + .collect::>() + .join(", "); + + let mut response_lines = vec![]; + + for row in rowset.rows { + let article = Article::try_from(&row)?; + + println!("article: {:#?}", article); + response_lines.push(format!("article: {:#?}", article)); + } + + // use it in business logic + + let response = format!( + "Found {} article(s) as follows:\n{}\n\n(Column info: {})\n", + response_lines.len(), + response_lines.join("\n"), + column_summary, + ); + + Ok(http::Response::builder().status(200).body(response)?) +} + +fn write(_req: Request<()>) -> Result> { + let address = std::env::var(DB_URL_ENV)?; + let conn = pg3::Connection::open(&address)?; + + let sql = + "INSERT INTO articletest (title, content, authorname, published) VALUES ('aaa', 'bbb', 'ccc', '2024-01-01')"; + let nrow_executed = conn.execute(sql, &[])?; + + println!("nrow_executed: {}", nrow_executed); + + let sql = "SELECT COUNT(id) FROM articletest"; + let rowset = conn.query(sql, &[])?; + let row = &rowset.rows[0]; + let count = i64::decode(&row[0])?; + let response = format!("Count: {}\n", count); + + Ok(http::Response::builder().status(200).body(response)?) +} + +fn pg_backend_pid(_req: Request<()>) -> Result> { + let address = std::env::var(DB_URL_ENV)?; + let conn = pg3::Connection::open(&address)?; + let sql = "SELECT pg_backend_pid()"; + + let get_pid = || { + let rowset = conn.query(sql, &[])?; + let row = &rowset.rows[0]; + + i32::decode(&row[0]) + }; + + assert_eq!(get_pid()?, get_pid()?); + + let response = format!("pg_backend_pid: {}\n", get_pid()?); + + Ok(http::Response::builder().status(200).body(response)?) +} + +fn format_col(column: &pg3::Column) -> String { + format!("{}:{:?}", column.name, column.data_type) +} diff --git a/src/lib.rs b/src/lib.rs index 99bd617..53f6b6f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,6 +34,7 @@ pub mod wit { } }); pub use fermyon::spin2_0_0 as v2; + pub use spin::postgres::postgres as pg3; } /// Needed by the export macro @@ -101,6 +102,9 @@ pub mod redis { /// Implementation of the spin postgres db interface. pub mod pg; +/// Implementation of the spin postgres v3 db interface. +pub mod pg3; + /// Implementation of the Spin MySQL database interface. pub mod mysql; diff --git a/src/pg3.rs b/src/pg3.rs new file mode 100644 index 0000000..a416488 --- /dev/null +++ b/src/pg3.rs @@ -0,0 +1,329 @@ +//! Conversions between Rust, WIT and **Postgres** types. +//! +//! # Types +//! +//! | Rust type | WIT (db-value) | Postgres type(s) | +//! |------------|-----------------------------------------------|----------------------------- | +//! | `bool` | boolean(bool) | BOOL | +//! | `i16` | int16(s16) | SMALLINT, SMALLSERIAL, INT2 | +//! | `i32` | int32(s32) | INT, SERIAL, INT4 | +//! | `i64` | int64(s64) | BIGINT, BIGSERIAL, INT8 | +//! | `f32` | floating32(float32) | REAL, FLOAT4 | +//! | `f64` | floating64(float64) | DOUBLE PRECISION, FLOAT8 | +//! | `String` | str(string) | VARCHAR, CHAR(N), TEXT | +//! | `Vec` | binary(list\) | BYTEA | +//! | `Date` | date(tuple) | DATE | +//! | `Time` | time(tuple) | TIME | +//! | `Datetime` | datetime(tuple) | TIMESTAMP | +//! | `Timestamp`| timestamp(s64) | BIGINT | + +#[doc(inline)] +pub use super::wit::pg3::{Error as PgError, *}; + +/// A pg error +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Failed to deserialize [`DbValue`] + #[error("error value decoding: {0}")] + Decode(String), + /// Pg query failed with an error + #[error(transparent)] + PgError(#[from] PgError), +} + +/// A type that can be decoded from the database. +pub trait Decode: Sized { + /// Decode a new value of this type using a [`DbValue`]. + fn decode(value: &DbValue) -> Result; +} + +impl Decode for Option +where + T: Decode, +{ + fn decode(value: &DbValue) -> Result { + match value { + DbValue::DbNull => Ok(None), + v => Ok(Some(T::decode(v)?)), + } + } +} + +impl Decode for bool { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Boolean(boolean) => Ok(*boolean), + _ => Err(Error::Decode(format_decode_err("BOOL", value))), + } + } +} + +impl Decode for i16 { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Int16(n) => Ok(*n), + _ => Err(Error::Decode(format_decode_err("SMALLINT", value))), + } + } +} + +impl Decode for i32 { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Int32(n) => Ok(*n), + _ => Err(Error::Decode(format_decode_err("INT", value))), + } + } +} + +impl Decode for i64 { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Int64(n) => Ok(*n), + _ => Err(Error::Decode(format_decode_err("BIGINT", value))), + } + } +} + +impl Decode for f32 { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Floating32(n) => Ok(*n), + _ => Err(Error::Decode(format_decode_err("REAL", value))), + } + } +} + +impl Decode for f64 { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Floating64(n) => Ok(*n), + _ => Err(Error::Decode(format_decode_err("DOUBLE PRECISION", value))), + } + } +} + +impl Decode for Vec { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Binary(n) => Ok(n.to_owned()), + _ => Err(Error::Decode(format_decode_err("BYTEA", value))), + } + } +} + +impl Decode for String { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Str(s) => Ok(s.to_owned()), + _ => Err(Error::Decode(format_decode_err( + "CHAR, VARCHAR, TEXT", + value, + ))), + } + } +} + +/// Native representation of the WIT postgres Date value. +#[derive(Clone, Debug, PartialEq)] +pub struct Date(pub chrono::NaiveDate); + +impl Decode for Date { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Date((year, month, day)) => { + let naive_date = + chrono::NaiveDate::from_ymd_opt(*year, (*month).into(), (*day).into()) + .ok_or_else(|| { + Error::Decode(format!( + "invalid date y={}, m={}, d={}", + year, month, day + )) + })?; + Ok(Date(naive_date)) + } + _ => Err(Error::Decode(format_decode_err("DATE", value))), + } + } +} + +/// Native representation of the WIT postgres Time value. +#[derive(Clone, Debug, PartialEq)] +pub struct Time(pub chrono::NaiveTime); + +impl Decode for Time { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Time((hour, minute, second, nanosecond)) => { + let naive_time = chrono::NaiveTime::from_hms_nano_opt( + (*hour).into(), + (*minute).into(), + (*second).into(), + *nanosecond, + ) + .ok_or_else(|| { + Error::Decode(format!( + "invalid time {}:{}:{}:{}", + hour, minute, second, nanosecond + )) + })?; + Ok(Time(naive_time)) + } + _ => Err(Error::Decode(format_decode_err("TIME", value))), + } + } +} + +/// Native representation of the WIT postgres DateTime value. +#[derive(Clone, Debug, PartialEq)] +pub struct DateTime(pub chrono::NaiveDateTime); + +impl Decode for DateTime { + fn decode(value: &DbValue) -> Result { + match value { + DbValue::Datetime((year, month, day, hour, minute, second, nanosecond)) => { + let naive_date = + chrono::NaiveDate::from_ymd_opt(*year, (*month).into(), (*day).into()) + .ok_or_else(|| { + Error::Decode(format!( + "invalid date y={}, m={}, d={}", + year, month, day + )) + })?; + let naive_time = chrono::NaiveTime::from_hms_nano_opt( + (*hour).into(), + (*minute).into(), + (*second).into(), + *nanosecond, + ) + .ok_or_else(|| { + Error::Decode(format!( + "invalid time {}:{}:{}:{}", + hour, minute, second, nanosecond + )) + })?; + let dt = chrono::NaiveDateTime::new(naive_date, naive_time); + Ok(DateTime(dt)) + } + _ => Err(Error::Decode(format_decode_err("DATETIME", value))), + } + } +} + +fn format_decode_err(types: &str, value: &DbValue) -> String { + format!("Expected {} from the DB but got {:?}", types, value) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn boolean() { + assert!(bool::decode(&DbValue::Boolean(true)).unwrap()); + assert!(bool::decode(&DbValue::Int32(0)).is_err()); + assert!(Option::::decode(&DbValue::DbNull).unwrap().is_none()); + } + + #[test] + fn int16() { + assert_eq!(i16::decode(&DbValue::Int16(0)).unwrap(), 0); + assert!(i16::decode(&DbValue::Int32(0)).is_err()); + assert!(Option::::decode(&DbValue::DbNull).unwrap().is_none()); + } + + #[test] + fn int32() { + assert_eq!(i32::decode(&DbValue::Int32(0)).unwrap(), 0); + assert!(i32::decode(&DbValue::Boolean(false)).is_err()); + assert!(Option::::decode(&DbValue::DbNull).unwrap().is_none()); + } + + #[test] + fn int64() { + assert_eq!(i64::decode(&DbValue::Int64(0)).unwrap(), 0); + assert!(i64::decode(&DbValue::Boolean(false)).is_err()); + assert!(Option::::decode(&DbValue::DbNull).unwrap().is_none()); + } + + #[test] + fn floating32() { + assert!(f32::decode(&DbValue::Floating32(0.0)).is_ok()); + assert!(f32::decode(&DbValue::Boolean(false)).is_err()); + assert!(Option::::decode(&DbValue::DbNull).unwrap().is_none()); + } + + #[test] + fn floating64() { + assert!(f64::decode(&DbValue::Floating64(0.0)).is_ok()); + assert!(f64::decode(&DbValue::Boolean(false)).is_err()); + assert!(Option::::decode(&DbValue::DbNull).unwrap().is_none()); + } + + #[test] + fn str() { + assert_eq!( + String::decode(&DbValue::Str(String::from("foo"))).unwrap(), + String::from("foo") + ); + + assert!(String::decode(&DbValue::Int32(0)).is_err()); + assert!(Option::::decode(&DbValue::DbNull) + .unwrap() + .is_none()); + } + + #[test] + fn binary() { + assert!(Vec::::decode(&DbValue::Binary(vec![0, 0])).is_ok()); + assert!(Vec::::decode(&DbValue::Boolean(false)).is_err()); + assert!(Option::>::decode(&DbValue::DbNull) + .unwrap() + .is_none()); + } + + #[test] + fn date() { + assert_eq!( + Date::decode(&DbValue::Date((1, 2, 4))).unwrap(), + Date(chrono::NaiveDate::from_ymd_opt(1, 2, 4).unwrap()) + ); + assert_ne!( + Date::decode(&DbValue::Date((1, 2, 4))).unwrap(), + Date(chrono::NaiveDate::from_ymd_opt(1, 2, 5).unwrap()) + ); + assert!(Option::::decode(&DbValue::DbNull).unwrap().is_none()); + } + + #[test] + fn time() { + assert_eq!( + Time::decode(&DbValue::Time((1, 2, 3, 4))).unwrap(), + Time(chrono::NaiveTime::from_hms_nano_opt(1, 2, 3, 4).unwrap()) + ); + assert_ne!( + Time::decode(&DbValue::Time((1, 2, 3, 4))).unwrap(), + Time(chrono::NaiveTime::from_hms_nano_opt(1, 2, 4, 5).unwrap()) + ); + assert!(Option::