diff --git a/CHANGELOG.md b/CHANGELOG.md index 187f95ed..bb48e142 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,15 @@ - Since modals have their own url inside the page, you can now link to a modal from another page, and if you refresh a page while the modal is open, the modal will stay open. - modals now have an `open` parameter to open the modal automatically when the page is loaded. - New [download](https://sql-page.com/component.sql?component=download) component to let the user download files. The files may be stored as BLOBs in the database, local files on the server, or may be fetched from a different server. + - **Enhanced BLOB Support**. You can now return binary data (BLOBs) directly to sqlpage, and it will automatically convert them to data URLs. This allows you to use database BLOBs directly wherever a link is expected, including in the new download component. + - supports columns of type `BYTEA` (PostgreSQL), `BLOB` (MySQL, SQLite), `VARBINARY` and `IMAGE` (mssql) + - Automatic detection of common file types based on magic bytes + - This means you can use a BLOB wherever an image url is expected. For instance: + ```sql + select 'list' as component; + select username as title, avatar_blob as image_url + from users; + ``` ## v0.36.1 - Fix regression introduced in v0.36.0: PostgreSQL money values showed as 0.0 diff --git a/docker-compose.yml b/docker-compose.yml index 8c04db65..561fc39c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,11 @@ # DATABASE_URL='postgres://root:Password123!@localhost/sqlpage' # DATABASE_URL='mssql://root:Password123!@localhost/sqlpage' # DATABASE_URL='mysql://root:Password123!@localhost/sqlpage' + +# Run for instance: +# docker compose up postgres +# and in another terminal: +# DATABASE_URL='db_url' cargo test services: web: build: { context: "." } diff --git a/examples/official-site/extensions-to-sql.md b/examples/official-site/extensions-to-sql.md index e1ad9f70..81ee0902 100644 --- a/examples/official-site/extensions-to-sql.md +++ b/examples/official-site/extensions-to-sql.md @@ -206,3 +206,52 @@ SET post_id = COALESCE($post_id, 0); -- Prepared statement (SQLite syntax) SELECT COALESCE(CAST(?1 AS TEXT), 0) ``` + +# Data types + +Each database has its own rich set of data types. +The data modal in SQLPage itself is simpler, mainly composed of text strings and json objects. + +### From the user to SQLPage + +Form fields and URL parameters may contain arrays. These are converted to JSON strings before processing. + +For instance, Loading `users.sql?user[]=Tim&user[]=Tom` will result in a single variable `$user` with the textual value `["Tim", "Tom"]`. + +### From SQLPage to the database + +SQLPage sends only text strings (`VARCHAR`) and `NULL`s to the database, since these are the only possible variable and function return values. + +### From the database to SQLPage + +Each row of data returned by a SQL query is converted to a JSON object before being passed to components. + +- Each column becomes a key in the json object. If a row has two columns of the same name, they become an array in the json object. +- Each value is converted to the closest JSON value + - all number types map to json numbers, booleans to booleans, and `NULL` to `null`, + - all text types map to json strings + - date and time types map to json strings containing ISO datetime values + - binary values (BLOBs) map to json strings containing [data URLs](https://developer.mozilla.org/en-US/docs/Web/URI/Reference/Schemes/data) + +#### Example + +The following PostgreSQL query: + +```sql +select + 1 as one, + 'x' as my_array, 'y' as my_array, + now() as today, + ''::bytea as my_image; +``` + +will result in the following JSON object being passed to components for rendering + +```json +{ + "one" : 1, + "my_array" : ["x","y"], + "today":"2025-08-30T06:40:13.894918+00:00", + "my_image":"data:image/svg+xml;base64,PHN2Zz48L3N2Zz4=" +} +``` \ No newline at end of file diff --git a/examples/official-site/extensions-to-sql.sql b/examples/official-site/extensions-to-sql.sql index dc5e3125..752eac72 100644 --- a/examples/official-site/extensions-to-sql.sql +++ b/examples/official-site/extensions-to-sql.sql @@ -1,7 +1,10 @@ select 'http_header' as component, 'public, max-age=300, stale-while-revalidate=3600, stale-if-error=86400' as "Cache-Control"; -select 'dynamic' as component, properties FROM example WHERE component = 'shell' LIMIT 1; +select 'dynamic' as component, json_patch(json_extract(properties, '$[0]'), json_object( + 'title', 'SQLPage - Extensions to SQL' +)) as properties +FROM example WHERE component = 'shell' LIMIT 1; -- Article by Matthew Larkin select 'text' as component, diff --git a/examples/official-site/sqlpage/migrations/65_download.sql b/examples/official-site/sqlpage/migrations/65_download.sql index 11bf3dd5..61c005a3 100644 --- a/examples/official-site/sqlpage/migrations/65_download.sql +++ b/examples/official-site/sqlpage/migrations/65_download.sql @@ -104,7 +104,25 @@ select ' ## Serve an image stored as a BLOB in the database -In PostgreSQL, you can use the [encode(bytes, format)](https://www.postgresql.org/docs/current/functions-binarystring.html#FUNCTION-ENCODE) function to encode the file content as Base64. +### Automatically detect the mime type + +If you have a table with a column `content` that contains a BLOB +(depending on the database, the type may be named `BYTEA`, `BLOB`, `VARBINARY`, or `IMAGE`), +you can just return its contents directly, and SQLPage will automatically detect the mime type, +and convert it to a data URL. + +```sql +select + ''download'' as component, + content as data_url +from document +where id = $doc_id; +``` + +### Customize the mime type + +In PostgreSQL, you can use the [encode(bytes, format)](https://www.postgresql.org/docs/current/functions-binarystring.html#FUNCTION-ENCODE) function to encode the file content as Base64, +and manually create your own data URL. ```sql select diff --git a/src/webserver/database/blob_to_data_url.rs b/src/webserver/database/blob_to_data_url.rs new file mode 100644 index 00000000..b8e1fad0 --- /dev/null +++ b/src/webserver/database/blob_to_data_url.rs @@ -0,0 +1,189 @@ +/// Detects MIME type based on file signatures (magic bytes). +/// Returns the most appropriate MIME type for common file formats. +#[must_use] +pub fn detect_mime_type(bytes: &[u8]) -> &'static str { + // PNG: 89 50 4E 47 0D 0A 1A 0A + if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { + return "image/png"; + } + // JPEG: FF D8 + if bytes.starts_with(b"\xFF\xD8") { + return "image/jpeg"; + } + // GIF87a/89a: GIF87a or GIF89a + if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { + return "image/gif"; + } + // BMP: 42 4D + if bytes.starts_with(b"BM") { + return "image/bmp"; + } + // WebP: RIFF....WEBP + if bytes.starts_with(b"RIFF") && bytes.len() >= 12 && &bytes[8..12] == b"WEBP" { + return "image/webp"; + } + // PDF: %PDF + if bytes.starts_with(b"%PDF") { + return "application/pdf"; + } + // ZIP: 50 4B 03 04 + if bytes.starts_with(b"PK\x03\x04") { + // Check for Office document types in ZIP central directory + if bytes.len() >= 50 { + let central_dir = &bytes[30..bytes.len().min(50)]; + if central_dir.windows(6).any(|w| w == b"word/") { + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + } + if central_dir.windows(3).any(|w| w == b"xl/") { + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + } + if central_dir.windows(4).any(|w| w == b"ppt/") { + return "application/vnd.openxmlformats-officedocument.presentationml.presentation"; + } + } + return "application/zip"; + } + + if bytes.starts_with(b" String { + let mime_type = detect_mime_type(bytes); + vec_to_data_uri_with_mime(bytes, mime_type) +} + +/// Converts binary data to a data URL string with a specific MIME type. +/// This function is used by both SQL type conversion and file reading functions. +#[must_use] +pub fn vec_to_data_uri_with_mime(bytes: &[u8], mime_type: &str) -> String { + let mut data_url = format!("data:{mime_type};base64,"); + base64::Engine::encode_string( + &base64::engine::general_purpose::STANDARD, + bytes, + &mut data_url, + ); + data_url +} + +/// Converts binary data to a data URL JSON value. +/// This is a convenience function for SQL type conversion. +#[must_use] +pub fn vec_to_data_uri_value(bytes: &[u8]) -> serde_json::Value { + serde_json::Value::String(vec_to_data_uri(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_detect_mime_type() { + // Test empty data + assert_eq!(detect_mime_type(&[]), "application/octet-stream"); + + // Test PNG + assert_eq!(detect_mime_type(b"\x89PNG\r\n\x1a\n"), "image/png"); + + // Test JPEG + assert_eq!(detect_mime_type(b"\xFF\xD8\xFF\xE0"), "image/jpeg"); + + // Test GIF87a + assert_eq!(detect_mime_type(b"GIF87a"), "image/gif"); + + // Test GIF89a + assert_eq!(detect_mime_type(b"GIF89a"), "image/gif"); + + // Test BMP + assert_eq!(detect_mime_type(b"BM\x00\x00"), "image/bmp"); + + // Test PDF + assert_eq!(detect_mime_type(b"%PDF-"), "application/pdf"); + + // Test SVG + assert_eq!( + detect_mime_type(b""), + "image/svg+xml" + ); + + // Test XML (non-SVG) + assert_eq!( + detect_mime_type(b"test"), + "application/xml" + ); + + // Test JSON + assert_eq!( + detect_mime_type(b"{\"key\": \"value\"}"), + "application/json" + ); + + // Test ZIP + assert_eq!(detect_mime_type(b"PK\x03\x04"), "application/zip"); + + // Test unknown data + assert_eq!( + detect_mime_type(&[0x00, 0x01, 0x02, 0x03]), + "application/octet-stream" + ); + } + + #[test] + fn test_vec_to_data_uri() { + // Test with empty bytes + let result = vec_to_data_uri(&[]); + assert_eq!(result, "data:application/octet-stream;base64,"); + + // Test with simple text + let result = vec_to_data_uri(b"Hello World"); + assert_eq!( + result, + "data:application/octet-stream;base64,SGVsbG8gV29ybGQ=" + ); + + // Test with binary data + let binary_data = [0, 1, 2, 255, 254, 253]; + let result = vec_to_data_uri(&binary_data); + assert_eq!(result, "data:application/octet-stream;base64,AAEC//79"); + } + + #[test] + fn test_vec_to_data_uri_with_mime() { + // Test with custom MIME type + let result = vec_to_data_uri_with_mime(b"Hello", "text/plain"); + assert_eq!(result, "data:text/plain;base64,SGVsbG8="); + + // Test with image MIME type + let result = vec_to_data_uri_with_mime(&[255, 216, 255], "image/jpeg"); + assert_eq!(result, "data:image/jpeg;base64,/9j/"); + + // Test with empty bytes and custom MIME + let result = vec_to_data_uri_with_mime(&[], "application/json"); + assert_eq!(result, "data:application/json;base64,"); + } + + #[test] + fn test_vec_to_data_uri_value() { + // Test that it returns a JSON string value + let result = vec_to_data_uri_value(b"test"); + match result { + serde_json::Value::String(s) => { + assert_eq!(s, "data:application/octet-stream;base64,dGVzdA=="); + } + _ => panic!("Expected String value"), + } + } +} diff --git a/src/webserver/database/mod.rs b/src/webserver/database/mod.rs index e9f0949e..49e465c5 100644 --- a/src/webserver/database/mod.rs +++ b/src/webserver/database/mod.rs @@ -1,3 +1,4 @@ +pub mod blob_to_data_url; mod connect; mod csv_import; pub mod execute_queries; diff --git a/src/webserver/database/sql_to_json.rs b/src/webserver/database/sql_to_json.rs index f54fe2ef..5c720ce1 100644 --- a/src/webserver/database/sql_to_json.rs +++ b/src/webserver/database/sql_to_json.rs @@ -1,4 +1,5 @@ use crate::utils::add_value_to_map; +use crate::webserver::database::blob_to_data_url; use chrono::{DateTime, FixedOffset, NaiveDateTime}; use serde_json::{self, Map, Value}; use sqlx::any::{AnyRow, AnyTypeInfo, AnyTypeInfoKind}; @@ -96,6 +97,9 @@ pub fn sql_nonnull_to_json<'r>(mut get_ref: impl FnMut() -> sqlx::any::AnyValueR decode_raw::(raw_value).into() } "JSON" | "JSON[]" | "JSONB" | "JSONB[]" => decode_raw::(raw_value), + "BLOB" | "BYTEA" | "FILESTREAM" | "VARBINARY" | "BIGVARBINARY" | "BINARY" | "IMAGE" => { + blob_to_data_url::vec_to_data_uri_value(&decode_raw::>(raw_value)) + } // Deserialize as a string by default _ => decode_raw::(raw_value).into(), } @@ -171,7 +175,7 @@ mod tests { }; let mut c = sqlx::AnyConnection::connect(&db_url).await?; let row = sqlx::query( - "SELECT + "SELECT 42::INT2 as small_int, 42::INT4 as integer, 42::INT8 as big_int, @@ -189,7 +193,8 @@ mod tests { '{\"key\": \"value\"}'::JSONB as jsonb, age('2024-03-14'::timestamp, '2024-01-01'::timestamp) as age_interval, justify_interval(interval '1 year 2 months 3 days') as justified_interval, - 1234.56::MONEY as money_val", + 1234.56::MONEY as money_val, + '\\x68656c6c6f20776f726c64'::BYTEA as blob_data", ) .fetch_one(&mut c) .await?; @@ -214,7 +219,8 @@ mod tests { "jsonb": {"key": "value"}, "age_interval": "2 mons 13 days", "justified_interval": "1 year 2 mons 3 days", - "money_val": "$1,234.56" + "money_val": "$1,234.56", + "blob_data": "data:application/octet-stream;base64,aGVsbG8gd29ybGQ=" }), ); Ok(()) @@ -235,7 +241,8 @@ mod tests { '2024-03-14 13:14:15+02:00'::TIMESTAMPTZ as timestamptz, INTERVAL '-01:02:03' as time_interval, '{\"key\": \"value\"}'::JSON as json, - 1234.56::MONEY as money_val + 1234.56::MONEY as money_val, + '\\x74657374'::BYTEA as blob_data where $1", ) .bind(true) @@ -250,7 +257,8 @@ mod tests { "timestamptz": "2024-03-14T11:14:15+00:00", "time_interval": "-01:02:03", "json": {"key": "value"}, - "money_val": "" // TODO: fix this bug: https://github.com/sqlpage/SQLPage/issues/983 + "money_val": "", // TODO: fix this bug: https://github.com/sqlpage/SQLPage/issues/983 + "blob_data": "data:application/octet-stream;base64,dGVzdA==" }), ); Ok(()) @@ -287,9 +295,10 @@ mod tests { year_val YEAR, char_val CHAR(10), varchar_val VARCHAR(50), - text_val TEXT - ) AS - SELECT + text_val TEXT, + blob_val BLOB + ) AS + SELECT 127 as tiny_int, 32767 as small_int, 8388607 as medium_int, @@ -311,7 +320,8 @@ mod tests { 2024 as year_val, 'CHAR' as char_val, 'VARCHAR' as varchar_val, - 'TEXT' as text_val", + 'TEXT' as text_val, + x'626c6f62' as blob_val", ) .execute(&mut c) .await?; @@ -344,7 +354,8 @@ mod tests { "year_val": 2024, "char_val": "CHAR", "varchar_val": "VARCHAR", - "text_val": "TEXT" + "text_val": "TEXT", + "blob_val": "data:application/octet-stream;base64,YmxvYg==" }), ); @@ -375,7 +386,7 @@ mod tests { "integer": 42, "real": 42.25, "string": "xxx", - "blob": "hello world", + "blob": "data:application/octet-stream;base64,aGVsbG8gd29ybGQ=", }), ); Ok(()) @@ -388,7 +399,7 @@ mod tests { }; let mut c = sqlx::AnyConnection::connect(&db_url).await?; let row = sqlx::query( - "SELECT + "SELECT CAST(1 AS BIT) as true_bit, CAST(0 AS BIT) as false_bit, CAST(NULL AS BIT) as null_bit, @@ -407,7 +418,8 @@ mod tests { N'Unicode String' as nvarchar, 'ASCII String' as varchar, CAST(1234.56 AS MONEY) as money_val, - CAST(12.34 AS SMALLMONEY) as small_money_val", + CAST(12.34 AS SMALLMONEY) as small_money_val, + CAST(0x6D7373716C AS VARBINARY(10)) as blob_data", ) .fetch_one(&mut c) .await?; @@ -433,7 +445,8 @@ mod tests { "nvarchar": "Unicode String", "varchar": "ASCII String", "money_val": 1234.56, - "small_money_val": 12.34 + "small_money_val": 12.34, + "blob_data": "data:application/octet-stream;base64,bXNzcWw=" }), ); Ok(()) diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index 2fe036e8..04e00da3 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -1,7 +1,8 @@ use super::RequestInfo; use crate::webserver::{ database::{ - execute_queries::DbConn, sqlpage_functions::url_parameter_deserializer::URLParameters, + blob_to_data_url::vec_to_data_uri_with_mime, execute_queries::DbConn, + sqlpage_functions::url_parameter_deserializer::URLParameters, }, http::SingleOrVec, http_client::make_http_client, @@ -504,12 +505,7 @@ async fn read_file_as_data_url<'a>( || Cow::Owned(mime_guess_from_filename(&file_path)), Cow::Borrowed, ); - let mut data_url = format!("data:{mime};base64,"); - base64::Engine::encode_string( - &base64::engine::general_purpose::STANDARD, - bytes, - &mut data_url, - ); + let data_url = vec_to_data_uri_with_mime(&bytes, &mime.to_string()); Ok(Some(Cow::Owned(data_url))) }