json: support streaming query results as json array, jsonlines, or server-sent events

lovasoa · lovasoa · commit f8241a9e7832 · 2024-10-22T23:45:53.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,9 @@
  - New `initial_search_value` property in the table component to pre-fill the search bar with a value. This allows to display the table rows that will initially be filtered out by the search bar.
  - Fix autoplay of carousels when embedded in a card.
  - Allow setting image width and height in carousels, in order to avoid differently sized images to cause layout janking when going through them. 
+ - Many improvements to the [json](https://sql.datapage.app/component.sql?component=json) component, making it easier and faster than ever to build REST APIs entirely in SQL.
+  - **Ease of use** : the component can now be used to automatically format any query result as a json array, without manually using your database''s json functions.
+  - **server-sent events** : the component can now be used to stream query results to the client in real-time using server-sent events.
 
 ## 0.29.0 (2024-09-25)
  - New columns component: `columns`. Useful to display a comparison between items, or large key figures to an user.
diff --git a/examples/official-site/sqlpage/migrations/11_json.sql b/examples/official-site/sqlpage/migrations/11_json.sql
@@ -1,35 +1,94 @@
-INSERT INTO component (name, description, icon, introduced_in_version)
-VALUES (
+INSERT INTO
+    component (name, description, icon, introduced_in_version)
+VALUES
+    (
         'json',
         'For advanced users, allows you to easily build an API over your database.
         The json component responds to the current HTTP request with a JSON object.
         This component must appear at the top of your SQL file, before any other data has been sent to the browser.',
         'code',
         '0.9.0'
     );
+
 -- Insert the parameters for the http_header component into the parameter table
-INSERT INTO parameter (
+INSERT INTO
+    parameter (
         component,
         name,
         description,
         type,
         top_level,
         optional
     )
-VALUES (
+VALUES
+    (
         'json',
         'contents',
-        'The JSON payload to send. You should use your database''s built-in json functions to build the value to enter here.',
+        'A single JSON payload to send. You can use your database''s built-in json functions to build the value to enter here. If not provided, the contents will be taken from the next SQL statements and rendered as a JSON array.',
         'TEXT',
         TRUE,
-        FALSE
+        TRUE
+    ),
+    (
+        'json',
+        'type',
+        'The type of the JSON payload to send. Defaults to "array" (each query result is rendered as a JSON object in the array). Other possible values are "jsonlines" (each query result is rendered as a JSON object in a new line, without a top-level array) and "sse" (each query result is rendered as a JSON object in a new line, prefixed by "data: ", which allows you to read the results as server-sent events in real-time from javascript).',
+        'TEXT',
+        TRUE,
+        TRUE
     );
+
 -- Insert an example usage of the http_header component into the example table
-INSERT INTO example (component, description)
-VALUES (
+INSERT INTO
+    example (component, description)
+VALUES
+    (
+        'json',
+        '
+## Send query results as a JSON array
+
+### SQL
+
+```sql
+select ''json'' AS component;
+select * from users;
+```
+
+### Result
+
+```json
+[
+    {"username":"James","userid":1},
+    {"username":"John","userid":2}
+]
+```
+    '
+    ),
+    (
+        'json',
+        '
+## Send a single JSON object
+
+### SQL
+
+```sql
+select ''json'' AS component, ''jsonlines'' AS type;
+select * from users where id = $user_id LIMIT 1;
+```
+
+### Result
+
+```json
+{ "username":"James", "userid":1 }
+```
+'
+    ),
+    (
         'json',
         '
-Creates an API endpoint that will allow developers to easily query a list of users stored in your database.
+## Create a complex API endpoint
+
+This will create an API endpoint that will allow developers to easily query a list of users stored in your database.
 
 You should use [the json functions provided by your database](/blog.sql?post=JSON%20in%20SQL%3A%20A%20Comprehensive%20Guide) to form the value you pass to the `contents` property.
 To build a json array out of rows from the database, you can use: 
@@ -69,4 +128,30 @@ you can use
  - the [`request_method` function](/functions.sql?function=request_method#function) to differentiate between GET and POST requests,
  - and the [`path` function](/functions.sql?function=path#function) to extract the `:id` parameter from the URL.
 '
-    );
+    ),
+    (
+        'json',
+        '
+## Access query results in real-time with server-sent events
+
+Using server-sent events, you can stream query results to the client in real-time.
+This means you can build dynamic applications that will process data as it arrives.
+
+### SQL
+
+```sql
+select ''json'' AS component, ''sse'' AS type;
+select * from users;
+```
+
+### JavaScript
+
+```javascript
+const eventSource = new EventSource("users.sql");
+eventSource.onmessage = function (event) {
+    const user = JSON.parse(event.data);
+    console.log(user.username);
+}
+```
+'
+    );
diff --git a/src/render.rs b/src/render.rs
@@ -59,7 +59,7 @@ impl<'a, W: std::io::Write> HeaderContext<W> {
             Some("status_code") => self.status_code(&data).map(PageContext::Header),
             Some("http_header") => self.add_http_header(&data).map(PageContext::Header),
             Some("redirect") => self.redirect(&data).map(PageContext::Close),
-            Some("json") => self.json(&data).map(PageContext::Close),
+            Some("json") => self.json(&data),
             Some("cookie") => self.add_cookie(&data).map(PageContext::Header),
             Some("authentication") => self.authentication(data).await,
             _ => self.start_body(data).await,
@@ -187,18 +187,37 @@ impl<'a, W: std::io::Write> HeaderContext<W> {
     }
 
     /// Answers to the HTTP request with a single json object
-    fn json(mut self, data: &JsonValue) -> anyhow::Result<HttpResponse> {
-        let contents = data
-            .get("contents")
-            .with_context(|| "Missing 'contents' property for the json component")?;
-        let json_response = if let Some(s) = contents.as_str() {
-            s.as_bytes().to_owned()
-        } else {
-            serde_json::to_vec(contents)?
-        };
+    fn json(mut self, data: &JsonValue) -> anyhow::Result<PageContext<W>> {
         self.response
             .insert_header((header::CONTENT_TYPE, "application/json"));
-        Ok(self.response.body(json_response))
+        if let Some(contents) = data.get("contents") {
+            let json_response = if let Some(s) = contents.as_str() {
+                s.as_bytes().to_owned()
+            } else {
+                serde_json::to_vec(contents)?
+            };
+            Ok(PageContext::Close(self.response.body(json_response)))
+        } else {
+            let body_type = get_object_str(data, "type");
+            let json_renderer = match body_type {
+                None | Some("array") => JsonBodyRenderer::new_array(self.writer),
+                Some("jsonlines") => JsonBodyRenderer::new_jsonlines(self.writer),
+                Some("sse") => {
+                    self.response
+                        .insert_header((header::CONTENT_TYPE, "text/event-stream"));
+                    JsonBodyRenderer::new_server_sent_events(self.writer)
+                }
+                _ => bail!(
+                    "Invalid value for the 'type' property of the json component: {body_type:?}"
+                ),
+            };
+            let renderer = AnyRenderBodyContext::Json(json_renderer);
+            let http_response = self.response;
+            Ok(PageContext::Body {
+                http_response,
+                renderer,
+            })
+        }
     }
 
     async fn authentication(mut self, mut data: JsonValue) -> anyhow::Result<PageContext<W>> {
@@ -290,41 +309,121 @@ fn take_object_str(json: &mut JsonValue, key: &str) -> Option<String> {
  */
 pub enum AnyRenderBodyContext<W: std::io::Write> {
     Html(HtmlRenderContext<W>),
+    Json(JsonBodyRenderer<W>),
 }
 
 /**
  * Dummy impl to dispatch method calls to the underlying renderer
  */
 impl<W: std::io::Write> AnyRenderBodyContext<W> {
     pub async fn handle_row(&mut self, data: &JsonValue) -> anyhow::Result<()> {
+        log::debug!(
+            "<- Rendering properties: {}",
+            serde_json::to_string(&data).unwrap_or_else(|e| e.to_string())
+        );
         match self {
             AnyRenderBodyContext::Html(render_context) => render_context.handle_row(data).await,
+            AnyRenderBodyContext::Json(json_body_renderer) => json_body_renderer.handle_row(data),
         }
     }
     pub async fn handle_error(&mut self, error: &anyhow::Error) -> anyhow::Result<()> {
+        log::error!("SQL error: {:?}", error);
         match self {
             AnyRenderBodyContext::Html(render_context) => render_context.handle_error(error).await,
+            AnyRenderBodyContext::Json(json_body_renderer) => {
+                json_body_renderer.handle_error(error)
+            }
         }
     }
     pub async fn finish_query(&mut self) -> anyhow::Result<()> {
         match self {
             AnyRenderBodyContext::Html(render_context) => render_context.finish_query().await,
+            AnyRenderBodyContext::Json(_json_body_renderer) => Ok(()),
         }
     }
 
     pub fn writer_mut(&mut self) -> &mut W {
         match self {
-            AnyRenderBodyContext::Html(HtmlRenderContext { writer, .. }) => writer,
+            AnyRenderBodyContext::Html(HtmlRenderContext { writer, .. })
+            | AnyRenderBodyContext::Json(JsonBodyRenderer { writer, .. }) => writer,
         }
     }
 
     pub async fn close(self) -> W {
         match self {
             AnyRenderBodyContext::Html(render_context) => render_context.close().await,
+            AnyRenderBodyContext::Json(json_body_renderer) => json_body_renderer.close(),
         }
     }
 }
 
+pub struct JsonBodyRenderer<W: std::io::Write> {
+    writer: W,
+    is_first: bool,
+    prefix: &'static [u8],
+    suffix: &'static [u8],
+    separator: &'static [u8],
+}
+
+impl<W: std::io::Write> JsonBodyRenderer<W> {
+    pub fn new_array(writer: W) -> JsonBodyRenderer<W> {
+        let mut renderer = Self {
+            writer,
+            is_first: true,
+            prefix: b"[\n",
+            suffix: b"\n]",
+            separator: b",\n",
+        };
+        let _ = renderer.write_prefix();
+        renderer
+    }
+    pub fn new_jsonlines(writer: W) -> JsonBodyRenderer<W> {
+        let mut renderer = Self {
+            writer,
+            is_first: true,
+            prefix: b"",
+            suffix: b"",
+            separator: b"\n",
+        };
+        renderer.write_prefix().unwrap();
+        renderer
+    }
+    pub fn new_server_sent_events(writer: W) -> JsonBodyRenderer<W> {
+        let mut renderer = Self {
+            writer,
+            is_first: true,
+            prefix: b"data: ",
+            suffix: b"",
+            separator: b"\n\ndata: ",
+        };
+        renderer.write_prefix().unwrap();
+        renderer
+    }
+    fn write_prefix(&mut self) -> anyhow::Result<()> {
+        self.writer.write_all(self.prefix)?;
+        Ok(())
+    }
+    pub fn handle_row(&mut self, data: &JsonValue) -> anyhow::Result<()> {
+        if self.is_first {
+            self.is_first = false;
+        } else {
+            let _ = self.writer.write_all(self.separator);
+        }
+        serde_json::to_writer(&mut self.writer, data)?;
+        Ok(())
+    }
+    pub fn handle_error(&mut self, error: &anyhow::Error) -> anyhow::Result<()> {
+        self.handle_row(&json!({
+            "error": error.to_string()
+        }))
+    }
+
+    pub fn close(mut self) -> W {
+        let _ = self.writer.write_all(self.suffix);
+        self.writer
+    }
+}
+
 #[allow(clippy::module_name_repetitions)]
 pub struct HtmlRenderContext<W: std::io::Write> {
     app_state: Arc<AppState>,
@@ -404,10 +503,6 @@ impl<W: std::io::Write> HtmlRenderContext<W> {
 
     #[async_recursion(? Send)]
     pub async fn handle_row(&mut self, data: &JsonValue) -> anyhow::Result<()> {
-        log::debug!(
-            "<- Rendering properties: {}",
-            serde_json::to_string(&data).unwrap_or_else(|e| e.to_string())
-        );
         let new_component = get_object_str(data, "component");
         let current_component = self
             .current_component
@@ -455,7 +550,6 @@ impl<W: std::io::Write> HtmlRenderContext<W> {
     /// Handles the rendering of an error.
     /// Returns whether the error is irrecoverable and the rendering must stop
     pub async fn handle_error(&mut self, error: &anyhow::Error) -> anyhow::Result<()> {
-        log::error!("SQL error: {:?}", error);
         self.close_component()?;
         let data = if self.app_state.config.environment.is_prod() {
             json!({
diff --git a/tests/index.rs b/tests/index.rs
@@ -3,7 +3,11 @@ use std::{collections::HashMap, path::PathBuf};
 use actix_web::{
     body::MessageBody,
     dev::{fn_service, ServerHandle, ServiceRequest, ServiceResponse},
-    http::{self, header::ContentType, StatusCode},
+    http::{
+        self,
+        header::{self, ContentType},
+        StatusCode,
+    },
     test::{self, TestRequest},
     HttpResponse,
 };
@@ -231,6 +235,27 @@ async fn test_overwrite_variable() -> actix_web::Result<()> {
     Ok(())
 }
 
+#[actix_web::test]
+async fn test_json_body() -> actix_web::Result<()> {
+    let req = get_request_to("/tests/json_data.sql")
+        .await?
+        .to_srv_request();
+    let resp = main_handler(req).await?;
+
+    assert_eq!(resp.status(), StatusCode::OK);
+    assert_eq!(
+        resp.headers().get(header::CONTENT_TYPE).unwrap(),
+        "application/json"
+    );
+    let body = test::read_body(resp).await;
+    let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap();
+    assert_eq!(
+        body_json,
+        serde_json::json!([{"message": "It works!"}, {"cool": "cool"}])
+    );
+    Ok(())
+}
+
 async fn test_file_upload(target: &str) -> actix_web::Result<()> {
     let req = get_request_to(target)
         .await?
diff --git a/tests/json_data.sql b/tests/json_data.sql
@@ -0,0 +1,3 @@
+select 'json' as component;
+select 'It works!' as message;
+select 'cool' as cool;

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+select 'json' as component;`
	`2`	`+select 'It works!' as message;`
	`3`	`+select 'cool' as cool;`