new function: sqlpage.fetch

fixes #7

see
 - #197
 - #215
 - #254

Author: lovasoa

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## 0.20.3 (unreleased)
 
+ - New `dropdown` row-level property in the [`form` component](https://sql.ophir.dev/documentation.sql?component=form#component)
+   - ![select dropdown in form](https://github.com/lovasoa/SQLpage/assets/552629/5a2268d3-4996-49c9-9fb5-d310e753f844)
+   - ![multiselect input](https://github.com/lovasoa/SQLpage/assets/552629/e8d62d1a-c851-4fef-8c5c-a22991ffadcf)
+ - Adds a new [`sqlpage.fetch`](https://sql.ophir.dev/functions.sql?function=fetch#function) function that allows sending http requests from SQLPage. This is useful to query external APIs. This avoids having to resort to `sqlpage.exec`.
  - Fixed a bug that occured when using both HTTP and HTTPS in the same SQLPage instance. SQLPage tried to bind to the same (HTTP)
    port twice instead of binding to the HTTPS port. This is now fixed, and SQLPage can now be used with both a non-443 `port` and
    an `https_domain` set in the configuration file.
@@ -11,9 +15,6 @@
  - Optimize queries like `select 'xxx' as component, sqlpage.some_function(...) as parameter`
    to avoid making an unneeded database query. 
    This is especially important for the performance of `sqlpage.run_sql` and the `dynamic` component.
- - New `dropdown` row-level property in the [`form` component](https://sql.ophir.dev/documentation.sql?component=form#component)
-   - ![select dropdown in form](https://github.com/lovasoa/SQLpage/assets/552629/5a2268d3-4996-49c9-9fb5-d310e753f844)
-   - ![multiselect input](https://github.com/lovasoa/SQLpage/assets/552629/e8d62d1a-c851-4fef-8c5c-a22991ffadcf)
 
 ## 0.20.2 (2024-04-01)
 

Cargo.toml

@@ -31,7 +31,7 @@ tokio = { version = "1.24.1", features = ["macros", "rt", "process", "sync"] }
 tokio-stream = "0.1.9"
 anyhow = "1"
 serde = "1"
-serde_json = { version = "1.0.82", features = ["preserve_order"] }
+serde_json = { version = "1.0.82", features = ["preserve_order", "raw_value"] }
 lambda-web = { version = "0.2.1", features = ["actix4"], optional = true }
 sqlparser = { version = "0.45.0", features = ["visitor"] }
 async-stream = "0.3"
@@ -49,6 +49,7 @@ base64 = "0.22"
 rustls-acme = "0.7.7"
 dotenvy = "0.15.7"
 csv-async = { version = "1.2.6", features = ["tokio"] }
+awc = { version = "3", features = ["rustls"] }
 
 [build-dependencies]
 awc = { version = "3", features = ["rustls"] }

examples/official-site/sqlpage/migrations/40_fetch.sql

@@ -0,0 +1,74 @@
+INSERT INTO sqlpage_functions (
+        "name",
+        "introduced_in_version",
+        "icon",
+        "description_md"
+    )
+VALUES (
+        'fetch',
+        '0.20.3',
+        'transfer-vertical',
+        'Sends an HTTP request and returns the results as a string.
+
+### Example
+
+#### Simple GET query
+
+In this example, we use an API call to find the latitude and longitude of a place
+the user searched for, and we display it on a map.
+
+We use the simplest form of the fetch function, that takes the URL to fetch as a string.
+
+
+```sql
+set url = ''https://nominatim.openstreetmap.org/search?format=json&q='' || sqlpage.url_encode($user_search)
+set api_results = sqlpage.fetch($url);
+
+select ''map'' as component;
+select $user_search as title,
+  CAST($api_results->>0->>''lat'' AS FLOAT) as latitude,
+  CAST($api_results->>0->>''lon'' AS FLOAT) as longitude;
+```
+#### POST query with a body
+
+In this example, we use the complex form of the function to make an
+authenticated POST request, with custom request headers and a custom request body.
+
+We use SQLite''s json functions to build the request body. 
+
+```sql
+set request = json_object(
+    ''method'', ''POST''
+    ''url'', ''https://postman-echo.com/post'',
+    ''headers'', json_object(
+        ''Content-Type'', ''application/json'',
+        ''Authorization'', ''Bearer '' || sqlpage.environment_variable(''MY_API_TOKEN'')
+    ),
+    ''body'', json_object(
+        ''Hello'', ''world'',
+    ),
+);
+set api_results = sqlpage.fetch($request);
+
+select ''code'' as component;
+select
+    ''API call results'' as title,
+    ''json'' as language,
+    $api_results as contents;
+```
+'
+    );
+INSERT INTO sqlpage_function_parameters (
+        "function",
+        "index",
+        "name",
+        "description_md",
+        "type"
+    )
+VALUES (
+        'fetch',
+        1,
+        'url',
+        'Either a string containing an URL to request, or a json object in the standard format of the request interface of the web fetch API.',
+        'TEXT'
+    );

src/webserver/database/sql_pseudofunctions.rs

@@ -1,7 +1,8 @@
-use std::{borrow::Cow, collections::HashMap};
+use std::{borrow::Cow, collections::HashMap, str::FromStr};
 
 use actix_web::http::StatusCode;
 use actix_web_httpauth::headers::authorization::Basic;
+use awc::http::Method;
 use base64::Engine;
 use mime_guess::{mime::APPLICATION_OCTET_STREAM, Mime};
 use sqlparser::ast::FunctionArg;
@@ -51,6 +52,7 @@ pub(super) enum StmtParam {
     ReadFileAsText(Box<StmtParam>),
     ReadFileAsDataUrl(Box<StmtParam>),
     RunSql(Box<StmtParam>),
+    Fetch(Box<StmtParam>),
     Path,
     Protocol,
 }
@@ -140,6 +142,7 @@ pub(super) fn func_call_to_param(func_name: &str, arguments: &mut [FunctionArg])
             extract_variable_argument("read_file_as_data_url", arguments),
         )),
         "run_sql" => StmtParam::RunSql(Box::new(extract_variable_argument("run_sql", arguments))),
+        "fetch" => StmtParam::Fetch(Box::new(extract_variable_argument("fetch", arguments))),
         unknown_name => StmtParam::Error(format!(
             "Unknown function {unknown_name}({})",
             FormatArguments(arguments)
@@ -389,6 +392,90 @@ async fn run_sql<'a>(
     Ok(Some(Cow::Owned(String::from_utf8(json_results_bytes)?)))
 }
 
+type HeaderVec<'a> = Vec<(Cow<'a, str>, Cow<'a, str>)>;
+#[derive(serde::Deserialize)]
+struct Req<'b> {
+    #[serde(borrow)]
+    url: Cow<'b, str>,
+    #[serde(borrow)]
+    method: Option<Cow<'b, str>>,
+    #[serde(borrow, deserialize_with = "deserialize_map_to_vec_pairs")]
+    headers: HeaderVec<'b>,
+    #[serde(borrow)]
+    body: Option<&'b serde_json::value::RawValue>,
+}
+
+fn deserialize_map_to_vec_pairs<'de, D: serde::Deserializer<'de>>(
+    deserializer: D,
+) -> Result<HeaderVec<'de>, D::Error> {
+    struct Visitor;
+
+    impl<'de> serde::de::Visitor<'de> for Visitor {
+        type Value = Vec<(Cow<'de, str>, Cow<'de, str>)>;
+
+        fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+            formatter.write_str("a map")
+        }
+
+        fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+        where
+            A: serde::de::MapAccess<'de>,
+        {
+            let mut vec = Vec::new();
+            while let Some((key, value)) = map.next_entry()? {
+                vec.push((key, value));
+            }
+            Ok(vec)
+        }
+    }
+
+    deserializer.deserialize_map(Visitor)
+}
+
+async fn fetch<'a>(
+    param0: &StmtParam,
+    request: &'a RequestInfo,
+) -> Result<Option<Cow<'a, str>>, anyhow::Error> {
+    let Some(fetch_target) = Box::pin(extract_req_param(param0, request)).await? else {
+        log::debug!("fetch: first argument is NULL, returning NULL");
+        return Ok(None);
+    };
+    let client = awc::Client::default();
+    let res = if fetch_target.starts_with("http") {
+        client.get(fetch_target.as_ref()).send()
+    } else {
+        let r = serde_json::from_str::<'_, Req<'_>>(&fetch_target)
+            .with_context(|| format!("Invalid request: {fetch_target}"))?;
+        let method = if let Some(method) = r.method {
+            Method::from_str(&method)?
+        } else {
+            Method::GET
+        };
+        let mut req = client.request(method, r.url.as_ref());
+        for (k, v) in r.headers {
+            req = req.insert_header((k.as_ref(), v.as_ref()));
+        }
+        if let Some(body) = r.body {
+            let val = body.get();
+            // The body can be either json, or a string representing a raw body
+            let body = if val.starts_with('"') {
+                serde_json::from_str::<'_, String>(val)?
+            } else {
+                req = req.content_type("application/json");
+                val.to_owned()
+            };
+            req.send_body(body)
+        } else {
+            req.send()
+        }
+    };
+    let mut res = res
+        .await
+        .map_err(|e| anyhow!("Unable to fetch {fetch_target}: {e}"))?;
+    let body = res.body().await?.to_vec();
+    Ok(Some(String::from_utf8(body)?.into()))
+}
+
 fn mime_from_upload<'a>(param0: &StmtParam, request: &'a RequestInfo) -> Option<&'a Mime> {
     if let StmtParam::UploadedFilePath(name) | StmtParam::UploadedFileMimeType(name) = param0 {
         request.uploaded_files.get(name)?.content_type.as_ref()
@@ -429,6 +516,7 @@ pub(super) async fn extract_req_param<'a>(
         StmtParam::ReadFileAsText(inner) => read_file_as_text(inner, request).await?,
         StmtParam::ReadFileAsDataUrl(inner) => read_file_as_data_url(inner, request).await?,
         StmtParam::RunSql(inner) => run_sql(inner, request).await?,
+        StmtParam::Fetch(inner) => fetch(inner, request).await?,
         StmtParam::PersistUploadedFile {
             field_name,
             folder,

tests/index.rs

@@ -1,7 +1,9 @@
 use actix_web::{
     body::MessageBody,
+    dev::{fn_service, ServerHandle, ServiceRequest, ServiceResponse},
     http::{self, header::ContentType, StatusCode},
     test::{self, TestRequest},
+    HttpResponse,
 };
 use sqlpage::{app_config::AppConfig, webserver::http::main_handler, AppState};
 
@@ -81,8 +83,40 @@ async fn test_concurrent_requests() {
     }
 }
 
+fn start_echo_server() -> ServerHandle {
+    async fn echo_server(mut r: ServiceRequest) -> actix_web::Result<ServiceResponse> {
+        let method = r.method();
+        let path = r.uri();
+        let mut headers_vec = r
+            .headers()
+            .into_iter()
+            .map(|(k, v)| format!("{k}: {}", String::from_utf8_lossy(v.as_bytes())))
+            .collect::<Vec<_>>();
+        headers_vec.sort();
+        let headers = headers_vec.join("\n");
+        let mut resp_bytes = format!("{method} {path}\n{headers}\n\n").into_bytes();
+        resp_bytes.extend(r.extract::<actix_web::web::Bytes>().await?);
+        let resp = HttpResponse::Ok().body(resp_bytes);
+        Ok(r.into_response(resp))
+    }
+    let server = actix_web::HttpServer::new(move || {
+        actix_web::App::new().default_service(fn_service(echo_server))
+    })
+    .bind("localhost:62802")
+    .unwrap()
+    .shutdown_timeout(5) // shutdown timeout
+    .run();
+
+    let handle = server.handle();
+    tokio::spawn(server);
+
+    handle
+}
+
 #[actix_web::test]
 async fn test_files() {
+    // start a dummy server that test files can query
+    let echo_server = start_echo_server();
     // Iterate over all the sql test files in the tests/ directory
     let path = std::path::Path::new("tests/sql_test_files");
     let app_data = make_app_data().await;
@@ -128,6 +162,7 @@ async fn test_files() {
             );
         }
     }
+    echo_server.stop(true).await
 }
 
 #[actix_web::test]

tests/sql_test_files/it_works_fetch_post.sql

@@ -0,0 +1,22 @@
+set res = sqlpage.fetch('{
+    "method": "POST",
+    "url": "http://localhost:62802/post",
+    "headers": {"x-custom": "1"},
+    "body": {"hello": "world"}
+}');
+set expected_like = 'POST /post
+accept-encoding: br, gzip, deflate, zstd
+content-length: 18
+content-type: application/json
+date: %
+host: localhost:62802
+x-custom: 1
+
+{"hello": "world"}';
+select 'text' as component,
+    case
+        when $res LIKE $expected_like then 'It works !'
+        else 'It failed ! Expected: 
+' || $expected_like || 'Got: 
+' || $res
+    end as contents;

tests/sql_test_files/it_works_fetch_simple.sql

@@ -0,0 +1,6 @@
+set res = sqlpage.fetch('http://localhost:62802/hello_world')
+select 'text' as component,
+    case
+        when $res LIKE 'GET /hello_world%' then 'It works !'
+        else 'It failed ! Got: ' || $res
+    end as contents;