diff --git a/Cargo.lock b/Cargo.lock index 48efda03..e5053a03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,9 +830,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.30" +version = "4.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92b7b18d71fad5313a1e320fa9897994228ce274b60faa4d694fe0ea89cd9e6d" +checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" dependencies = [ "clap_builder", "clap_derive", @@ -840,9 +840,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.30" +version = "4.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35db2071778a7344791a4fb4f95308b5673d219dee3ae348b86642574ecc90c" +checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" dependencies = [ "anstream", "anstyle", @@ -1254,9 +1254,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "either" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" [[package]] name = "encoding_rs" @@ -1346,9 +1346,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", "miniz_oxide", @@ -2049,9 +2049,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" [[package]] name = "libflate" @@ -2616,7 +2616,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.1", + "rand_core 0.9.2", "zerocopy 0.8.20", ] @@ -2637,7 +2637,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.1", + "rand_core 0.9.2", ] [[package]] @@ -2651,9 +2651,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" +checksum = "7a509b1a2ffbe92afab0e55c8fd99dea1c280e8171bd2d88682bb20bc41cbc2c" dependencies = [ "getrandom 0.3.1", "zerocopy 0.8.20", @@ -2728,9 +2728,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "ring" -version = "0.17.10" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34b5020fcdea098ef7d95e9f89ec15952123a4a039badd09fabebe9e963e839" +checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73" dependencies = [ "cc", "cfg-if", diff --git a/build.rs b/build.rs index 3c2bdeea..8c3ae77f 100644 --- a/build.rs +++ b/build.rs @@ -1,4 +1,5 @@ use actix_rt::spawn; +use actix_rt::time::sleep; use libflate::gzip; use std::collections::hash_map::DefaultHasher; use std::fs::File; @@ -99,21 +100,37 @@ fn copy_cached_to_opened_file(source: &Path, outfile: &mut impl std::io::Write) } async fn download_url_to_path(client: &awc::Client, url: &str, path: &Path) { - let mut resp = client.get(url).send().await.unwrap_or_else(|err| { - let path = make_url_path(url); - panic!( - "We need to download external frontend dependencies to build the static frontend. \n\ - Could not download static asset. You can manually download the file with: \n\ - curl {url:?} > {path:?} \n\ - {err}" - ) - }); - if resp.status() != 200 { - panic!("Received {} status code from {}", resp.status(), url); + let mut attempt = 1; + let max_attempts = 2; + + loop { + match client.get(url).send().await { + Ok(mut resp) => { + if resp.status() != 200 { + panic!("Received {} status code from {}", resp.status(), url); + } + let bytes = resp.body().limit(128 * 1024 * 1024).await.unwrap(); + std::fs::write(path, &bytes) + .expect("Failed to write external frontend dependency to local file"); + break; + } + Err(err) => { + if attempt >= max_attempts { + let path = make_url_path(url); + panic!( + "We need to download external frontend dependencies to build the static frontend. \n\ + Could not download static asset after {} attempts. You can manually download the file with: \n\ + curl {url:?} > {path:?} \n\ + {err}", + max_attempts + ); + } + sleep(Duration::from_secs(1)).await; + println!("cargo:warning=Retrying download of {url} after {err}."); + attempt += 1; + } + } } - let bytes = resp.body().limit(128 * 1024 * 1024).await.unwrap(); - std::fs::write(path, &bytes) - .expect("Failed to write external frontend dependency to local file"); } // Given a filename, creates a new unique filename based on the file contents diff --git a/examples/official-site/sqlpage/migrations/40_fetch.sql b/examples/official-site/sqlpage/migrations/40_fetch.sql index cad1af2e..002bc0c9 100644 --- a/examples/official-site/sqlpage/migrations/40_fetch.sql +++ b/examples/official-site/sqlpage/migrations/40_fetch.sql @@ -88,6 +88,12 @@ The fetch function accepts either a URL string, or a JSON object with the follow - `username`: Optional username for HTTP Basic Authentication. Introduced in version 0.33.0. - `password`: Optional password for HTTP Basic Authentication. Only used if username is provided. Introduced in version 0.33.0. +# Error handling and reading response headers + +If the request fails, this function throws an error, that will be displayed to the user. +The response headers are not available for inspection. + +If you need to handle errors or inspect the response headers, use [`sqlpage.fetch_with_meta`](?function=fetch_with_meta). ' ); INSERT INTO sqlpage_function_parameters ( diff --git a/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql b/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql new file mode 100644 index 00000000..296071d2 --- /dev/null +++ b/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql @@ -0,0 +1,86 @@ +INSERT INTO sqlpage_functions ( + "name", + "introduced_in_version", + "icon", + "description_md" + ) +VALUES ( + 'fetch_with_meta', + '0.34.0', + 'transfer-vertical', + 'Sends an HTTP request and returns detailed metadata about the response, including status code, headers, and body. + +This function is similar to [`fetch`](?function=fetch), but returns a JSON object containing detailed information about the response. +The returned object has the following structure: +```json +{ + "status": 200, + "headers": { + "content-type": "text/html", + "content-length": "1234" + }, + "body": "a string, or a json object, depending on the content type", + "error": "error message if any" +} +``` + +If the request fails or encounters an error (e.g., network issues, invalid UTF-8 response), instead of throwing an error, +the function returns a JSON object with an "error" field containing the error message. + +### Example: Basic Usage + +```sql +-- Make a request and get detailed response information +set response = sqlpage.fetch_with_meta(''https://pokeapi.co/api/v2/pokemon/ditto''); + +-- redirect the user to an error page if the request failed +select ''redirect'' as component, ''error.sql'' as url +where + json_extract($response, ''$.error'') is not null + or json_extract($response, ''$.status'') != 200; + +-- Extract data from the response json body +select ''card'' as component; +select + json_extract($response, ''$.body.name'') as title, + json_extract($response, ''$.body.abilities[0].ability.name'') as description +from $response; +``` + +### Example: Advanced Request with Authentication + +```sql +set request = json_object( + ''method'', ''POST'', + ''url'', ''https://sqlpage.free.beeceptor.com'', + ''headers'', json_object( + ''Content-Type'', ''application/json'', + ''Authorization'', ''Bearer '' || sqlpage.environment_variable(''API_TOKEN'') + ), + ''body'', json_object( + ''key'', ''value'' + ) +); +set response = sqlpage.fetch_with_meta($request); + +-- Check response content type +select ''debug'' as component, $response as response; +``` + +The function accepts the same parameters as the [`fetch` function](?function=fetch).' + ); + +INSERT INTO sqlpage_function_parameters ( + "function", + "index", + "name", + "description_md", + "type" + ) +VALUES ( + 'fetch_with_meta', + 1, + 'url', + 'Either a string containing an URL to request, or a json object in the standard format of the request interface of the web fetch API.', + 'TEXT' + ); \ No newline at end of file diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index 0f32f4cb..1b52f706 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -24,6 +24,7 @@ super::function_definition_macro::sqlpage_functions! { exec((&RequestInfo), program_name: Cow, args: Vec>); fetch((&RequestInfo), http_request: SqlPageFunctionParam>); + fetch_with_meta((&RequestInfo), http_request: SqlPageFunctionParam>); hash_password(password: Option); header((&RequestInfo), name: Cow); @@ -135,16 +136,13 @@ async fn exec<'a>( Ok(String::from_utf8_lossy(&res.stdout).into_owned()) } -async fn fetch( - request: &RequestInfo, - http_request: super::http_fetch_request::HttpFetchRequest<'_>, -) -> anyhow::Result { +fn build_request<'a>( + client: &'a awc::Client, + http_request: &'a super::http_fetch_request::HttpFetchRequest<'_>, +) -> anyhow::Result { use awc::http::Method; - let client = make_http_client(&request.app_state.config) - .with_context(|| "Unable to create an HTTP client")?; - - let method = if let Some(method) = http_request.method { - Method::from_str(&method).with_context(|| format!("Invalid HTTP method: {method}"))? + let method = if let Some(method) = &http_request.method { + Method::from_str(method).with_context(|| format!("Invalid HTTP method: {method}"))? } else { Method::GET }; @@ -152,36 +150,56 @@ async fn fetch( if let Some(timeout) = http_request.timeout_ms { req = req.timeout(core::time::Duration::from_millis(timeout)); } - for (k, v) in http_request.headers { + for (k, v) in &http_request.headers { req = req.insert_header((k.as_ref(), v.as_ref())); } - if let Some(username) = http_request.username { - let password = http_request.password.unwrap_or_default(); + if let Some(username) = &http_request.username { + let password = http_request.password.as_deref().unwrap_or_default(); req = req.basic_auth(username, password); } + Ok(req) +} + +fn prepare_request_body( + body: &serde_json::value::RawValue, + mut req: awc::ClientRequest, +) -> anyhow::Result<(String, awc::ClientRequest)> { + let val = body.get(); + let body_str = if val.starts_with('"') { + serde_json::from_str::<'_, String>(val).with_context(|| { + format!("Invalid JSON string in the body of the HTTP request: {val}") + })? + } else { + req = req.content_type("application/json"); + val.to_owned() + }; + Ok((body_str, req)) +} + +async fn fetch( + request: &RequestInfo, + http_request: super::http_fetch_request::HttpFetchRequest<'_>, +) -> anyhow::Result { + let client = make_http_client(&request.app_state.config) + .with_context(|| "Unable to create an HTTP client")?; + let req = build_request(&client, &http_request)?; + log::info!("Fetching {}", http_request.url); - let mut response = if let Some(body) = http_request.body { - let val = body.get(); - // The body can be either json, or a string representing a raw body - let body = if val.starts_with('"') { - serde_json::from_str::<'_, String>(val).with_context(|| { - format!("Invalid JSON string in the body of the HTTP request: {val}") - })? - } else { - req = req.content_type("application/json"); - val.to_owned() - }; + let mut response = if let Some(body) = &http_request.body { + let (body, req) = prepare_request_body(body, req)?; req.send_body(body) } else { req.send() } .await .map_err(|e| anyhow!("Unable to fetch {}: {e}", http_request.url))?; + log::debug!( "Finished fetching {}. Status: {}", http_request.url, response.status() ); + let body = response .body() .await @@ -199,6 +217,90 @@ async fn fetch( Ok(response_str) } +async fn fetch_with_meta( + request: &RequestInfo, + http_request: super::http_fetch_request::HttpFetchRequest<'_>, +) -> anyhow::Result { + use serde::{ser::SerializeMap, Serializer}; + + let client = make_http_client(&request.app_state.config) + .with_context(|| "Unable to create an HTTP client")?; + let req = build_request(&client, &http_request)?; + + log::info!("Fetching {} with metadata", http_request.url); + let response_result = if let Some(body) = &http_request.body { + let (body, req) = prepare_request_body(body, req)?; + req.send_body(body).await + } else { + req.send().await + }; + + let mut resp_str = Vec::new(); + let mut encoder = serde_json::Serializer::new(&mut resp_str); + let mut obj = encoder.serialize_map(Some(3))?; + match response_result { + Ok(mut response) => { + obj.serialize_entry("status", &response.status().as_u16())?; + + let headers = response.headers(); + + let is_json = headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .starts_with("application/json"); + + obj.serialize_entry( + "headers", + &headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or_default())) + .collect::>(), + )?; + + match response.body().await { + Ok(body) => { + let body_bytes = body.to_vec(); + let body_str = String::from_utf8(body_bytes); + + match body_str { + Ok(body_str) if is_json => { + obj.serialize_entry( + "body", + &serde_json::value::RawValue::from_string(body_str)?, + )?; + } + Ok(body_str) => { + obj.serialize_entry("body", &body_str)?; + } + Err(utf8_err) => { + let mut base64_string = String::new(); + base64::Engine::encode_string( + &base64::engine::general_purpose::STANDARD, + utf8_err.as_bytes(), + &mut base64_string, + ); + obj.serialize_entry("body", &base64_string)?; + } + } + } + Err(e) => { + log::warn!("Failed to read response body: {e}"); + obj.serialize_entry("error", &format!("Failed to read response body: {e}"))?; + } + } + } + Err(e) => { + log::warn!("Request failed: {e}"); + obj.serialize_entry("error", &format!("Request failed: {e}"))?; + } + } + + obj.end()?; + let return_value = String::from_utf8(resp_str)?; + Ok(return_value) +} + static NATIVE_CERTS: OnceLock> = OnceLock::new(); fn make_http_client(config: &crate::app_config::AppConfig) -> anyhow::Result { diff --git a/tests/index.rs b/tests/index.rs index 590eb520..1f1dee8b 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -136,7 +136,10 @@ fn start_echo_server() -> ServerHandle { } f.push(b'|'); f.extend_from_slice(&r.extract::().await?); - let resp = HttpResponse::Ok().body(f); + let resp = HttpResponse::Ok() + .insert_header((header::DATE, "Mon, 24 Feb 2025 12:00:00 GMT")) + .insert_header((header::CONTENT_TYPE, "text/plain")) + .body(f); Ok(r.into_response(resp)) } let server = actix_web::HttpServer::new(move || { @@ -201,7 +204,7 @@ async fn test_files() { ); assert!( !lowercase_body.contains("error"), - "{body}\nexpected to not contain: error" + "{req_str}\n{body}\nexpected to not contain: error" ); } else if stem.starts_with("error_") { let rest = stem.strip_prefix("error_").unwrap(); diff --git a/tests/sql_test_files/it_works_fetch_with_meta_error.sql b/tests/sql_test_files/it_works_fetch_with_meta_error.sql new file mode 100644 index 00000000..4c6d2c88 --- /dev/null +++ b/tests/sql_test_files/it_works_fetch_with_meta_error.sql @@ -0,0 +1,7 @@ +set res = sqlpage.fetch_with_meta('http://not-a-real-url'); + +select 'text' as component, + case + when json_extract($res, '$.error') LIKE '%Request failed%' then 'It works !' + else CONCAT('Error! Got: ', $res) + end as contents; \ No newline at end of file diff --git a/tests/sql_test_files/it_works_fetch_with_meta_simple.sql b/tests/sql_test_files/it_works_fetch_with_meta_simple.sql new file mode 100644 index 00000000..d4d8bfaa --- /dev/null +++ b/tests/sql_test_files/it_works_fetch_with_meta_simple.sql @@ -0,0 +1,13 @@ +set res = sqlpage.fetch_with_meta('{ + "method": "PUT", + "url": "http://localhost:62802/hello_world", + "headers": { + "user-agent": "myself" + } +}'); + +select 'text' as component, + case + when $res LIKE '%"status":200%' AND $res LIKE '%"headers":{%' AND $res LIKE '%"body":"%' then 'It works !' + else 'Error! Got: ' || $res + end as contents; \ No newline at end of file