new function: fetch_with_meta (#827)

* new function: fetch_with_meta

closes #792

* fmt

* clippy auto

* test fetch_with_meta

* fix tests

* update deps

* retry failed deps downloads

* better json serializing

* test fetch_with_meta error handling

* add logging

* fix tests



closes #792

Author: lovasoa

Cargo.lock

@@ -1,4 +1,5 @@
 use actix_rt::spawn;
+use actix_rt::time::sleep;
 use libflate::gzip;
 use std::collections::hash_map::DefaultHasher;
 use std::fs::File;
@@ -99,21 +100,37 @@ fn copy_cached_to_opened_file(source: &Path, outfile: &mut impl std::io::Write)
 }
 
 async fn download_url_to_path(client: &awc::Client, url: &str, path: &Path) {
-    let mut resp = client.get(url).send().await.unwrap_or_else(|err| {
-        let path = make_url_path(url);
-        panic!(
-            "We need to download external frontend dependencies to build the static frontend. \n\
-                Could not download static asset. You can manually download the file with: \n\
-                curl {url:?} > {path:?} \n\
-                {err}"
-        )
-    });
-    if resp.status() != 200 {
-        panic!("Received {} status code from {}", resp.status(), url);
+    let mut attempt = 1;
+    let max_attempts = 2;
+
+    loop {
+        match client.get(url).send().await {
+            Ok(mut resp) => {
+                if resp.status() != 200 {
+                    panic!("Received {} status code from {}", resp.status(), url);
+                }
+                let bytes = resp.body().limit(128 * 1024 * 1024).await.unwrap();
+                std::fs::write(path, &bytes)
+                    .expect("Failed to write external frontend dependency to local file");
+                break;
+            }
+            Err(err) => {
+                if attempt >= max_attempts {
+                    let path = make_url_path(url);
+                    panic!(
+                        "We need to download external frontend dependencies to build the static frontend. \n\
+                        Could not download static asset after {} attempts. You can manually download the file with: \n\
+                        curl {url:?} > {path:?} \n\
+                        {err}",
+                        max_attempts
+                    );
+                }
+                sleep(Duration::from_secs(1)).await;
+                println!("cargo:warning=Retrying download of {url} after {err}.");
+                attempt += 1;
+            }
+        }
     }
-    let bytes = resp.body().limit(128 * 1024 * 1024).await.unwrap();
-    std::fs::write(path, &bytes)
-        .expect("Failed to write external frontend dependency to local file");
 }
 
 // Given a filename, creates a new unique filename based on the file contents

build.rs

@@ -88,6 +88,12 @@ The fetch function accepts either a URL string, or a JSON object with the follow
  - `username`: Optional username for HTTP Basic Authentication. Introduced in version 0.33.0.
  - `password`: Optional password for HTTP Basic Authentication. Only used if username is provided. Introduced in version 0.33.0.
 
+# Error handling and reading response headers
+
+If the request fails, this function throws an error, that will be displayed to the user.
+The response headers are not available for inspection.
+
+If you need to handle errors or inspect the response headers, use [`sqlpage.fetch_with_meta`](?function=fetch_with_meta).
 '
     );
 INSERT INTO sqlpage_function_parameters (

examples/official-site/sqlpage/migrations/40_fetch.sql

@@ -0,0 +1,86 @@
+INSERT INTO sqlpage_functions (
+        "name",
+        "introduced_in_version",
+        "icon",
+        "description_md"
+    )
+VALUES (
+        'fetch_with_meta',
+        '0.34.0',
+        'transfer-vertical',
+        'Sends an HTTP request and returns detailed metadata about the response, including status code, headers, and body.
+
+This function is similar to [`fetch`](?function=fetch), but returns a JSON object containing detailed information about the response.
+The returned object has the following structure:
+```json
+{
+    "status": 200,
+    "headers": {
+        "content-type": "text/html",
+        "content-length": "1234"
+    },
+    "body": "a string, or a json object, depending on the content type",
+    "error": "error message if any"
+}
+```
+
+If the request fails or encounters an error (e.g., network issues, invalid UTF-8 response), instead of throwing an error,
+the function returns a JSON object with an "error" field containing the error message.
+
+### Example: Basic Usage
+
+```sql
+-- Make a request and get detailed response information
+set response = sqlpage.fetch_with_meta(''https://pokeapi.co/api/v2/pokemon/ditto'');
+
+-- redirect the user to an error page if the request failed
+select ''redirect'' as component, ''error.sql'' as url
+where
+    json_extract($response, ''$.error'') is not null
+    or json_extract($response, ''$.status'') != 200;
+
+-- Extract data from the response json body
+select ''card'' as component;
+select
+    json_extract($response, ''$.body.name'') as title,
+    json_extract($response, ''$.body.abilities[0].ability.name'') as description
+from $response;
+```
+
+### Example: Advanced Request with Authentication
+
+```sql
+set request = json_object(
+    ''method'', ''POST'',
+    ''url'', ''https://sqlpage.free.beeceptor.com'',
+    ''headers'', json_object(
+        ''Content-Type'', ''application/json'',
+        ''Authorization'', ''Bearer '' || sqlpage.environment_variable(''API_TOKEN'')
+    ),
+    ''body'', json_object(
+        ''key'', ''value''
+    )
+);
+set response = sqlpage.fetch_with_meta($request);
+
+-- Check response content type
+select ''debug'' as component, $response as response;
+```
+
+The function accepts the same parameters as the [`fetch` function](?function=fetch).'
+    );
+
+INSERT INTO sqlpage_function_parameters (
+        "function",
+        "index",
+        "name",
+        "description_md",
+        "type"
+    )
+VALUES (
+        'fetch_with_meta',
+        1,
+        'url',
+        'Either a string containing an URL to request, or a json object in the standard format of the request interface of the web fetch API.',
+        'TEXT'
+    );