add an easy was to persist uploaded files

fixes #199

Author: lovasoa

CHANGELOG.md

@@ -3,6 +3,8 @@
 ## 0.20.1 (unreleased)
 
  - More than 200 new icons, with [tabler icons v3](https://tabler.io/icons/changelog#3.0)
+ - New [`sqlpage.persist_uploaded_file`](https://sql.ophir.dev/functions.sql?function=persist_uploaded_file#function) function to save uploaded files to a permanent location on the local filesystem (where SQLPage is running). This is useful to store files uploaded by users in a safe location, and to serve them back to users later.
+ - Correct error handling for file uploads. SQLPage used to silently ignore file uploads that failed (because they exceeded [max_uploaded_file_size](./configuration.md), for instance), but now it displays a clear error message to the user.
 
 ## 0.20.0 (2024-03-12)
 

Cargo.lock

@@ -0,0 +1 @@
+images/

examples/image gallery with user uploads/.gitignore

@@ -1,3 +1,3 @@
 {
-    "max_uploaded_file_size": 500000
+    "max_uploaded_file_size": 5000000
 }

examples/image gallery with user uploads/sqlpage/sqlpage.json

@@ -11,7 +11,10 @@ insert or ignore into image (title, description, image_url)
 values (
     :Title,
     :Description,
-    sqlpage.read_file_as_data_url(sqlpage.uploaded_file_path('Image'))
+    -- Persist the uploaded file to the local "images" folder at the root of the website and return the path
+    sqlpage.persist_uploaded_file('Image', 'images', 'jpg,jpeg,png,gif')
+    -- alternatively, if the images are small, you could store them in the database directly with the following line
+    -- sqlpage.read_file_as_data_url(sqlpage.uploaded_file_path('Image'))
 )
 returning 'redirect' as component,
           format('/?created_id=%d', id) as link;

examples/image gallery with user uploads/upload.sql

@@ -48,27 +48,12 @@ insert into text_documents (title, path) values (:title, sqlpage.read_file_as_te
 When the uploaded file is larger than a few megabytes, it is not recommended to store it in the database.
 Instead, one can save the file to a permanent location on the server, and store the path to the file in the database.
 
-You can move the file to a permanent location using the [`sqlpage.exec`](?function=exec#function) function:
-
-```sql
-set file_name = sqlpage.random_string(10);
-set exec_result = sqlpage.exec(''mv'', sqlpage.uploaded_file_path(''myfile''), ''/my_upload_directory/'' || $file_name);
-insert into uploaded_files (title, path) values (:title, $file_name);
-```
-
-> *Notes*: 
->  - The `sqlpage.exec` function is disabled by default, and you need to enable it in the [configuration file](https://github.com/lovasoa/SQLpage/blob/main/configuration.md).
->  - `mv` is specific to MacOS and Linux. On Windows, you can use [`move`](https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/move) instead:
->    - ```sql
->      SET image_path = sqlpage.uploaded_file_path(''myfile'');
->      SET exec_result = sqlpage.exec(''cmd'', ''/C'', ''move'', $image_path, ''C:\MyUploadDirectory'');
->      ```
-
+You can move the file to a permanent location using the [`sqlpage.persist_uploaded_file`](?function=persist_uploaded_file#function) function.
 ### Advanced file handling
 
 For more advanced file handling, such as uploading files to a cloud storage service,
 you can write a small script in your favorite programming language,
-and call it using the `sqlpage.exec` function.
+and call it using the [`sqlpage.exec`](?function=exec#function) function.
 
 For instance, one could save the following small bash script to `/usr/local/bin/upload_to_s3`:
 

examples/official-site/sqlpage/migrations/23_uploaded_file_functions.sql

@@ -0,0 +1,64 @@
+INSERT INTO sqlpage_functions (
+        "name",
+        "introduced_in_version",
+        "icon",
+        "description_md"
+    )
+VALUES (
+        'persist_uploaded_file',
+        '0.20.1',
+        'device-floppy',
+        'Persists an uploaded file to the local filesystem, and returns its path.
+
+### Example
+
+#### User profile picture
+
+##### `upload_form.sql`
+
+```sql
+select ''form'' as component, ''persist_uploaded_file.sql'' as action;
+select ''file'' as type, ''profile_picture'' as name, ''Upload your profile picture'' as label;
+```
+
+##### `persist_uploaded_file.sql`
+
+```sql
+update user
+set profile_picture = sqlpage.persist_uploaded_file(''profile_picture'', ''profile_pictures'', ''jpg,jpeg,png,gif,webp'')
+where id = (
+    select user_id from session where session_id = sqlpage.cookie(''session_id'')
+);
+```
+
+'
+    );
+INSERT INTO sqlpage_function_parameters (
+        "function",
+        "index",
+        "name",
+        "description_md",
+        "type"
+    )
+VALUES (
+        'persist_uploaded_file',
+        1,
+        'file',
+        'Name of the form field containing the uploaded file. The current page must be referenced in the `action` property of a `form` component that contains a file input field.',
+        'TEXT'
+    ),
+    (
+        'persist_uploaded_file',
+        2,
+        'destination_folder',
+        'Optional. Path to the folder where the file will be saved, relative to the web root (the root folder of your website files). By default, the file will be saved in the `uploads` folder.',
+        'TEXT'
+    ),
+    (
+        'persist_uploaded_file',
+        3,
+        'allowed_extensions',
+        'Optional. Comma-separated list of allowed file extensions. By default: jpg,jpeg,png,gif,bmp,webp,pdf,txt,doc,docx,xls,xlsx,csv,mp3,mp4,wav,avi,mov.
+Changing this may be dangerous ! If you add "sql", "svg" or "html" to the list, an attacker could execute arbitrary SQL queries on your database, or impersonate other users.',
+        'TEXT'
+    );

examples/official-site/sqlpage/migrations/39_persist_uploaded_file.sql

@@ -42,6 +42,11 @@ pub(super) enum StmtParam {
     Literal(String),
     UploadedFilePath(String),
     UploadedFileMimeType(String),
+    PersistUploadedFile {
+        field_name: Box<StmtParam>,
+        folder: Option<Box<StmtParam>>,
+        allowed_extensions: Option<Box<StmtParam>>,
+    },
     ReadFileAsText(Box<StmtParam>),
     ReadFileAsDataUrl(Box<StmtParam>),
     RunSql(Box<StmtParam>),
@@ -107,6 +112,25 @@ pub(super) fn func_call_to_param(func_name: &str, arguments: &mut [FunctionArg])
             extract_single_quoted_string("uploaded_file_mime_type", arguments)
                 .map_or_else(StmtParam::Error, StmtParam::UploadedFileMimeType)
         }
+        "persist_uploaded_file" => {
+            let field_name = Box::new(extract_variable_argument(
+                "persist_uploaded_file",
+                arguments,
+            ));
+            let folder = arguments
+                .get_mut(1)
+                .and_then(function_arg_to_stmt_param)
+                .map(Box::new);
+            let allowed_extensions = arguments
+                .get_mut(2)
+                .and_then(function_arg_to_stmt_param)
+                .map(Box::new);
+            StmtParam::PersistUploadedFile {
+                field_name,
+                folder,
+                allowed_extensions,
+            }
+        }
         "read_file_as_text" => StmtParam::ReadFileAsText(Box::new(extract_variable_argument(
             "read_file_as_text",
             arguments,
@@ -135,10 +159,88 @@ pub(super) async fn extract_req_param<'a>(
         StmtParam::ReadFileAsText(inner) => read_file_as_text(inner, request).await?,
         StmtParam::ReadFileAsDataUrl(inner) => read_file_as_data_url(inner, request).await?,
         StmtParam::RunSql(inner) => run_sql(inner, request).await?,
+        StmtParam::PersistUploadedFile {
+            field_name,
+            folder,
+            allowed_extensions,
+        } => {
+            persist_uploaded_file(
+                field_name,
+                folder.as_deref(),
+                allowed_extensions.as_deref(),
+                request,
+            )
+            .await?
+        }
         _ => extract_req_param_non_nested(param, request)?,
     })
 }
 
+const DEFAULT_ALLOWED_EXTENSIONS: &str =
+    "jpg,jpeg,png,gif,bmp,webp,pdf,txt,doc,docx,xls,xlsx,csv,mp3,mp4,wav,avi,mov";
+
+async fn persist_uploaded_file<'a>(
+    field_name: &StmtParam,
+    folder: Option<&StmtParam>,
+    allowed_extensions: Option<&StmtParam>,
+    request: &'a RequestInfo,
+) -> anyhow::Result<Option<Cow<'a, str>>> {
+    let field_name = extract_req_param_non_nested(field_name, request)?
+        .ok_or_else(|| anyhow!("persist_uploaded_file: field_name is NULL"))?;
+    let folder = folder
+        .map_or_else(|| Ok(None), |x| extract_req_param_non_nested(x, request))?
+        .unwrap_or(Cow::Borrowed("uploads"));
+    let allowed_extensions_str = &allowed_extensions
+        .map_or_else(|| Ok(None), |x| extract_req_param_non_nested(x, request))?
+        .unwrap_or(Cow::Borrowed(DEFAULT_ALLOWED_EXTENSIONS));
+    let allowed_extensions = allowed_extensions_str.split(',');
+    let uploaded_file = request
+        .uploaded_files
+        .get(&field_name.to_string())
+        .ok_or_else(|| {
+            anyhow!("persist_uploaded_file: no file uploaded with field name {field_name}. Uploaded files: {:?}", request.uploaded_files.keys())
+        })?;
+    let file_name = &uploaded_file.file_name.as_deref().unwrap_or_default();
+    let extension = file_name.split('.').last().unwrap_or_default();
+    if !allowed_extensions
+        .clone()
+        .any(|x| x.eq_ignore_ascii_case(extension))
+    {
+        let exts = allowed_extensions.collect::<Vec<_>>().join(", ");
+        bail!(
+            "persist_uploaded_file: file extension {extension} is not allowed. Allowed extensions: {exts}"
+        );
+    }
+    // resolve the folder path relative to the web root
+    let web_root = &request.app_state.config.web_root;
+    let target_folder = web_root.join(&*folder);
+    // create the folder if it doesn't exist
+    tokio::fs::create_dir_all(&target_folder)
+        .await
+        .with_context(|| {
+            format!("persist_uploaded_file: unable to create folder {target_folder:?}")
+        })?;
+    let date = chrono::Utc::now().format("%Y-%m-%d %Hh%Mm%Ss");
+    let random_part = random_string(8);
+    let random_target_name = format!("{date} {random_part}.{extension}");
+    let target_path = target_folder.join(&random_target_name);
+    tokio::fs::copy(&uploaded_file.file.path(), &target_path)
+        .await
+        .with_context(|| {
+            format!(
+                "persist_uploaded_file: unable to copy uploaded file {field_name:?} to {target_path:?}"
+            )
+        })?;
+    // remove the WEB_ROOT prefix from the path, but keep the leading slash
+    let path = "/".to_string() + target_path
+        .strip_prefix(web_root)?
+        .to_str()
+        .with_context(|| {
+            format!("persist_uploaded_file: unable to convert path {target_path:?} to a string")
+        })?;
+    Ok(Some(Cow::Owned(path)))
+}
+
 fn url_encode<'a>(
     inner: &StmtParam,
     request: &'a RequestInfo,
@@ -357,6 +459,9 @@ pub(super) fn extract_req_param_non_nested<'a>(
             .get(x)
             .and_then(|x| x.file.path().to_str())
             .map(Cow::Borrowed),
+        StmtParam::PersistUploadedFile { .. } => {
+            bail!("Nested persist_uploaded_file() function not allowed")
+        }
         StmtParam::UploadedFileMimeType(x) => request
             .uploaded_files
             .get(x)

sqlpage/sqlpage.db

@@ -219,7 +219,9 @@ async fn render_sql(
         .clone() // Cheap reference count increase
         .into_inner();
 
-    let mut req_param = extract_request_info(srv_req, Arc::clone(&app_state)).await;
+    let mut req_param = extract_request_info(srv_req, Arc::clone(&app_state))
+        .await
+        .map_err(anyhow_err_to_actix)?;
     log::debug!("Received a request with the following parameters: {req_param:?}");
 
     let (resp_send, resp_recv) = tokio::sync::oneshot::channel::<HttpResponse>();