Merge pull request #6770 from tannewt/upload_folder

Add uploading a directory and its contents

Author: tannewt

supervisor/shared/bluetooth/file_transfer.c

@@ -47,11 +47,11 @@
 #include "supervisor/shared/reload.h"
 #include "supervisor/shared/bluetooth/file_transfer.h"
 #include "supervisor/shared/bluetooth/file_transfer_protocol.h"
+#include "supervisor/shared/workflow.h"
 #include "supervisor/shared/tick.h"
 #include "supervisor/usb.h"
 
 #include "py/mpstate.h"
-#include "py/stackctrl.h"
 
 STATIC bleio_service_obj_t supervisor_ble_service;
 STATIC bleio_uuid_obj_t supervisor_ble_service_uuid;
@@ -387,39 +387,6 @@ STATIC uint8_t _process_write_data(const uint8_t *raw_buf, size_t command_len) {
     return WRITE_DATA;
 }
 
-STATIC FRESULT _delete_directory_contents(FATFS *fs, const TCHAR *path) {
-    FF_DIR dir;
-    FRESULT res = f_opendir(fs, &dir, path);
-    FILINFO file_info;
-    // Check the stack since we're putting paths on it.
-    if (mp_stack_usage() >= MP_STATE_THREAD(stack_limit)) {
-        return FR_INT_ERR;
-    }
-    while (res == FR_OK) {
-        res = f_readdir(&dir, &file_info);
-        if (res != FR_OK || file_info.fname[0] == '\0') {
-            break;
-        }
-        size_t pathlen = strlen(path);
-        size_t fnlen = strlen(file_info.fname);
-        TCHAR full_path[pathlen + 1 + fnlen];
-        memcpy(full_path, path, pathlen);
-        full_path[pathlen] = '/';
-        size_t full_pathlen = pathlen + 1 + fnlen;
-        memcpy(full_path + pathlen + 1, file_info.fname, fnlen);
-        full_path[full_pathlen] = '\0';
-        if ((file_info.fattrib & AM_DIR) != 0) {
-            res = _delete_directory_contents(fs, full_path);
-        }
-        if (res != FR_OK) {
-            break;
-        }
-        res = f_unlink(fs, full_path);
-    }
-    f_closedir(&dir);
-    return res;
-}
-
 STATIC uint8_t _process_delete(const uint8_t *raw_buf, size_t command_len) {
     const struct delete_command *command = (struct delete_command *)raw_buf;
     size_t header_size = sizeof(struct delete_command);
@@ -446,7 +413,7 @@ STATIC uint8_t _process_delete(const uint8_t *raw_buf, size_t command_len) {
     FRESULT result = f_stat(fs, path, &file);
     if (result == FR_OK) {
         if ((file.fattrib & AM_DIR) != 0) {
-            result = _delete_directory_contents(fs, path);
+            result = supervisor_workflow_delete_directory_contents(fs, path);
         }
         if (result == FR_OK) {
             result = f_unlink(fs, path);
@@ -503,7 +470,7 @@ STATIC uint8_t _process_mkdir(const uint8_t *raw_buf, size_t command_len) {
     DWORD fattime;
     response.truncated_time = truncate_time(command->modification_time, &fattime);
     override_fattime(fattime);
-    FRESULT result = f_mkdir(fs, path);
+    FRESULT result = supervisor_workflow_mkdir_parents(fs, path);
     override_fattime(0);
     #if CIRCUITPY_USB_MSC
     usb_msc_unlock();

supervisor/shared/web_workflow/static/directory.html

@@ -16,7 +16,9 @@ <h1><a href="/"><img src="/favicon.ico"/></a>&nbsp;<span id="path"></span></h1>
         <tbody></tbody>
     </table>
     <hr>
-    <input type="file" id="files" multiple><button type="submit" id="upload">Upload</button>
+    <label>📄 <input type="file" id="files" multiple></label>
+    <label for="dirs">📁 <input type="file" id="dirs" multiple webkitdirectory></label>
+    <label>Upload progress:<progress value="0"></progress></label>
     <hr>
     +📁&nbsp;<input type="text" id="name"><button type="submit" id="mkdir">Create Directory</button>
 </body></html>

supervisor/shared/web_workflow/static/directory.js

@@ -1,5 +1,6 @@
 let new_directory_name = document.getElementById("name");
 let files = document.getElementById("files");
+let dirs = document.getElementById("dirs");
 
 var url_base = window.location;
 var current_path;
@@ -15,10 +16,12 @@ function compareValues(a, b) {
     }
 }
 
-async function refresh_list() {
-
-
+function set_upload_enabled(enabled) {
+    files.disabled = !enabled;
+    dirs.disabled = !enabled;
+}
 
+async function refresh_list() {
     current_path = window.location.hash.substr(1);
     if (current_path == "") {
         current_path = "/";
@@ -49,7 +52,7 @@ async function refresh_list() {
         );
         editable = status.headers.get("Access-Control-Allow-Methods").includes("DELETE");
         new_directory_name.disabled = !editable;
-        files.disabled = !editable;
+        set_upload_enabled(editable);
         if (!editable) {
             let usbwarning = document.querySelector("#usbwarning");
             usbwarning.style.display = "block";
@@ -169,8 +172,25 @@ async function mkdir(e) {
 }
 
 async function upload(e) {
-    for (const file of files.files) {
-        let file_path = new URL("/fs" + current_path + file.name, url_base);
+    set_upload_enabled(false);
+    let progress = document.querySelector("progress");
+    let made_dirs = new Set();
+    progress.max = files.files.length + dirs.files.length;
+    progress.value = 0;
+    for (const file of [...files.files, ...dirs.files]) {
+        let file_name = file.name;
+        if (file.webkitRelativePath) {
+            file_name = file.webkitRelativePath;
+            let components = file_name.split("/");
+            components.pop();
+            let parent_dir = components.join("/");
+            if (!made_dirs.has(parent_dir)) {
+                new_directory_name.value = parent_dir;
+                await mkdir(null);
+                made_dirs.add(parent_dir);
+            }
+        }
+        let file_path = new URL("/fs" + current_path + file_name, url_base);
         const response = await fetch(file_path,
             {
                 method: "PUT",
@@ -184,9 +204,12 @@ async function upload(e) {
         if (response.ok) {
             refresh_list();
         }
+        progress.value += 1;
     }
     files.value = "";
-    upload_button.disabled = true;
+    dirs.value = "";
+    progress.value = 0;
+    set_upload_enabled(true);
 }
 
 async function del(e) {
@@ -234,14 +257,8 @@ find_devices();
 let mkdir_button = document.getElementById("mkdir");
 mkdir_button.onclick = mkdir;
 
-let upload_button = document.getElementById("upload");
-upload_button.onclick = upload;
-
-upload_button.disabled = files.files.length == 0;
-
-files.onchange = () => {
-    upload_button.disabled = files.files.length == 0;
-}
+files.onchange = upload;
+dirs.onchange = upload;
 
 mkdir_button.disabled = new_directory_name.value.length == 0;
 

supervisor/shared/web_workflow/web_workflow.c

@@ -42,6 +42,7 @@
 #include "supervisor/shared/translate/translate.h"
 #include "supervisor/shared/web_workflow/web_workflow.h"
 #include "supervisor/shared/web_workflow/websocket.h"
+#include "supervisor/shared/workflow.h"
 #include "supervisor/usb.h"
 
 #include "shared-bindings/hashlib/__init__.h"
@@ -773,40 +774,6 @@ static void _reply_with_version_json(socketpool_socket_obj_t *socket, _request *
     _send_chunk(socket, "");
 }
 
-// Copied from ble file_transfer.c. We should share it.
-STATIC FRESULT _delete_directory_contents(FATFS *fs, const TCHAR *path) {
-    FF_DIR dir;
-    FRESULT res = f_opendir(fs, &dir, path);
-    FILINFO file_info;
-    // Check the stack since we're putting paths on it.
-    if (mp_stack_usage() >= MP_STATE_THREAD(stack_limit)) {
-        return FR_INT_ERR;
-    }
-    while (res == FR_OK) {
-        res = f_readdir(&dir, &file_info);
-        if (res != FR_OK || file_info.fname[0] == '\0') {
-            break;
-        }
-        size_t pathlen = strlen(path);
-        size_t fnlen = strlen(file_info.fname);
-        TCHAR full_path[pathlen + 1 + fnlen];
-        memcpy(full_path, path, pathlen);
-        full_path[pathlen] = '/';
-        size_t full_pathlen = pathlen + 1 + fnlen;
-        memcpy(full_path + pathlen + 1, file_info.fname, fnlen);
-        full_path[full_pathlen] = '\0';
-        if ((file_info.fattrib & AM_DIR) != 0) {
-            res = _delete_directory_contents(fs, full_path);
-        }
-        if (res != FR_OK) {
-            break;
-        }
-        res = f_unlink(fs, full_path);
-    }
-    f_closedir(&dir);
-    return res;
-}
-
 // FATFS has a two second timestamp resolution but the BLE API allows for nanosecond resolution.
 // This function truncates the time the time to a resolution storable by FATFS and fills in the
 // FATFS encoded version into fattime.
@@ -1065,7 +1032,7 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
                 FRESULT result = f_stat(fs, path, &file);
                 if (result == FR_OK) {
                     if ((file.fattrib & AM_DIR) != 0) {
-                        result = _delete_directory_contents(fs, path);
+                        result = supervisor_workflow_delete_directory_contents(fs, path);
                     }
                     if (result == FR_OK) {
                         result = f_unlink(fs, path);
@@ -1144,7 +1111,7 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
                         truncate_time(request->timestamp_ms * 1000000, &fattime);
                         override_fattime(fattime);
                     }
-                    FRESULT result = f_mkdir(fs, path);
+                    FRESULT result = supervisor_workflow_mkdir_parents(fs, path);
                     override_fattime(0);
                     #if CIRCUITPY_USB_MSC
                     usb_msc_unlock();

supervisor/shared/workflow.c

@@ -26,6 +26,8 @@
 
 #include <stdbool.h>
 #include "py/mpconfig.h"
+#include "py/mpstate.h"
+#include "py/stackctrl.h"
 #include "supervisor/background_callback.h"
 #include "supervisor/workflow.h"
 #include "supervisor/serial.h"
@@ -115,3 +117,60 @@ void supervisor_workflow_start(void) {
     supervisor_start_web_workflow();
     #endif
 }
+
+FRESULT supervisor_workflow_mkdir_parents(FATFS *fs, char *path) {
+    FRESULT result = FR_OK;
+    // Make parent directories.
+    for (size_t j = 1; j < strlen(path); j++) {
+        if (path[j] == '/') {
+            path[j] = '\0';
+            result = f_mkdir(fs, path);
+            path[j] = '/';
+            if (result != FR_OK && result != FR_EXIST) {
+                return result;
+            }
+        }
+    }
+    // Make the target directory.
+    return f_mkdir(fs, path);
+}
+
+FRESULT supervisor_workflow_delete_directory_contents(FATFS *fs, const TCHAR *path) {
+    FF_DIR dir;
+    FILINFO file_info;
+    // Check the stack since we're putting paths on it.
+    if (mp_stack_usage() >= MP_STATE_THREAD(stack_limit)) {
+        return FR_INT_ERR;
+    }
+    FRESULT res = FR_OK;
+    while (res == FR_OK) {
+        res = f_opendir(fs, &dir, path);
+        if (res != FR_OK) {
+            break;
+        }
+        res = f_readdir(&dir, &file_info);
+        // We close and reopen the directory every time since we're deleting
+        // entries and it may invalidate the directory handle.
+        f_closedir(&dir);
+        if (res != FR_OK || file_info.fname[0] == '\0') {
+            break;
+        }
+        size_t pathlen = strlen(path);
+        size_t fnlen = strlen(file_info.fname);
+        TCHAR full_path[pathlen + 1 + fnlen];
+        memcpy(full_path, path, pathlen);
+        full_path[pathlen] = '/';
+        size_t full_pathlen = pathlen + 1 + fnlen;
+        memcpy(full_path + pathlen + 1, file_info.fname, fnlen);
+        full_path[full_pathlen] = '\0';
+        if ((file_info.fattrib & AM_DIR) != 0) {
+            res = supervisor_workflow_delete_directory_contents(fs, full_path);
+        }
+        if (res != FR_OK) {
+            break;
+        }
+        res = f_unlink(fs, full_path);
+    }
+    f_closedir(&dir);
+    return res;
+}

supervisor/shared/workflow.h

@@ -26,4 +26,10 @@
 
 #pragma once
 
+#include "lib/oofatfs/ff.h"
+
 extern bool supervisor_workflow_connecting(void);
+
+// File system helpers for workflow code.
+FRESULT supervisor_workflow_mkdir_parents(FATFS *fs, char *path);
+FRESULT supervisor_workflow_delete_directory_contents(FATFS *fs, const TCHAR *path);