implement blob store based attachment upload for slate

- this works
- still need to make it work for codemirror markdown editor
- also need to implement anti-abuse measures

Author: williamstein

src/packages/frontend/editors/slate/upload.tsx

@@ -6,17 +6,10 @@
 import { Transforms } from "slate";
 import { SlateEditor } from "./editable-markdown";
 import { useEffect, useMemo, useRef } from "react";
-import { Dropzone, FileUploadWrapper } from "@cocalc/frontend/file-upload";
-import { join } from "path";
-import { aux_file, path_split } from "@cocalc/util/misc";
-const AUX_FILE_EXT = "upload";
+import { Dropzone, BlobUpload } from "@cocalc/frontend/file-upload";
 import { getFocus } from "./format/commands";
 import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context";
-
-function uploadTarget(path: string, file: { name: string }): string {
-  // path to our upload target, but relative to path.
-  return join(path_split(aux_file(path, AUX_FILE_EXT)).tail, file.name);
-}
+import { BASE_URL } from "@cocalc/frontend/misc";
 
 export default function useUpload(
   editor: SlateEditor,
@@ -69,22 +62,26 @@ export default function useUpload(
       sending: ({ name }) => {
         actionsRef.current?.set_status?.(`Uploading ${name}...`);
       },
-      complete: (file: { type: string; name: string; status: string }) => {
+      complete: (file) => {
         actionsRef.current?.set_status?.("");
+        const { uuid } = JSON.parse(file.xhr.responseText);
+        const url = `${BASE_URL}/blobs/${encodeURIComponent(
+          file.upload.filename,
+        )}?uuid=${uuid}`;
         let node;
-        if (file.type.indexOf("image") == -1) {
+        if (!file.type.includes("image")) {
           node = {
             type: "link",
             isInline: true,
             children: [{ text: file.name }],
-            url: uploadTarget(pathRef.current, file),
+            url,
           };
         } else {
           node = {
             type: "image",
             isInline: true,
             isVoid: true,
-            src: uploadTarget(pathRef.current, file),
+            src: url,
             children: [{ text: "" }],
           };
         }
@@ -96,16 +93,15 @@ export default function useUpload(
   }, []);
 
   return (
-    <FileUploadWrapper
+    <BlobUpload
       className="smc-vfill"
       project_id={project_id}
-      dest_path={aux_file(path, AUX_FILE_EXT)}
       event_handlers={updloadEventHandlers}
       style={{ height: "100%", width: "100%" }}
       dropzone_ref={dropzoneRef}
       show_upload={true}
     >
       {body}
-    </FileUploadWrapper>
+    </BlobUpload>
   );
 }

src/packages/frontend/file-upload.tsx

@@ -10,7 +10,6 @@ import {
   DropzoneComponent,
   DropzoneComponentHandlers,
 } from "react-dropzone-component";
-
 import ReactDOMServer from "react-dom/server"; // for dropzone below
 import { encode_path, defaults, merge, is_array } from "@cocalc/util/misc";
 import {
@@ -26,6 +25,7 @@ import { Icon, Tip } from "@cocalc/frontend/components";
 import { join } from "path";
 import { useStudentProjectFunctionality } from "@cocalc/frontend/course";
 import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
+import { Button } from "antd";
 
 // 3GB upload limit --  since that's the default filesystem quota
 // and it should be plenty?
@@ -37,7 +37,8 @@ const TIMEOUT_S = 100;
 
 const CLOSE_BUTTON_STYLE = {
   position: "absolute",
-  right: 0,
+  right: "15px",
+  top: "5px",
   zIndex: 1, // so it floats above text/markdown buttons
   background: "white",
   cursor: "pointer",
@@ -78,15 +79,26 @@ const DROPSTYLE: React.CSSProperties = {
   margin: "10px 0",
 };
 
-const Header = () => {
+const Header = ({ close_preview }: { close_preview?: Function }) => {
   return (
     <Tip
       icon="file"
       title="Drag and drop files"
       placement="bottom"
       tip="Drag and drop files from your computer into the box below to upload them into your project."
     >
-      <h4 style={{ color: "#666" }}>Drag and drop files from your computer</h4>
+      <h4 style={{ color: "#666", marginLeft: "10px" }}>
+        Drag and drop files from your computer
+        {close_preview && (
+          <Button
+            size="small"
+            style={{ marginLeft: "30px" }}
+            onClick={() => close_preview()}
+          >
+            Close
+          </Button>
+        )}
+      </h4>
     </Tip>
   );
 };
@@ -131,14 +143,16 @@ export const FileUpload: React.FC<FileUploadProps> = (props) => {
 
   function render_close_button() {
     return (
-      <div className="close-button" style={CLOSE_BUTTON_STYLE}>
-        <span
-          onClick={props.close_button_onclick}
-          className="close-button-x"
-          style={{ cursor: "pointer", fontSize: "18px", color: "gray" }}
-        >
-          <Icon name={"times"} />
-        </span>
+      <div style={{ position: "relative" }}>
+        <div className="close-button" style={CLOSE_BUTTON_STYLE}>
+          <span
+            onClick={props.close_button_onclick}
+            className="close-button-x"
+            style={{ cursor: "pointer", fontSize: "18px", color: "gray" }}
+          >
+            <Icon name={"times"} />
+          </span>
+        </div>
       </div>
     );
   }
@@ -319,23 +333,24 @@ export const FileUploadWrapper: React.FC<FileUploadWrapperProps> = (props) => {
 
     return (
       <div style={style}>
-        <div className="close-button" style={CLOSE_BUTTON_STYLE}>
-          <span
-            onClick={() => {
-              close_preview();
-            }}
-            className="close-button-x"
-            style={{
-              cursor: "pointer",
-              fontSize: "18px",
-              color: "gray",
-              marginRight: "20px",
-            }}
-          >
-            <Icon name={"times"} />
-          </span>
+        <div style={{ position: "relative" }}>
+          <div className="close-button" style={CLOSE_BUTTON_STYLE}>
+            <span
+              onClick={() => {
+                close_preview();
+              }}
+              className="close-button-x"
+              style={{
+                cursor: "pointer",
+                fontSize: "18px",
+                color: "gray",
+              }}
+            >
+              <Icon name={"times"} />
+            </span>
+          </div>
         </div>
-        {<Header />}
+        {<Header close_preview={close_preview} />}
         <div
           ref={preview_ref}
           className="filepicker dropzone"
@@ -513,3 +528,14 @@ export function UploadLink({
     </FileUploadWrapper>
   );
 }
+
+export function BlobUpload(props) {
+  const url = `${join(appBasePath, "blobs")}?project_id=${props.project_id}`;
+  return (
+    <FileUploadWrapper
+      {...props}
+      dest_path={""}
+      config={{ url, ...props.config }}
+    />
+  );
+}

src/packages/hub/blobs.coffee

@@ -10,9 +10,7 @@ winston = require('./logger').getLogger('blobs')
 misc_node = require('@cocalc/backend/misc_node')
 misc    = require('@cocalc/util/misc')
 {defaults, required} = misc
-
-MAX_BLOB_SIZE       = 15000000
-MAX_BLOB_SIZE_HUMAN = "15MB"
+{MAX_BLOB_SIZE} = require('@cocalc/util/db-schema/blobs')
 
 # save a blob in the blobstore database with given misc_node.uuidsha1 hash.
 exports.save_blob = (opts) ->
@@ -42,7 +40,7 @@ exports.save_blob = (opts) ->
         err = "save_blob: BUG -- error in call to save_blob; received a save_blob request without corresponding project_id"
 
     else if opts.blob.length > MAX_BLOB_SIZE
-        err = "save_blob: blobs are limited to #{MAX_BLOB_SIZE_HUMAN} and you just tried to save one of size #{opts.blob.length/1000000}MB"
+        err = "save_blob: blobs are limited to #{misc.human_readable_size(MAX_BLOB_SIZE)} and you just tried to save one of size #{opts.blob.length/1000000}MB"
 
     else if opts.check and opts.uuid != misc_node.uuidsha1(opts.blob)
         err = "save_blob: uuid=#{opts.uuid} must be derived from the Sha1 hash of blob, but it is not (possible malicious attack)"

src/packages/hub/package.json

@@ -21,6 +21,7 @@
     "@passport-next/passport-google-oauth2": "^1.0.0",
     "@passport-next/passport-oauth2": "^2.1.4",
     "@types/express": "^4.17.13",
+    "@types/formidable": "^3.4.5",
     "@types/primus": "^7.3.6",
     "@types/react": "^18.0.26",
     "@types/uuid": "^8.3.1",
@@ -37,6 +38,7 @@
     "debug": "^4.3.2",
     "escape-html": "^1.0.3",
     "express": "^4.19.2",
+    "formidable": "^3.5.1",
     "http-proxy": "^1.18.1",
     "immutable": "^4.3.0",
     "jquery": "^3.6.0",

src/packages/hub/servers/app/blob-upload.ts

@@ -2,59 +2,105 @@
 Support user uploading a blob directly from their browser to the CoCalc database,
 mainly for markdown documents.  This is meant to be very similar to how GitHub
 allows for attaching files to github issue comments.
-
-
 */
 
-// Note that github has a 10MB limit -- https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/attaching-files
-const MAX_BLOB_SIZE_MB = 10;
-
-// some throttling -- note that after a bit, most blobs end up longterm
-// cloud storage and are never accessed.
-// this is just a limit to prevent abuse, and it's done on a per-hub
-// basis using an in-memory data structure.
-const MAX_BLOBS_SIZE_MB_PER_USER_PER_DAY = 250;
+// See also src/packages/project/upload.ts
 
 import { Router } from "express";
-import { database } from "../database";
-import { is_valid_uuid_string } from "@cocalc/util/misc";
-import { database_is_working } from "@cocalc/hub/hub_register";
 import { callback2 } from "@cocalc/util/async-utils";
+import { database } from "../database";
+const { save_blob } = require("@cocalc/hub/blobs");
 import { getLogger } from "@cocalc/hub/logger";
+import {
+  MAX_BLOB_SIZE,
+  //MAX_BLOB_SIZE_PER_PROJECT_PER_DAY,
+} from "@cocalc/util/db-schema/blobs";
+import getAccount from "@cocalc/server/auth/get-account";
+import isCollaborator from "@cocalc/server/projects/is-collaborator";
+import formidable from "formidable";
+import { readFile, unlink } from "fs/promises";
+import { uuidsha1 } from "@cocalc/backend/misc_node";
+
+const logger = getLogger("hub:servers:app:blob-upload");
+function dbg(...args): void {
+  logger.debug("upload ", ...args);
+}
 
-const logger = getLogger("hub:servers:app:blobs");
 export default function init(router: Router) {
-  // return uuid-indexed blobs (mainly used for graphics)
   router.post("/blobs", async (req, res) => {
-    logger.debug(`${JSON.stringify(req.query)}, ${req.path}`);
-    const uuid = `${req.query.uuid}`;
-    if (!is_valid_uuid_string(uuid)) {
-      res.status(404).send(`invalid uuid=${uuid}`);
+    const account_id = await getAccount(req);
+    if (!account_id) {
+      res.status(500).send("user must be signed in to upload files");
       return;
     }
-    if (!database_is_working()) {
-      res.status(404).send("can't get blob -- not connected to database");
+    const { project_id, ttl } = req.query;
+    if (!project_id || typeof project_id != "string") {
+      res.status(500).send("project_id must be specified");
+      return;
+    }
+    if (!(await isCollaborator({ account_id, project_id }))) {
+      res.status(500).send("user must be collaborator on project");
       return;
     }
 
+    dbg({ account_id, project_id });
+
+    // TODO: check for throttling/limits
     try {
-      const data = await callback2(database.get_blob, { uuid });
-      if (data == null) {
-        res.status(404).send(`blob ${uuid} not found`);
-      } else {
-        const filename = req.path.slice(req.path.lastIndexOf("/") + 1);
-        if (req.query.download != null) {
-          // tell browser to download the link as a file instead
-          // of displaying it in browser
-          res.attachment(filename);
-        } else {
-          res.type(filename);
+      const form = formidable({
+        keepExtensions: true,
+        maxFileSize: MAX_BLOB_SIZE,
+        hashAlgorithm: "sha1",
+      });
+
+      dbg("parsing form data...");
+      // https://github.com/node-formidable/formidable?tab=readme-ov-file#parserequest-callback
+      const [_, files] = await form.parse(req);
+      //dbg(`finished parsing form data. ${JSON.stringify({ fields, files })}`);
+
+      /* Just for the sake of understanding this, this is how this looks like in the real world (formidable@3):
+      > files.file[0]
+      {
+        size: 80789,
+        filepath: '/home/hsy/p/cocalc/src/data/projects/c8787b71-a85f-437b-9d1b-29833c3a199e/asdf/asdf/8e3e4367333e45275a8d1aa03.png',
+        newFilename: '8e3e4367333e45275a8d1aa03.png',
+        mimetype: 'application/octet-stream',
+        mtime: '2024-04-23T09:25:53.197Z',
+        originalFilename: 'Screenshot from 2024-04-23 09-20-40.png'
+      }
+      */
+      let uuid: string | undefined = undefined;
+      if (files.file?.[0] != null) {
+        const { filepath, hash } = files.file[0];
+        try {
+          dbg("got", files);
+          if (typeof hash == "string") {
+            uuid = uuidsha1("", hash);
+          }
+          const blob = await readFile(filepath);
+          await callback2(save_blob, {
+            uuid,
+            blob,
+            ttl,
+            project_id,
+            database,
+          });
+        } finally {
+          try {
+            await unlink(filepath);
+          } catch (err) {
+            dbg("WARNING -- failed to delete uploaded file", err);
+          }
         }
-        res.send(data);
       }
+      if (!uuid) {
+        res.status(500).send("no file got uploaded");
+        return;
+      }
+      res.send({ uuid });
     } catch (err) {
-      logger.error(`internal error ${err} getting blob ${uuid}`);
-      res.status(500).send(`internal error: ${err}`);
+      dbg("upload failed ", err);
+      res.status(500).send(`upload failed -- ${err}`);
     }
   });
 }