Allow external blobs

Author: alexdachin

apps/server/spec/db/document.db

@@ -0,0 +1,94 @@
+import { Application } from "express";
+import { beforeAll, describe, expect, it } from "vitest";
+import supertest from "supertest";
+import { login } from "./utils.js";
+import config from "../../src/services/config.js";
+import sql from "../../src/services/sql.js";
+
+let app: Application;
+let token: string;
+
+const USER = "etapi";
+
+describe("etapi/external-blobs", () => {
+    beforeAll(async () => {
+        config.General.noAuthentication = false;
+        config.ExternalBlobStorage.enabled = true;
+        config.ExternalBlobStorage.thresholdBytes = 10;
+
+        const buildApp = (await import("../../src/app.js")).default;
+        app = await buildApp();
+        token = await login(app);
+    });
+
+    it("stores small note content internally", async () => {
+        const payload = "a".repeat(10);
+        const createRes = await supertest(app)
+            .post("/etapi/create-note")
+            .auth(USER, token, { "type": "basic"})
+            .send({
+                "parentNoteId": "root",
+                "title": "Internal Blob Test",
+                "mime": "text/plain",
+                "type": "text",
+                "content": payload
+            })
+            .expect(201);
+
+        const createdNoteId: string = createRes.body.note.noteId;
+        expect(createdNoteId).toBeTruthy();
+
+        const blobId = sql.getValue<string>("SELECT blobId FROM notes WHERE noteId = ?", [createdNoteId]);
+        expect(blobId).toBeTruthy();
+
+        const row = sql.getRow<{ contentLocation: string; content: string | null; contentLength: number }>(
+            "SELECT contentLocation, content, contentLength FROM blobs WHERE blobId = ?",
+            [blobId]
+        );
+
+        expect(row).toBeTruthy();
+        expect(row.contentLength).toEqual(payload.length);
+        expect(row.contentLocation).toEqual("internal");
+        expect(row.content).toEqual(payload);
+    });
+
+    it("stores large note content externally and serves it back", async () => {
+        const payload = "a".repeat(11);
+        const createRes = await supertest(app)
+            .post("/etapi/create-note")
+            .auth(USER, token, { "type": "basic"})
+            .send({
+                "parentNoteId": "root",
+                "title": "External Blob Test",
+                "mime": "application/octet-stream",
+                "type": "file",
+                "content": payload
+            })
+            .expect(201);
+
+        const createdNoteId: string = createRes.body.note.noteId;
+        expect(createdNoteId).toBeTruthy();
+
+        const blobId = sql.getValue<string>("SELECT blobId FROM notes WHERE noteId = ?", [createdNoteId]);
+        expect(blobId).toBeTruthy();
+
+        const row = sql.getRow<{ contentLocation: string; content: string | null; contentLength: number }>(
+            "SELECT contentLocation, content, contentLength FROM blobs WHERE blobId = ?",
+            [blobId]
+        );
+
+        expect(row).toBeTruthy();
+        expect(row.contentLength).toEqual(payload.length);
+        expect(row.contentLocation.startsWith("file://")).toBe(true);
+        expect(row.content).toBeNull();
+
+        const getRes = await supertest(app)
+            .get(`/etapi/notes/${createdNoteId}/content`)
+            .auth(USER, token, { "type": "basic"})
+            .expect(200);
+
+        expect(getRes.body.toString()).toEqual(payload);
+    });
+});
+
+

apps/server/spec/etapi/external-blobs.spec.ts

@@ -107,6 +107,8 @@ CREATE TABLE IF NOT EXISTS "recent_notes"
 CREATE TABLE IF NOT EXISTS "blobs" (
                                                `blobId`	TEXT NOT NULL,
                                                `content`	TEXT NULL DEFAULT NULL,
+                                               `contentLocation`	TEXT NOT NULL DEFAULT 'internal',
+                                               `contentLength`	INTEGER NOT NULL DEFAULT 0,
                                                `dateModified` TEXT NOT NULL,
                                                `utcDateModified` TEXT NOT NULL,
                                                PRIMARY KEY(`blobId`)

apps/server/src/assets/db/schema.sql

@@ -171,7 +171,7 @@ export default class Becca {
         opts.includeContentLength = !!opts.includeContentLength;
 
         const query = opts.includeContentLength
-            ? /*sql*/`SELECT attachments.*, LENGTH(blobs.content) AS contentLength
+            ? /*sql*/`SELECT attachments.*, blobs.contentLength AS contentLength
                 FROM attachments
                 JOIN blobs USING (blobId)
                 WHERE attachmentId = ? AND isDeleted = 0`
@@ -197,7 +197,7 @@ export default class Becca {
             return null;
         }
 
-        const row = sql.getRow<BlobRow | null>("SELECT *, LENGTH(content) AS contentLength FROM blobs WHERE blobId = ?", [entity.blobId]);
+        const row = sql.getRow<BlobRow | null>("SELECT * FROM blobs WHERE blobId = ?", [entity.blobId]);
         return row ? new BBlob(row) : null;
     }
 

apps/server/src/becca/becca-interface.ts

@@ -9,8 +9,11 @@ import cls from "../../services/cls.js";
 import log from "../../services/log.js";
 import protectedSessionService from "../../services/protected_session.js";
 import blobService from "../../services/blob.js";
+import blobStorageService from "../../services/blob-storage.js";
+import type { Blob } from "../../services/blob-interface.js";
 import type { default as Becca, ConstructorData } from "../becca-interface.js";
 import becca from "../becca.js";
+import type { BlobContentLocation, BlobRow } from "@triliumnext/commons";
 
 interface ContentOpts {
     forceSave?: boolean;
@@ -195,6 +198,11 @@ abstract class AbstractBeccaEntity<T extends AbstractBeccaEntity<T>> {
             return;
         }
 
+        if (blobStorageService.hasExternalContentColumns()) {
+            const row = sql.getRow<{ contentLocation: string }>("SELECT contentLocation FROM blobs WHERE blobId = ?", [oldBlobId]);
+            blobStorageService.deleteExternal(row);
+        }
+
         sql.execute("DELETE FROM blobs WHERE blobId = ?", [oldBlobId]);
         // blobs are not marked as erased in entity_changes, they are just purged completely
         // this is because technically every keystroke can create a new blob, and there would be just too many
@@ -225,14 +233,39 @@ abstract class AbstractBeccaEntity<T extends AbstractBeccaEntity<T>> {
             return newBlobId;
         }
 
-        const pojo = {
+        // Check if we should store this blob externally
+        const shouldStoreExternally = blobStorageService.shouldStoreExternally(content);
+        let contentLocation: BlobContentLocation = "internal";
+        if (shouldStoreExternally) {
+            try {
+                contentLocation = blobStorageService.saveExternal(newBlobId, content);
+            } catch (error) {
+                log.error(`Failed to store blob ${newBlobId} externally, falling back to internal storage: ${error}`);
+                contentLocation = "internal";
+            }
+        }
+
+        const contentLength = blobService.getContentLength(content);
+
+        const pojo: BlobRow = {
             blobId: newBlobId,
-            content: content,
+            content: contentLocation === 'internal' ? content : null,
+            contentLocation,
+            contentLength,
             dateModified: dateUtils.localNowDateTime(),
             utcDateModified: dateUtils.utcNowDateTime()
         };
 
-        sql.upsert("blobs", "blobId", pojo);
+        // external content columns might not be present when applying older migrations
+        const pojoToSave = blobStorageService.hasExternalContentColumns()
+            ? pojo
+            : {
+                blobId: pojo.blobId,
+                content,
+                dateModified: pojo.dateModified,
+                utcDateModified: pojo.utcDateModified
+            };
+        sql.upsert("blobs", "blobId", pojoToSave);
 
         // we can't reuse blobId as an entity_changes hash, because this one has to be calculatable without having
         // access to the decrypted content
@@ -259,14 +292,20 @@ abstract class AbstractBeccaEntity<T extends AbstractBeccaEntity<T>> {
     }
 
     protected _getContent(): string | Buffer {
-        const row = sql.getRow<{ content: string | Buffer }>(/*sql*/`SELECT content FROM blobs WHERE blobId = ?`, [this.blobId]);
+        const query = blobStorageService.hasExternalContentColumns()
+            ? /*sql*/`SELECT content, contentLocation FROM blobs WHERE blobId = ?`
+            : /*sql*/`SELECT content, 'internal' as contentLocation FROM blobs WHERE blobId = ?`;
+
+        const row = sql.getRow<{ content: string | Buffer, contentLocation: string }>(query, [this.blobId]);
 
         if (!row) {
             const constructorData = this.constructor as unknown as ConstructorData<T>;
             throw new Error(`Cannot find content for ${constructorData.primaryKeyName} '${(this as any)[constructorData.primaryKeyName]}', blobId '${this.blobId}'`);
         }
 
-        return blobService.processContent(row.content, this.isProtected || false, this.hasStringContent());
+        const content = blobStorageService.getContent(row);
+
+        return blobService.processContent(content, this.isProtected || false, this.hasStringContent());
     }
 
     /**

apps/server/src/becca/entities/abstract_becca_entity.ts

@@ -1,5 +1,5 @@
 import AbstractBeccaEntity from "./abstract_becca_entity.js";
-import type { BlobRow } from "@triliumnext/commons";
+import type { BlobRow, BlobContentLocation } from "@triliumnext/commons";
 
 // TODO: Why this does not extend the abstract becca?
 class BBlob extends AbstractBeccaEntity<BBlob> {
@@ -10,11 +10,12 @@ class BBlob extends AbstractBeccaEntity<BBlob> {
         return "blobId";
     }
     static get hashedProperties() {
-        return ["blobId", "content"];
+        return ["blobId", "content", "contentLocation"];
     }
 
-    content!: string | Buffer;
+    content!: string | Buffer | null;
     contentLength!: number;
+    contentLocation!: BlobContentLocation;
 
     constructor(row: BlobRow) {
         super();
@@ -25,6 +26,7 @@ class BBlob extends AbstractBeccaEntity<BBlob> {
         this.blobId = row.blobId;
         this.content = row.content;
         this.contentLength = row.contentLength;
+        this.contentLocation = row.contentLocation;
         this.dateModified = row.dateModified;
         this.utcDateModified = row.utcDateModified;
     }
@@ -34,6 +36,7 @@ class BBlob extends AbstractBeccaEntity<BBlob> {
             blobId: this.blobId,
             content: this.content || null,
             contentLength: this.contentLength,
+            contentLocation: this.contentLocation,
             dateModified: this.dateModified,
             utcDateModified: this.utcDateModified
         };

apps/server/src/becca/entities/bblob.ts

@@ -10,6 +10,7 @@ import dateUtils from "../../services/date_utils.js";
 import AbstractBeccaEntity from "./abstract_becca_entity.js";
 import BRevision from "./brevision.js";
 import BAttachment from "./battachment.js";
+import blobStorageService from "../../services/blob-storage.js";
 import TaskContext from "../../services/task_context.js";
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
@@ -1107,8 +1108,12 @@ class BNote extends AbstractBeccaEntity<BNote> {
         // from testing, it looks like calculating length does not make a difference in performance even on large-ish DB
         // given that we're always fetching attachments only for a specific note, we might just do it always
 
+        const contentLengthColumn = blobStorageService.hasExternalContentColumns()
+            ? "blobs.contentLength"
+            : "LENGTH(COALESCE(blobs.content, ''))";
+
         const query = opts.includeContentLength
-            ? /*sql*/`SELECT attachments.*, LENGTH(blobs.content) AS contentLength
+            ? /*sql*/`SELECT attachments.*, ${contentLengthColumn} AS contentLength
                 FROM attachments
                 JOIN blobs USING (blobId)
                 WHERE ownerId = ? AND isDeleted = 0
@@ -1121,8 +1126,12 @@ class BNote extends AbstractBeccaEntity<BNote> {
     getAttachmentById(attachmentId: string, opts: AttachmentOpts = {}) {
         opts.includeContentLength = !!opts.includeContentLength;
 
+        const contentLengthColumn = blobStorageService.hasExternalContentColumns()
+            ? "blobs.contentLength"
+            : "LENGTH(COALESCE(blobs.content, ''))";
+
         const query = opts.includeContentLength
-            ? /*sql*/`SELECT attachments.*, LENGTH(blobs.content) AS contentLength
+            ? /*sql*/`SELECT attachments.*, ${contentLengthColumn} AS contentLength
                 FROM attachments
                 JOIN blobs USING (blobId)
                 WHERE ownerId = ? AND attachmentId = ? AND isDeleted = 0`

apps/server/src/becca/entities/bnote.ts

@@ -6,6 +6,7 @@ import dateUtils from "../../services/date_utils.js";
 import becca from "../becca.js";
 import AbstractBeccaEntity from "./abstract_becca_entity.js";
 import sql from "../../services/sql.js";
+import blobStorageService from "../../services/blob-storage.js";
 import BAttachment from "./battachment.js";
 import type { AttachmentRow, NoteType, RevisionPojo, RevisionRow } from "@triliumnext/commons";
 import eraseService from "../../services/erase.js";
@@ -140,8 +141,12 @@ class BRevision extends AbstractBeccaEntity<BRevision> {
     getAttachmentById(attachmentId: String, opts: GetByIdOpts = {}): BAttachment | null {
         opts.includeContentLength = !!opts.includeContentLength;
 
+        const contentLengthColumn = blobStorageService.hasExternalContentColumns()
+            ? "blobs.contentLength"
+            : "LENGTH(COALESCE(blobs.content, ''))";
+
         const query = opts.includeContentLength
-            ? /*sql*/`SELECT attachments.*, LENGTH(blobs.content) AS contentLength
+            ? /*sql*/`SELECT attachments.*, ${contentLengthColumn} AS contentLength
                 FROM attachments
                 JOIN blobs USING (blobId)
                 WHERE ownerId = ? AND attachmentId = ? AND isDeleted = 0`

apps/server/src/becca/entities/brevision.ts

@@ -6,6 +6,19 @@
 
 // Migrations should be kept in descending order, so the latest migration is first.
 const MIGRATIONS: (SqlMigration | JsMigration)[] = [
+    // Add external blob storage support
+    {
+        version: 234,
+        sql: /*sql*/`
+            -- Add contentLocation column
+            ALTER TABLE blobs ADD contentLocation TEXT DEFAULT 'internal';
+            UPDATE blobs SET contentLocation = 'internal' WHERE contentLocation IS NULL;
+
+            -- Add contentLength column
+            ALTER TABLE blobs ADD contentLength INTEGER DEFAULT 0;
+            UPDATE blobs SET contentLength = CASE WHEN content IS NULL THEN 0 ELSE LENGTH(content) END WHERE contentLength IS NULL;
+        `,
+    },
     // Migrate geo map to collection
     {
         version: 233,

apps/server/src/migrations/migrations.ts

@@ -7,6 +7,7 @@ import cls from "../../services/cls.js";
 import path from "path";
 import becca from "../../becca/becca.js";
 import blobService from "../../services/blob.js";
+import blobStorageService from "../../services/blob-storage.js";
 import eraseService from "../../services/erase.js";
 import type { Request, Response } from "express";
 import type BRevision from "../../becca/entities/brevision.js";
@@ -33,10 +34,14 @@ function getRevisionBlob(req: Request) {
 }
 
 function getRevisions(req: Request) {
+    const contentLengthColumn = blobStorageService.hasExternalContentColumns()
+        ? "blobs.contentLength"
+        : "LENGTH(COALESCE(blobs.content, ''))";
+
     return becca.getRevisionsFromQuery(
         `
         SELECT revisions.*,
-                LENGTH(blobs.content) AS contentLength
+                ${contentLengthColumn} AS contentLength
         FROM revisions
         JOIN blobs ON revisions.blobId = blobs.blobId
         WHERE revisions.noteId = ?