diff --git a/Cargo.lock b/Cargo.lock index a0efbd425a..aa36c3270b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18020,6 +18020,7 @@ name = "tauri-plugin-export" version = "0.1.0" dependencies = [ "criterion", + "frontmatter", "insta", "markdown", "mdast_util_to_markdown", diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 65deaedeea..be92273676 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -29,19 +29,20 @@ "@hypr/api-client": "workspace:*", "@hypr/codemirror": "workspace:^", "@hypr/plugin-analytics": "workspace:*", - "@hypr/plugin-audio-priority": "workspace:*", "@hypr/plugin-apple-calendar": "workspace:*", + "@hypr/plugin-audio-priority": "workspace:*", "@hypr/plugin-auth": "workspace:*", "@hypr/plugin-cli2": "workspace:*", "@hypr/plugin-db2": "workspace:*", "@hypr/plugin-deeplink2": "workspace:*", "@hypr/plugin-detect": "workspace:*", "@hypr/plugin-export": "workspace:*", - "@hypr/plugin-folder": "workspace:*", "@hypr/plugin-extensions": "workspace:*", + "@hypr/plugin-folder": "workspace:*", "@hypr/plugin-hooks": "workspace:*", "@hypr/plugin-icon": "workspace:*", "@hypr/plugin-importer": "workspace:*", + "@hypr/plugin-js": "workspace:*", "@hypr/plugin-listener": "workspace:*", "@hypr/plugin-listener2": "workspace:*", "@hypr/plugin-local-stt": "workspace:*", @@ -51,15 +52,14 @@ "@hypr/plugin-notify": "workspace:*", "@hypr/plugin-overlay": "workspace:*", "@hypr/plugin-path2": "workspace:*", - "@hypr/plugin-tantivy": "workspace:*", "@hypr/plugin-pdf": "workspace:*", "@hypr/plugin-permissions": "workspace:*", "@hypr/plugin-settings": "workspace:*", "@hypr/plugin-sfx": "workspace:*", + "@hypr/plugin-tantivy": "workspace:*", "@hypr/plugin-template": "workspace:*", "@hypr/plugin-updater2": "workspace:*", "@hypr/plugin-windows": "workspace:*", - "@hypr/plugin-js": "workspace:*", "@hypr/store": "workspace:*", "@hypr/tiptap": "workspace:^", "@hypr/ui": "workspace:^", diff --git a/apps/desktop/src/store/tinybase/persister/human/collect.ts b/apps/desktop/src/store/tinybase/persister/human/collect.ts new file mode 100644 index 0000000000..2177adad1a --- /dev/null +++ b/apps/desktop/src/store/tinybase/persister/human/collect.ts @@ -0,0 +1,60 @@ +import type { MergeableStore, OptionalSchemas } from "tinybase/with-schemas"; + +import type { FrontmatterInput, JsonValue } from "@hypr/plugin-export"; +import type { HumanStorage } from "@hypr/store"; + +import type { CollectorResult, TablesContent } from "../utils"; +import { getHumanDir, getHumanFilePath } from "./utils"; + +export interface HumanCollectorResult extends CollectorResult { + validHumanIds: Set; +} + +type HumansTable = Record; + +export function collectHumanWriteOps( + _store: MergeableStore, + tables: TablesContent, + dataDir: string, +): HumanCollectorResult { + const dirs = new Set(); + const operations: CollectorResult["operations"] = []; + const validHumanIds = new Set(); + + const humansDir = getHumanDir(dataDir); + dirs.add(humansDir); + + const humans = (tables as { humans?: HumansTable }).humans ?? {}; + + const frontmatterItems: [FrontmatterInput, string][] = []; + + for (const [humanId, human] of Object.entries(humans)) { + validHumanIds.add(humanId); + + const { memo, ...frontmatterFields } = human; + + const frontmatter: Record = { + user_id: frontmatterFields.user_id ?? "", + created_at: frontmatterFields.created_at ?? "", + name: frontmatterFields.name ?? "", + email: frontmatterFields.email ?? "", + org_id: frontmatterFields.org_id ?? "", + job_title: frontmatterFields.job_title ?? "", + linkedin_username: frontmatterFields.linkedin_username ?? "", + }; + + const body = memo ?? ""; + const filePath = getHumanFilePath(dataDir, humanId); + + frontmatterItems.push([{ frontmatter, content: body }, filePath]); + } + + if (frontmatterItems.length > 0) { + operations.push({ + type: "frontmatter-batch", + items: frontmatterItems, + }); + } + + return { dirs, operations, validHumanIds }; +} diff --git a/apps/desktop/src/store/tinybase/persister/human/load.ts b/apps/desktop/src/store/tinybase/persister/human/load.ts new file mode 100644 index 0000000000..bdaa28d0b5 --- /dev/null +++ b/apps/desktop/src/store/tinybase/persister/human/load.ts @@ -0,0 +1,99 @@ +import { readDir, readTextFile, remove } from "@tauri-apps/plugin-fs"; + +import type { HumanStorage } from "@hypr/store"; + +import { isFileNotFoundError, isUUID } from "../utils"; +import { + getHumanDir, + getHumanFilePath, + parseMarkdownWithFrontmatter, +} from "./utils"; + +export async function loadAllHumans( + dataDir: string, +): Promise> { + const result: Record = {}; + const humansDir = getHumanDir(dataDir); + + let entries: { name: string; isDirectory: boolean }[]; + try { + entries = await readDir(humansDir); + } catch (error) { + if (isFileNotFoundError(error)) { + return result; + } + throw error; + } + + for (const entry of entries) { + if (entry.isDirectory) continue; + if (!entry.name.endsWith(".md")) continue; + + const humanId = entry.name.replace(/\.md$/, ""); + if (!isUUID(humanId)) { + console.warn(`[HumanPersister] Skipping non-UUID file: ${entry.name}`); + continue; + } + + try { + const filePath = getHumanFilePath(dataDir, humanId); + const content = await readTextFile(filePath); + const { frontmatter, body } = await parseMarkdownWithFrontmatter(content); + + result[humanId] = { + user_id: String(frontmatter.user_id ?? ""), + created_at: String(frontmatter.created_at ?? ""), + name: String(frontmatter.name ?? ""), + email: String(frontmatter.email ?? ""), + org_id: String(frontmatter.org_id ?? ""), + job_title: String(frontmatter.job_title ?? ""), + linkedin_username: String(frontmatter.linkedin_username ?? ""), + memo: body, + }; + } catch (error) { + console.error(`[HumanPersister] Failed to load human ${humanId}:`, error); + continue; + } + } + + return result; +} + +export async function cleanupOrphanHumanFiles( + dataDir: string, + validHumanIds: Set, +): Promise { + const humansDir = getHumanDir(dataDir); + + let entries: { name: string; isDirectory: boolean }[]; + try { + entries = await readDir(humansDir); + } catch (error) { + if (isFileNotFoundError(error)) { + return; + } + throw error; + } + + for (const entry of entries) { + if (entry.isDirectory) continue; + if (!entry.name.endsWith(".md")) continue; + + const humanId = entry.name.replace(/\.md$/, ""); + if (!isUUID(humanId)) continue; + + if (!validHumanIds.has(humanId)) { + try { + const filePath = getHumanFilePath(dataDir, humanId); + await remove(filePath); + } catch (error) { + if (!isFileNotFoundError(error)) { + console.error( + `[HumanPersister] Failed to remove orphan file ${entry.name}:`, + error, + ); + } + } + } + } +} diff --git a/apps/desktop/src/store/tinybase/persister/human/migrate.ts b/apps/desktop/src/store/tinybase/persister/human/migrate.ts new file mode 100644 index 0000000000..14d48dc5eb --- /dev/null +++ b/apps/desktop/src/store/tinybase/persister/human/migrate.ts @@ -0,0 +1,76 @@ +import { sep } from "@tauri-apps/api/path"; +import { exists, mkdir, readTextFile, remove } from "@tauri-apps/plugin-fs"; + +import { + commands as exportCommands, + type FrontmatterInput, + type JsonValue, +} from "@hypr/plugin-export"; +import type { HumanStorage } from "@hypr/store"; + +import { isFileNotFoundError } from "../utils"; +import { getHumanDir, getHumanFilePath } from "./utils"; + +export async function migrateHumansJsonIfNeeded( + dataDir: string, +): Promise { + const humansJsonPath = [dataDir, "humans.json"].join(sep()); + const humansDir = getHumanDir(dataDir); + + const jsonExists = await exists(humansJsonPath); + if (!jsonExists) { + return; + } + + const dirExists = await exists(humansDir); + if (dirExists) { + return; + } + + console.log("[HumanPersister] Migrating from humans.json to humans/*.md"); + + try { + const content = await readTextFile(humansJsonPath); + const humans = JSON.parse(content) as Record; + + await mkdir(humansDir, { recursive: true }); + + const batchItems: [FrontmatterInput, string][] = []; + + for (const [humanId, human] of Object.entries(humans)) { + const { memo, ...frontmatterFields } = human; + + const frontmatter: Record = { + user_id: frontmatterFields.user_id ?? "", + created_at: frontmatterFields.created_at ?? "", + name: frontmatterFields.name ?? "", + email: frontmatterFields.email ?? "", + org_id: frontmatterFields.org_id ?? "", + job_title: frontmatterFields.job_title ?? "", + linkedin_username: frontmatterFields.linkedin_username ?? "", + }; + + const body = memo ?? ""; + const filePath = getHumanFilePath(dataDir, humanId); + + batchItems.push([{ frontmatter, content: body }, filePath]); + } + + if (batchItems.length > 0) { + const result = await exportCommands.exportFrontmatterBatch(batchItems); + if (result.status === "error") { + throw new Error(`Failed to export migrated humans: ${result.error}`); + } + } + + await remove(humansJsonPath); + + console.log( + `[HumanPersister] Migration complete: ${Object.keys(humans).length} humans migrated`, + ); + } catch (error) { + if (!isFileNotFoundError(error)) { + console.error("[HumanPersister] Migration failed:", error); + } + } +} diff --git a/apps/desktop/src/store/tinybase/persister/human/persister.test.ts b/apps/desktop/src/store/tinybase/persister/human/persister.test.ts index e3e3d9c3a0..28cd9a7886 100644 --- a/apps/desktop/src/store/tinybase/persister/human/persister.test.ts +++ b/apps/desktop/src/store/tinybase/persister/human/persister.test.ts @@ -4,6 +4,7 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; import { SCHEMA, type Schemas } from "@hypr/store"; import { createHumanPersister } from "./persister"; +import { parseMarkdownWithFrontmatter } from "./utils"; vi.mock("@hypr/plugin-path2", () => ({ commands: { @@ -11,18 +12,41 @@ vi.mock("@hypr/plugin-path2", () => ({ }, })); +vi.mock("@hypr/plugin-export", () => ({ + commands: { + parseFrontmatter: vi.fn(), + exportFrontmatterBatch: vi + .fn() + .mockResolvedValue({ status: "ok", data: null }), + }, +})); + vi.mock("@tauri-apps/plugin-fs", () => ({ mkdir: vi.fn().mockResolvedValue(undefined), + readDir: vi.fn(), readTextFile: vi.fn(), writeTextFile: vi.fn().mockResolvedValue(undefined), + exists: vi.fn(), + remove: vi.fn().mockResolvedValue(undefined), })); +function serializeFrontmatterSync( + frontmatter: Record, + body: string, +): string { + const lines = Object.entries(frontmatter).map(([k, v]) => `${k}: ${v}`); + return `---\n${lines.join("\n")}\n---\n\n${body}`; +} + function createTestStore() { return createMergeableStore() .setTablesSchema(SCHEMA.table) .setValuesSchema(SCHEMA.value); } +const HUMAN_UUID_1 = "550e8400-e29b-41d4-a716-446655440000"; +const HUMAN_UUID_2 = "550e8400-e29b-41d4-a716-446655440001"; + describe("createHumanPersister", () => { let store: ReturnType; @@ -41,11 +65,23 @@ describe("createHumanPersister", () => { }); describe("load", () => { - test("loads humans from json file", async () => { - const { readTextFile } = await import("@tauri-apps/plugin-fs"); + test("loads humans from markdown files", async () => { + const { readDir, readTextFile, exists } = + await import("@tauri-apps/plugin-fs"); + const { commands: exportCommands } = await import("@hypr/plugin-export"); + + vi.mocked(exists).mockResolvedValue(false); + vi.mocked(readDir).mockResolvedValue([ + { + name: `${HUMAN_UUID_1}.md`, + isDirectory: false, + isFile: true, + isSymlink: false, + }, + ]); - const mockData = { - "human-1": { + const mockMdContent = serializeFrontmatterSync( + { user_id: "user-1", created_at: "2024-01-01T00:00:00Z", name: "John Doe", @@ -53,24 +89,49 @@ describe("createHumanPersister", () => { org_id: "org-1", job_title: "Engineer", linkedin_username: "johndoe", - memo: "Some notes", }, - }; - vi.mocked(readTextFile).mockResolvedValue(JSON.stringify(mockData)); + "Some notes", + ); + vi.mocked(readTextFile).mockResolvedValue(mockMdContent); + vi.mocked(exportCommands.parseFrontmatter).mockResolvedValue({ + status: "ok", + data: { + frontmatter: { + user_id: "user-1", + created_at: "2024-01-01T00:00:00Z", + name: "John Doe", + email: "john@example.com", + org_id: "org-1", + job_title: "Engineer", + linkedin_username: "johndoe", + }, + content: "Some notes", + }, + }); const persister = createHumanPersister(store); await persister.load(); - expect(readTextFile).toHaveBeenCalledWith( - "/mock/data/dir/hyprnote/humans.json", - ); - expect(store.getTable("humans")).toEqual(mockData); + expect(readDir).toHaveBeenCalledWith("/mock/data/dir/hyprnote/humans"); + + const humans = store.getTable("humans"); + expect(humans[HUMAN_UUID_1]).toEqual({ + user_id: "user-1", + created_at: "2024-01-01T00:00:00Z", + name: "John Doe", + email: "john@example.com", + org_id: "org-1", + job_title: "Engineer", + linkedin_username: "johndoe", + memo: "Some notes", + }); }); - test("returns empty humans when file does not exist", async () => { - const { readTextFile } = await import("@tauri-apps/plugin-fs"); + test("returns empty humans when directory does not exist", async () => { + const { readDir, exists } = await import("@tauri-apps/plugin-fs"); - vi.mocked(readTextFile).mockRejectedValue( + vi.mocked(exists).mockResolvedValue(false); + vi.mocked(readDir).mockRejectedValue( new Error("No such file or directory"), ); @@ -79,13 +140,72 @@ describe("createHumanPersister", () => { expect(store.getTable("humans")).toEqual({}); }); + + test("skips non-UUID files", async () => { + const { readDir, readTextFile, exists } = + await import("@tauri-apps/plugin-fs"); + const { commands: exportCommands } = await import("@hypr/plugin-export"); + + vi.mocked(exists).mockResolvedValue(false); + vi.mocked(readDir).mockResolvedValue([ + { + name: "not-a-uuid.md", + isDirectory: false, + isFile: true, + isSymlink: false, + }, + { + name: `${HUMAN_UUID_1}.md`, + isDirectory: false, + isFile: true, + isSymlink: false, + }, + ]); + + const mockMdContent = serializeFrontmatterSync( + { + user_id: "user-1", + created_at: "2024-01-01T00:00:00Z", + name: "John Doe", + email: "john@example.com", + org_id: "", + job_title: "", + linkedin_username: "", + }, + "", + ); + vi.mocked(readTextFile).mockResolvedValue(mockMdContent); + vi.mocked(exportCommands.parseFrontmatter).mockResolvedValue({ + status: "ok", + data: { + frontmatter: { + user_id: "user-1", + created_at: "2024-01-01T00:00:00Z", + name: "John Doe", + email: "john@example.com", + org_id: "", + job_title: "", + linkedin_username: "", + }, + content: "", + }, + }); + + const persister = createHumanPersister(store); + await persister.load(); + + const humans = store.getTable("humans"); + expect(Object.keys(humans)).toHaveLength(1); + expect(humans[HUMAN_UUID_1]).toBeDefined(); + }); }); describe("save", () => { - test("saves humans to json file", async () => { - const { writeTextFile } = await import("@tauri-apps/plugin-fs"); + test("saves humans to markdown files via export plugin", async () => { + const { mkdir } = await import("@tauri-apps/plugin-fs"); + const { commands: exportCommands } = await import("@hypr/plugin-export"); - store.setRow("humans", "human-1", { + store.setRow("humans", HUMAN_UUID_1, { user_id: "user-1", created_at: "2024-01-01T00:00:00Z", name: "John Doe", @@ -99,44 +219,42 @@ describe("createHumanPersister", () => { const persister = createHumanPersister(store); await persister.save(); - expect(writeTextFile).toHaveBeenCalledWith( - "/mock/data/dir/hyprnote/humans.json", - expect.any(String), - ); - - const writtenContent = vi.mocked(writeTextFile).mock.calls[0][1]; - const parsed = JSON.parse(writtenContent); - - expect(parsed).toEqual({ - "human-1": { - user_id: "user-1", - created_at: "2024-01-01T00:00:00Z", - name: "John Doe", - email: "john@example.com", - org_id: "org-1", - job_title: "Engineer", - linkedin_username: "johndoe", - memo: "Some notes", - }, + expect(mkdir).toHaveBeenCalledWith("/mock/data/dir/hyprnote/humans", { + recursive: true, }); + + expect(exportCommands.exportFrontmatterBatch).toHaveBeenCalledWith([ + [ + { + frontmatter: { + user_id: "user-1", + created_at: "2024-01-01T00:00:00Z", + name: "John Doe", + email: "john@example.com", + org_id: "org-1", + job_title: "Engineer", + linkedin_username: "johndoe", + }, + content: "Some notes", + }, + `/mock/data/dir/hyprnote/humans/${HUMAN_UUID_1}.md`, + ], + ]); }); - test("writes empty object when no humans exist", async () => { - const { writeTextFile } = await import("@tauri-apps/plugin-fs"); + test("does not write when no humans exist", async () => { + const { commands: exportCommands } = await import("@hypr/plugin-export"); const persister = createHumanPersister(store); await persister.save(); - expect(writeTextFile).toHaveBeenCalledWith( - "/mock/data/dir/hyprnote/humans.json", - "{}", - ); + expect(exportCommands.exportFrontmatterBatch).not.toHaveBeenCalled(); }); - test("saves multiple humans", async () => { - const { writeTextFile } = await import("@tauri-apps/plugin-fs"); + test("saves multiple humans in single batch call", async () => { + const { commands: exportCommands } = await import("@hypr/plugin-export"); - store.setRow("humans", "human-1", { + store.setRow("humans", HUMAN_UUID_1, { user_id: "user-1", created_at: "2024-01-01T00:00:00Z", name: "John Doe", @@ -147,7 +265,7 @@ describe("createHumanPersister", () => { memo: "", }); - store.setRow("humans", "human-2", { + store.setRow("humans", HUMAN_UUID_2, { user_id: "user-1", created_at: "2024-01-02T00:00:00Z", name: "Jane Smith", @@ -161,12 +279,125 @@ describe("createHumanPersister", () => { const persister = createHumanPersister(store); await persister.save(); - const writtenContent = vi.mocked(writeTextFile).mock.calls[0][1]; - const parsed = JSON.parse(writtenContent); + expect(exportCommands.exportFrontmatterBatch).toHaveBeenCalledTimes(1); + + const batchItems = vi.mocked(exportCommands.exportFrontmatterBatch).mock + .calls[0][0]; + expect(batchItems).toHaveLength(2); + + const paths = batchItems.map((item: [unknown, string]) => item[1]); + expect(paths).toContain( + `/mock/data/dir/hyprnote/humans/${HUMAN_UUID_1}.md`, + ); + expect(paths).toContain( + `/mock/data/dir/hyprnote/humans/${HUMAN_UUID_2}.md`, + ); + }); + }); + + describe("migration", () => { + test("migrates from humans.json when it exists and humans dir does not", async () => { + const { exists, readTextFile, mkdir, remove } = + await import("@tauri-apps/plugin-fs"); + const { commands: exportCommands } = await import("@hypr/plugin-export"); + + vi.mocked(exists).mockImplementation(async (path: string | URL) => { + const p = typeof path === "string" ? path : path.toString(); + if (p.endsWith("humans.json")) return true; + if (p.endsWith("humans")) return false; + return false; + }); + + const mockJsonData = { + [HUMAN_UUID_1]: { + user_id: "user-1", + created_at: "2024-01-01T00:00:00Z", + name: "John Doe", + email: "john@example.com", + org_id: "org-1", + job_title: "Engineer", + linkedin_username: "johndoe", + memo: "Some notes", + }, + }; + vi.mocked(readTextFile).mockResolvedValue(JSON.stringify(mockJsonData)); + + const persister = createHumanPersister(store); + await persister.load(); + + expect(mkdir).toHaveBeenCalledWith("/mock/data/dir/hyprnote/humans", { + recursive: true, + }); + expect(exportCommands.exportFrontmatterBatch).toHaveBeenCalledWith([ + [ + { + frontmatter: { + user_id: "user-1", + created_at: "2024-01-01T00:00:00Z", + name: "John Doe", + email: "john@example.com", + org_id: "org-1", + job_title: "Engineer", + linkedin_username: "johndoe", + }, + content: "Some notes", + }, + `/mock/data/dir/hyprnote/humans/${HUMAN_UUID_1}.md`, + ], + ]); + expect(remove).toHaveBeenCalledWith( + "/mock/data/dir/hyprnote/humans.json", + ); + }); + }); +}); + +describe("utils", () => { + describe("parseMarkdownWithFrontmatter", () => { + test("parses markdown with frontmatter via plugin", async () => { + const { commands: exportCommands } = await import("@hypr/plugin-export"); + + vi.mocked(exportCommands.parseFrontmatter).mockResolvedValue({ + status: "ok", + data: { + frontmatter: { + name: "John Doe", + email: "john@example.com", + }, + content: "Some notes about John", + }, + }); + + const content = `--- +name: John Doe +email: john@example.com +--- + +Some notes about John`; + + const { frontmatter, body } = await parseMarkdownWithFrontmatter(content); + + expect(frontmatter).toEqual({ + name: "John Doe", + email: "john@example.com", + }); + expect(body).toBe("Some notes about John"); + }); + + test("returns empty frontmatter when plugin returns error", async () => { + const { commands: exportCommands } = await import("@hypr/plugin-export"); + + vi.mocked(exportCommands.parseFrontmatter).mockResolvedValue({ + status: "error", + error: "Parse error", + }); + + const content = "Just some text without frontmatter"; + + const { frontmatter, body } = await parseMarkdownWithFrontmatter(content); - expect(Object.keys(parsed)).toHaveLength(2); - expect(parsed["human-1"].name).toBe("John Doe"); - expect(parsed["human-2"].name).toBe("Jane Smith"); + expect(frontmatter).toEqual({}); + expect(body).toBe("Just some text without frontmatter"); }); }); }); diff --git a/apps/desktop/src/store/tinybase/persister/human/persister.ts b/apps/desktop/src/store/tinybase/persister/human/persister.ts index d7b26be10c..1e8788d9ab 100644 --- a/apps/desktop/src/store/tinybase/persister/human/persister.ts +++ b/apps/desktop/src/store/tinybase/persister/human/persister.ts @@ -1,13 +1,32 @@ -import type { MergeableStore, OptionalSchemas } from "tinybase/with-schemas"; +import type { + Content, + MergeableStore, + OptionalSchemas, +} from "tinybase/with-schemas"; -import { createSingleTablePersister } from "../utils"; +import { createSessionDirPersister, getDataDir } from "../utils"; +import { collectHumanWriteOps, type HumanCollectorResult } from "./collect"; +import { cleanupOrphanHumanFiles, loadAllHumans } from "./load"; +import { migrateHumansJsonIfNeeded } from "./migrate"; export function createHumanPersister( store: MergeableStore, ) { - return createSingleTablePersister(store, { - tableName: "humans", - filename: "humans.json", + return createSessionDirPersister(store, { label: "HumanPersister", + collect: collectHumanWriteOps, + load: async (): Promise | undefined> => { + const dataDir = await getDataDir(); + await migrateHumansJsonIfNeeded(dataDir); + const humans = await loadAllHumans(dataDir); + if (Object.keys(humans).length === 0) { + return undefined; + } + return [{ humans }, {}] as unknown as Content; + }, + postSave: async (dataDir, result) => { + const { validHumanIds } = result as HumanCollectorResult; + await cleanupOrphanHumanFiles(dataDir, validHumanIds); + }, }); } diff --git a/apps/desktop/src/store/tinybase/persister/human/utils.ts b/apps/desktop/src/store/tinybase/persister/human/utils.ts new file mode 100644 index 0000000000..2f88b4b2d6 --- /dev/null +++ b/apps/desktop/src/store/tinybase/persister/human/utils.ts @@ -0,0 +1,33 @@ +import { sep } from "@tauri-apps/api/path"; + +import { commands } from "@hypr/plugin-export"; + +export interface ParsedMarkdown { + frontmatter: Record; + body: string; +} + +export async function parseMarkdownWithFrontmatter( + content: string, +): Promise { + const result = await commands.parseFrontmatter(content); + if (result.status === "error") { + console.error( + "[HumanPersister] Failed to parse frontmatter:", + result.error, + ); + return { frontmatter: {}, body: content }; + } + return { + frontmatter: result.data.frontmatter as Record, + body: result.data.content.trim(), + }; +} + +export function getHumanDir(dataDir: string): string { + return [dataDir, "humans"].join(sep()); +} + +export function getHumanFilePath(dataDir: string, humanId: string): string { + return [dataDir, "humans", `${humanId}.md`].join(sep()); +} diff --git a/apps/desktop/src/store/tinybase/persister/utils.ts b/apps/desktop/src/store/tinybase/persister/utils.ts index 812a017c59..aea0589ef9 100644 --- a/apps/desktop/src/store/tinybase/persister/utils.ts +++ b/apps/desktop/src/store/tinybase/persister/utils.ts @@ -10,6 +10,7 @@ import type { import { commands as exportCommands, type JsonValue as ExportJsonValue, + type FrontmatterInput, } from "@hypr/plugin-export"; import { events as notifyEvents } from "@hypr/plugin-notify"; import { commands as path2Commands } from "@hypr/plugin-path2"; @@ -226,7 +227,9 @@ export function createModeAwarePersister( export type WriteOperation = | { type: "json"; path: string; content: unknown } - | { type: "md-batch"; items: Array<[ExportJsonValue, string]> }; + | { type: "md-batch"; items: Array<[ExportJsonValue, string]> } + | { type: "text"; path: string; content: string } + | { type: "frontmatter-batch"; items: Array<[FrontmatterInput, string]> }; export type CollectorResult = { dirs: Set; @@ -263,12 +266,18 @@ export function createSessionDirPersister( const jsonBatchItems: Array<[ExportJsonValue, string]> = []; let mdBatchItems: Array<[ExportJsonValue, string]> = []; + let frontmatterBatchItems: Array<[FrontmatterInput, string]> = []; + const textItems: Array<{ path: string; content: string }> = []; for (const op of operations) { if (op.type === "json") { jsonBatchItems.push([op.content as ExportJsonValue, op.path]); } else if (op.type === "md-batch") { mdBatchItems = mdBatchItems.concat(op.items); + } else if (op.type === "frontmatter-batch") { + frontmatterBatchItems = frontmatterBatchItems.concat(op.items); + } else if (op.type === "text") { + textItems.push({ path: op.path, content: op.content }); } } @@ -294,6 +303,29 @@ export function createSessionDirPersister( } } + if (frontmatterBatchItems.length > 0) { + const exportResult = await exportCommands.exportFrontmatterBatch( + frontmatterBatchItems, + ); + if (exportResult.status === "error") { + console.error( + `[${options.label}] Failed to export frontmatter batch:`, + exportResult.error, + ); + } + } + + for (const item of textItems) { + try { + await writeTextFile(item.path, item.content); + } catch (e) { + console.error( + `[${options.label}] Failed to write text file ${item.path}:`, + e, + ); + } + } + if (options.postSave) { await options.postSave(dataDir, result); } diff --git a/plugins/export/Cargo.toml b/plugins/export/Cargo.toml index 6519858a09..33e83ae279 100644 --- a/plugins/export/Cargo.toml +++ b/plugins/export/Cargo.toml @@ -21,6 +21,7 @@ specta-typescript = { workspace = true } tokio = { workspace = true, features = ["macros"] } [dependencies] +frontmatter = { path = "../../crates/frontmatter" } markdown = { workspace = true } mdast_util_to_markdown = { workspace = true } diff --git a/plugins/export/js/bindings.gen.ts b/plugins/export/js/bindings.gen.ts index 7cab2ad85e..5ef1a2f26a 100644 --- a/plugins/export/js/bindings.gen.ts +++ b/plugins/export/js/bindings.gen.ts @@ -37,6 +37,52 @@ async exportTiptapJsonToMdBatch(items: ([JsonValue, string])[]) : Promise> { + try { + return { status: "ok", data: await TAURI_INVOKE("plugin:export|parse_frontmatter", { markdown }) }; +} catch (e) { + if(e instanceof Error) throw e; + else return { status: "error", error: e as any }; +} +}, +/** + * Serialize frontmatter and content into markdown with YAML frontmatter. + */ +async serializeFrontmatter(input: FrontmatterInput) : Promise> { + try { + return { status: "ok", data: await TAURI_INVOKE("plugin:export|serialize_frontmatter", { input }) }; +} catch (e) { + if(e instanceof Error) throw e; + else return { status: "error", error: e as any }; +} +}, +/** + * Write markdown with frontmatter to a file. + */ +async exportFrontmatter(input: FrontmatterInput, path: string) : Promise> { + try { + return { status: "ok", data: await TAURI_INVOKE("plugin:export|export_frontmatter", { input, path }) }; +} catch (e) { + if(e instanceof Error) throw e; + else return { status: "error", error: e as any }; +} +}, +/** + * Batch write multiple markdown files with frontmatter. + */ +async exportFrontmatterBatch(items: ([FrontmatterInput, string])[]) : Promise> { + try { + return { status: "ok", data: await TAURI_INVOKE("plugin:export|export_frontmatter_batch", { items }) }; +} catch (e) { + if(e instanceof Error) throw e; + else return { status: "error", error: e as any }; +} } } @@ -50,7 +96,9 @@ async exportTiptapJsonToMdBatch(items: ([JsonValue, string])[]) : Promise; content: string } export type JsonValue = null | boolean | number | string | JsonValue[] | Partial<{ [key in string]: JsonValue }> +export type ParsedFrontmatter = { frontmatter: Partial<{ [key in string]: JsonValue }>; content: string } /** tauri-specta globals **/ diff --git a/plugins/export/src/commands.rs b/plugins/export/src/commands.rs index 9f5f8980ef..f39cf94298 100644 --- a/plugins/export/src/commands.rs +++ b/plugins/export/src/commands.rs @@ -1,5 +1,8 @@ +use frontmatter::Document; use rayon::prelude::*; +use serde::{Deserialize, Serialize}; use serde_json::Value; +use std::collections::HashMap; /// For many small files with frequent writes, sync I/O with rayon parallelism /// is more efficient than async I/O (avoids per-file async task overhead). @@ -12,6 +15,18 @@ macro_rules! spawn_blocking { }; } +#[derive(Debug, Clone, Serialize, Deserialize, specta::Type)] +pub struct ParsedFrontmatter { + pub frontmatter: HashMap, + pub content: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, specta::Type)] +pub struct FrontmatterInput { + pub frontmatter: HashMap, + pub content: String, +} + #[tauri::command] #[specta::specta] pub(crate) async fn export_json(json: Value, path: String) -> Result<(), String> { @@ -56,3 +71,67 @@ pub(crate) async fn export_tiptap_json_to_md_batch( }) }) } + +/// Parse markdown with YAML frontmatter into structured data. +/// Returns frontmatter as a HashMap and the content body. +/// If the markdown has no frontmatter, returns empty frontmatter and the full content as body. +#[tauri::command] +#[specta::specta] +pub(crate) async fn parse_frontmatter(markdown: String) -> Result { + spawn_blocking!({ + match Document::>::from_str(&markdown) { + Ok(doc) => Ok(ParsedFrontmatter { + frontmatter: doc.frontmatter, + content: doc.content, + }), + Err(frontmatter::Error::MissingOpeningDelimiter) + | Err(frontmatter::Error::MissingClosingDelimiter) => { + // No frontmatter, treat entire content as body + Ok(ParsedFrontmatter { + frontmatter: HashMap::new(), + content: markdown, + }) + } + Err(e) => Err(e.to_string()), + } + }) +} + +/// Serialize frontmatter and content into markdown with YAML frontmatter. +#[tauri::command] +#[specta::specta] +pub(crate) async fn serialize_frontmatter(input: FrontmatterInput) -> Result { + spawn_blocking!({ + let doc = Document::new(input.frontmatter, input.content); + doc.to_string().map_err(|e| e.to_string()) + }) +} + +/// Write markdown with frontmatter to a file. +#[tauri::command] +#[specta::specta] +pub(crate) async fn export_frontmatter( + input: FrontmatterInput, + path: String, +) -> Result<(), String> { + spawn_blocking!({ + let doc = Document::new(input.frontmatter, input.content); + let content = doc.to_string().map_err(|e| e.to_string())?; + std::fs::write(path, content).map_err(|e| e.to_string()) + }) +} + +/// Batch write multiple markdown files with frontmatter. +#[tauri::command] +#[specta::specta] +pub(crate) async fn export_frontmatter_batch( + items: Vec<(FrontmatterInput, String)>, +) -> Result<(), String> { + spawn_blocking!({ + items.into_par_iter().try_for_each(|(input, path)| { + let doc = Document::new(input.frontmatter, input.content); + let content = doc.to_string().map_err(|e| e.to_string())?; + std::fs::write(path, content).map_err(|e| e.to_string()) + }) + }) +} diff --git a/plugins/export/src/lib.rs b/plugins/export/src/lib.rs index 3423d31d8b..5b43d902a5 100644 --- a/plugins/export/src/lib.rs +++ b/plugins/export/src/lib.rs @@ -18,6 +18,10 @@ fn make_specta_builder() -> tauri_specta::Builder { commands::export_json_batch, commands::export_tiptap_json_to_md, commands::export_tiptap_json_to_md_batch, + commands::parse_frontmatter, + commands::serialize_frontmatter, + commands::export_frontmatter, + commands::export_frontmatter_batch, ]) .error_handling(tauri_specta::ErrorHandlingMode::Result) }