feat(desktop): store humans as markdown files with YAML frontmatter (#2791)

Author: devin-ai-integration[bot]

Cargo.lock

@@ -29,19 +29,20 @@
     "@hypr/api-client": "workspace:*",
     "@hypr/codemirror": "workspace:^",
     "@hypr/plugin-analytics": "workspace:*",
-    "@hypr/plugin-audio-priority": "workspace:*",
     "@hypr/plugin-apple-calendar": "workspace:*",
+    "@hypr/plugin-audio-priority": "workspace:*",
     "@hypr/plugin-auth": "workspace:*",
     "@hypr/plugin-cli2": "workspace:*",
     "@hypr/plugin-db2": "workspace:*",
     "@hypr/plugin-deeplink2": "workspace:*",
     "@hypr/plugin-detect": "workspace:*",
     "@hypr/plugin-export": "workspace:*",
-    "@hypr/plugin-folder": "workspace:*",
     "@hypr/plugin-extensions": "workspace:*",
+    "@hypr/plugin-folder": "workspace:*",
     "@hypr/plugin-hooks": "workspace:*",
     "@hypr/plugin-icon": "workspace:*",
     "@hypr/plugin-importer": "workspace:*",
+    "@hypr/plugin-js": "workspace:*",
     "@hypr/plugin-listener": "workspace:*",
     "@hypr/plugin-listener2": "workspace:*",
     "@hypr/plugin-local-stt": "workspace:*",
@@ -51,15 +52,14 @@
     "@hypr/plugin-notify": "workspace:*",
     "@hypr/plugin-overlay": "workspace:*",
     "@hypr/plugin-path2": "workspace:*",
-    "@hypr/plugin-tantivy": "workspace:*",
     "@hypr/plugin-pdf": "workspace:*",
     "@hypr/plugin-permissions": "workspace:*",
     "@hypr/plugin-settings": "workspace:*",
     "@hypr/plugin-sfx": "workspace:*",
+    "@hypr/plugin-tantivy": "workspace:*",
     "@hypr/plugin-template": "workspace:*",
     "@hypr/plugin-updater2": "workspace:*",
     "@hypr/plugin-windows": "workspace:*",
-    "@hypr/plugin-js": "workspace:*",
     "@hypr/store": "workspace:*",
     "@hypr/tiptap": "workspace:^",
     "@hypr/ui": "workspace:^",

apps/desktop/package.json

@@ -0,0 +1,60 @@
+import type { MergeableStore, OptionalSchemas } from "tinybase/with-schemas";
+
+import type { FrontmatterInput, JsonValue } from "@hypr/plugin-export";
+import type { HumanStorage } from "@hypr/store";
+
+import type { CollectorResult, TablesContent } from "../utils";
+import { getHumanDir, getHumanFilePath } from "./utils";
+
+export interface HumanCollectorResult extends CollectorResult {
+  validHumanIds: Set<string>;
+}
+
+type HumansTable = Record<string, HumanStorage>;
+
+export function collectHumanWriteOps<Schemas extends OptionalSchemas>(
+  _store: MergeableStore<Schemas>,
+  tables: TablesContent,
+  dataDir: string,
+): HumanCollectorResult {
+  const dirs = new Set<string>();
+  const operations: CollectorResult["operations"] = [];
+  const validHumanIds = new Set<string>();
+
+  const humansDir = getHumanDir(dataDir);
+  dirs.add(humansDir);
+
+  const humans = (tables as { humans?: HumansTable }).humans ?? {};
+
+  const frontmatterItems: [FrontmatterInput, string][] = [];
+
+  for (const [humanId, human] of Object.entries(humans)) {
+    validHumanIds.add(humanId);
+
+    const { memo, ...frontmatterFields } = human;
+
+    const frontmatter: Record<string, JsonValue> = {
+      user_id: frontmatterFields.user_id ?? "",
+      created_at: frontmatterFields.created_at ?? "",
+      name: frontmatterFields.name ?? "",
+      email: frontmatterFields.email ?? "",
+      org_id: frontmatterFields.org_id ?? "",
+      job_title: frontmatterFields.job_title ?? "",
+      linkedin_username: frontmatterFields.linkedin_username ?? "",
+    };
+
+    const body = memo ?? "";
+    const filePath = getHumanFilePath(dataDir, humanId);
+
+    frontmatterItems.push([{ frontmatter, content: body }, filePath]);
+  }
+
+  if (frontmatterItems.length > 0) {
+    operations.push({
+      type: "frontmatter-batch",
+      items: frontmatterItems,
+    });
+  }
+
+  return { dirs, operations, validHumanIds };
+}

apps/desktop/src/store/tinybase/persister/human/collect.ts

@@ -0,0 +1,99 @@
+import { readDir, readTextFile, remove } from "@tauri-apps/plugin-fs";
+
+import type { HumanStorage } from "@hypr/store";
+
+import { isFileNotFoundError, isUUID } from "../utils";
+import {
+  getHumanDir,
+  getHumanFilePath,
+  parseMarkdownWithFrontmatter,
+} from "./utils";
+
+export async function loadAllHumans(
+  dataDir: string,
+): Promise<Record<string, HumanStorage>> {
+  const result: Record<string, HumanStorage> = {};
+  const humansDir = getHumanDir(dataDir);
+
+  let entries: { name: string; isDirectory: boolean }[];
+  try {
+    entries = await readDir(humansDir);
+  } catch (error) {
+    if (isFileNotFoundError(error)) {
+      return result;
+    }
+    throw error;
+  }
+
+  for (const entry of entries) {
+    if (entry.isDirectory) continue;
+    if (!entry.name.endsWith(".md")) continue;
+
+    const humanId = entry.name.replace(/\.md$/, "");
+    if (!isUUID(humanId)) {
+      console.warn(`[HumanPersister] Skipping non-UUID file: ${entry.name}`);
+      continue;
+    }
+
+    try {
+      const filePath = getHumanFilePath(dataDir, humanId);
+      const content = await readTextFile(filePath);
+      const { frontmatter, body } = await parseMarkdownWithFrontmatter(content);
+
+      result[humanId] = {
+        user_id: String(frontmatter.user_id ?? ""),
+        created_at: String(frontmatter.created_at ?? ""),
+        name: String(frontmatter.name ?? ""),
+        email: String(frontmatter.email ?? ""),
+        org_id: String(frontmatter.org_id ?? ""),
+        job_title: String(frontmatter.job_title ?? ""),
+        linkedin_username: String(frontmatter.linkedin_username ?? ""),
+        memo: body,
+      };
+    } catch (error) {
+      console.error(`[HumanPersister] Failed to load human ${humanId}:`, error);
+      continue;
+    }
+  }
+
+  return result;
+}
+
+export async function cleanupOrphanHumanFiles(
+  dataDir: string,
+  validHumanIds: Set<string>,
+): Promise<void> {
+  const humansDir = getHumanDir(dataDir);
+
+  let entries: { name: string; isDirectory: boolean }[];
+  try {
+    entries = await readDir(humansDir);
+  } catch (error) {
+    if (isFileNotFoundError(error)) {
+      return;
+    }
+    throw error;
+  }
+
+  for (const entry of entries) {
+    if (entry.isDirectory) continue;
+    if (!entry.name.endsWith(".md")) continue;
+
+    const humanId = entry.name.replace(/\.md$/, "");
+    if (!isUUID(humanId)) continue;
+
+    if (!validHumanIds.has(humanId)) {
+      try {
+        const filePath = getHumanFilePath(dataDir, humanId);
+        await remove(filePath);
+      } catch (error) {
+        if (!isFileNotFoundError(error)) {
+          console.error(
+            `[HumanPersister] Failed to remove orphan file ${entry.name}:`,
+            error,
+          );
+        }
+      }
+    }
+  }
+}

apps/desktop/src/store/tinybase/persister/human/load.ts

@@ -0,0 +1,76 @@
+import { sep } from "@tauri-apps/api/path";
+import { exists, mkdir, readTextFile, remove } from "@tauri-apps/plugin-fs";
+
+import {
+  commands as exportCommands,
+  type FrontmatterInput,
+  type JsonValue,
+} from "@hypr/plugin-export";
+import type { HumanStorage } from "@hypr/store";
+
+import { isFileNotFoundError } from "../utils";
+import { getHumanDir, getHumanFilePath } from "./utils";
+
+export async function migrateHumansJsonIfNeeded(
+  dataDir: string,
+): Promise<void> {
+  const humansJsonPath = [dataDir, "humans.json"].join(sep());
+  const humansDir = getHumanDir(dataDir);
+
+  const jsonExists = await exists(humansJsonPath);
+  if (!jsonExists) {
+    return;
+  }
+
+  const dirExists = await exists(humansDir);
+  if (dirExists) {
+    return;
+  }
+
+  console.log("[HumanPersister] Migrating from humans.json to humans/*.md");
+
+  try {
+    const content = await readTextFile(humansJsonPath);
+    const humans = JSON.parse(content) as Record<string, HumanStorage>;
+
+    await mkdir(humansDir, { recursive: true });
+
+    const batchItems: [FrontmatterInput, string][] = [];
+
+    for (const [humanId, human] of Object.entries(humans)) {
+      const { memo, ...frontmatterFields } = human;
+
+      const frontmatter: Record<string, JsonValue> = {
+        user_id: frontmatterFields.user_id ?? "",
+        created_at: frontmatterFields.created_at ?? "",
+        name: frontmatterFields.name ?? "",
+        email: frontmatterFields.email ?? "",
+        org_id: frontmatterFields.org_id ?? "",
+        job_title: frontmatterFields.job_title ?? "",
+        linkedin_username: frontmatterFields.linkedin_username ?? "",
+      };
+
+      const body = memo ?? "";
+      const filePath = getHumanFilePath(dataDir, humanId);
+
+      batchItems.push([{ frontmatter, content: body }, filePath]);
+    }
+
+    if (batchItems.length > 0) {
+      const result = await exportCommands.exportFrontmatterBatch(batchItems);
+      if (result.status === "error") {
+        throw new Error(`Failed to export migrated humans: ${result.error}`);
+      }
+    }
+
+    await remove(humansJsonPath);
+
+    console.log(
+      `[HumanPersister] Migration complete: ${Object.keys(humans).length} humans migrated`,
+    );
+  } catch (error) {
+    if (!isFileNotFoundError(error)) {
+      console.error("[HumanPersister] Migration failed:", error);
+    }
+  }
+}