frontend/starred file bookmarks: migrate to conat

Author: haraldschilly

src/CLAUDE.md

@@ -73,6 +73,8 @@ CoCalc is organized as a monorepo with key packages:
 - **TypeScript React Components**: All frontend code is TypeScript with proper typing
 - **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.)
 - **WebSocket Communication**: Real-time communication with backend via WebSocket messages
+- **Authentication Waiting**: When frontend code needs to wait for user authentication, use `redux.getStore("account").async_wait({ until: () => store.get_account_id() != null, timeout: 0 })` to wait indefinitely until authentication completes
+- **Conat DKV Usage**: For key-value storage with real-time sync, use `webapp_client.conat_client.dkv({ account_id, name: "store-name" })` to get a distributed key-value store that syncs across sessions
 
 #### Backend Architecture
 
@@ -81,6 +83,8 @@ CoCalc is organized as a monorepo with key packages:
 - **Conat System**: Container orchestration for compute servers
 - **Event-Driven Architecture**: Extensive use of EventEmitter patterns
 - **Microservice-like Packages**: Each package handles specific functionality
+- **Database Access**: Use `getPool()` from `@cocalc/database/pool` for direct database queries in hub/backend code. Example: `const pool = getPool(); const { rows } = await pool.query('SELECT * FROM table WHERE id = $1', [id]);`
+- **Hub Migration Functions**: Migration functions in hub should be designed to run once at startup, use batch processing with delays between batches to avoid database saturation
 
 #### Communication Patterns
 

src/packages/frontend/project/page/flyouts/store.ts

@@ -3,115 +3,104 @@
  *  License: MS-RSL – see LICENSE.md for details
  */
 
-import { merge, sortBy, throttle, uniq, xor } from "lodash";
+/**
+// To debug starred files in the browser console:
+c = cc.client.conat_client
+bm = await c.dkv({account_id: cc.client.account_id, name: 'bookmark-starred-files'})
+// Check all bookmark data
+console.log('All bookmarks:', bm.getAll())
+// Check specific project bookmarks
+console.log('Project bookmarks (get):', bm.get("[project_id]"))
+// Set starred files for a project
+bm.set(project_id, ['file1.txt', 'folder/file2.md'])
+// Listen to changes
+bm.on('change', (e) => console.log('Bookmark change:', e))
+ */
+
+import { sortBy, uniq } from "lodash";
 import { useState } from "react";
 import useAsyncEffect from "use-async-effect";
 
-import api from "@cocalc/frontend/client/api";
-import { STARRED_FILES } from "@cocalc/util/consts/bookmarks";
-import {
-  GetStarredBookmarks,
-  GetStarredBookmarksPayload,
-  SetStarredBookmarks,
-} from "@cocalc/util/types/bookmarks";
-import {
-  FlyoutActiveStarred,
-  getFlyoutActiveStarred,
-  storeFlyoutState,
-} from "./state";
+import { redux } from "@cocalc/frontend/app-framework";
+import { webapp_client } from "@cocalc/frontend/webapp-client";
+import { CONAT_BOOKMARKS_KEY } from "@cocalc/util/consts/bookmarks";
+import type { FlyoutActiveStarred } from "./state";
 
-// Additionally to local storage, we back the state of the starred files in the database.
-// Errors with the API are ignored, because we primarily rely on local storage.
-// The only really important situation to think of are when there is nothing in local storage but in the database,
-// or when there is
+// Starred files are now managed entirely through conat with in-memory state.
+// No local storage dependency - conat handles synchronization and persistence.
 export function useStarredFilesManager(project_id: string) {
-  const [starred, setStarred] = useState<FlyoutActiveStarred>(
-    getFlyoutActiveStarred(project_id),
-  );
+  const [starred, setStarred] = useState<FlyoutActiveStarred>([]);
+  const [bookmarks, setBookmarks] = useState<any>(null);
+  const [isInitialized, setIsInitialized] = useState(false);
 
-  // once after mounting this, we update the starred bookmarks (which merges with what we have) and then stores it
+  // Initialize conat bookmarks once on mount, waiting for authentication
   useAsyncEffect(async () => {
-    await updateStarred();
+    // Wait until account is authenticated
+    const store = redux.getStore("account");
+    await store.async_wait({
+      until: () => store.get_account_id() != null,
+      timeout: 0, // indefinite timeout
+    });
+    
+    await initializeConatBookmarks();
   }, []);
 
-  function setStarredLS(starred: string[]) {
-    setStarred(starred);
-    storeFlyoutState(project_id, "active", { starred: starred });
+  async function initializeConatBookmarks() {
+    try {
+      const conatBookmarks = await webapp_client.conat_client.dkv<string[]>({
+        account_id: webapp_client.client.account_id,
+        name: CONAT_BOOKMARKS_KEY,
+      });
+
+      setBookmarks(conatBookmarks);
+
+      // Listen for changes from other clients
+      conatBookmarks.on(
+        "change",
+        (changeEvent: { key: string; value?: string[]; prev?: string[] }) => {
+          if (changeEvent.key === project_id) {
+            const remoteStars = changeEvent.value || [];
+            setStarred(sortBy(uniq(remoteStars)));
+          }
+        },
+      );
+
+      // Load initial data from conat
+      const initialStars = conatBookmarks.get(project_id) || [];
+      if (Array.isArray(initialStars)) {
+        setStarred(sortBy(uniq(initialStars)));
+      }
+
+      setIsInitialized(true);
+    } catch (err) {
+      console.warn(`conat bookmark initialization warning -- ${err}`);
+      setIsInitialized(true); // Set initialized even on error to avoid infinite loading
+    }
   }
 
-  // TODO: there are also add/remove API endpoints, but for now we stick with set. Hardly worth optimizing.
   function setStarredPath(path: string, starState: boolean) {
+    if (!bookmarks || !isInitialized) {
+      console.warn("Conat bookmarks not yet initialized");
+      return;
+    }
+
     const next = starState
-      ? [...starred, path]
+      ? sortBy(uniq([...starred, path]))
       : starred.filter((p) => p !== path);
-    setStarredLS(next);
-    storeStarred(next);
-  }
 
-  async function storeStarred(stars: string[]) {
+    // Update local state immediately for responsive UI
+    setStarred(next);
+
+    // Store to conat (this will also trigger the change event for other clients)
     try {
-      const payload: SetStarredBookmarks = {
-        type: STARRED_FILES,
-        project_id,
-        stars,
-      };
-      await api("bookmarks/set", payload);
+      bookmarks.set(project_id, next);
     } catch (err) {
-      console.warn(`bookmark: warning -- ${err}`);
+      console.warn(`conat bookmark storage warning -- ${err}`);
+      // Revert local state on error
+      setStarred(starred);
     }
   }
 
-  // this is called once, when the flyout/tabs component is mounted
-  // throtteled, to usually take 1 sec from opening the panel to loading the stars
-  const updateStarred = throttle(
-    async () => {
-      try {
-        const payload: GetStarredBookmarksPayload = {
-          type: STARRED_FILES,
-          project_id,
-        };
-        const data: GetStarredBookmarks = await api("bookmarks/get", payload);
-
-        const { type, status } = data;
-
-        if (type !== STARRED_FILES) {
-          console.error(
-            `flyout/store/starred type must be ${STARRED_FILES} but we got`,
-            type,
-          );
-          return;
-        }
-
-        if (status === "success") {
-          const { stars } = data;
-          if (
-            Array.isArray(stars) &&
-            stars.every((x) => typeof x === "string")
-          ) {
-            stars.sort(); // sorted for the xor check below
-            const next = sortBy(uniq(merge(starred, stars)));
-            setStarredLS(next);
-            if (xor(stars, next).length > 0) {
-              // if there is a change (e.g. nothing in the database stored yet), store the stars
-              await storeStarred(next);
-            }
-          } else {
-            console.error("flyout/store/starred invalid payload", stars);
-          }
-        } else if (status === "error") {
-          const { error } = data;
-          console.error("flyout/store/starred error", error);
-        } else {
-          console.error("flyout/store/starred error: unknown status", status);
-        }
-      } catch (err) {
-        console.warn(`bookmark: warning -- ${err}`);
-      }
-    },
-    1000,
-    { trailing: true, leading: false },
-  );
-
   return {
     starred,
     setStarredPath,

src/packages/hub/hub.ts

@@ -11,6 +11,7 @@ import { callback } from "awaiting";
 import blocked from "blocked";
 import { spawn } from "child_process";
 import { program as commander, Option } from "commander";
+
 import basePath from "@cocalc/backend/base-path";
 import {
   pghost as DEFAULT_DB_HOST,
@@ -21,10 +22,18 @@ import { trimLogFileSize } from "@cocalc/backend/logger";
 import port from "@cocalc/backend/port";
 import { init_start_always_running_projects } from "@cocalc/database/postgres/always-running";
 import { load_server_settings_from_env } from "@cocalc/database/settings/server-settings";
+import {
+  initConatApi,
+  initConatChangefeedServer,
+  initConatPersist,
+  loadConatConfiguration,
+} from "@cocalc/server/conat";
+import { initConatServer } from "@cocalc/server/conat/socketio";
 import { init_passport } from "@cocalc/server/hub/auth";
 import { initialOnPremSetup } from "@cocalc/server/initial-onprem-setup";
 import initHandleMentions from "@cocalc/server/mentions/handle";
 import initMessageMaintenance from "@cocalc/server/messages/maintenance";
+import { start as startHubRegister } from "@cocalc/server/metrics/hub_register";
 import initProjectControl, {
   COCALC_MODES,
 } from "@cocalc/server/projects/control";
@@ -35,22 +44,15 @@ import initSalesloftMaintenance from "@cocalc/server/salesloft/init";
 import { stripe_sync } from "@cocalc/server/stripe/sync";
 import { callback2, retry_until_success } from "@cocalc/util/async-utils";
 import { set_agent_endpoint } from "./health-checks";
-import { start as startHubRegister } from "@cocalc/server/metrics/hub_register";
 import { getLogger } from "./logger";
 import initDatabase, { database } from "./servers/database";
 import initExpressApp from "./servers/express-app";
-import {
-  loadConatConfiguration,
-  initConatChangefeedServer,
-  initConatApi,
-  initConatPersist,
-} from "@cocalc/server/conat";
-import { initConatServer } from "@cocalc/server/conat/socketio";
 
 import initHttpRedirect from "./servers/http-redirect";
 
-import * as MetricsRecorder from "@cocalc/server/metrics/metrics-recorder";
 import { addErrorListeners } from "@cocalc/server/metrics/error-listener";
+import * as MetricsRecorder from "@cocalc/server/metrics/metrics-recorder";
+import { migrateBookmarksToConat } from "./migrate-bookmarks";
 
 // Logger tagged with 'hub' for this file.
 const logger = getLogger("hub");
@@ -291,6 +293,10 @@ async function startServer(): Promise<void> {
     // upgrades of projects.
     initPurchasesMaintenanceLoop();
     initSalesloftMaintenance();
+    // Migrate bookmarks from database to conat (runs once at startup)
+    migrateBookmarksToConat().catch((err) => {
+      logger.error("Failed to migrate bookmarks to conat:", err);
+    });
     setInterval(trimLogFileSize, 1000 * 60 * 3);
   }
 

src/packages/hub/migrate-bookmarks.ts

@@ -0,0 +1,111 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2024 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+import { delay } from "awaiting";
+
+import { conat } from "@cocalc/backend/conat/conat";
+import { dkv } from "@cocalc/backend/conat/sync";
+import {
+  CONAT_BOOKMARKS_KEY,
+  STARRED_FILES,
+} from "@cocalc/util/consts/bookmarks";
+import { getLogger } from "./logger";
+import getPool from "@cocalc/database/pool";
+
+const L = getLogger("hub:migrate-bookmarks");
+
+const BATCH_SIZE = 100;
+const MIGRATION_DELAY = 500; // ms between batches to avoid database saturation
+
+export async function migrateBookmarksToConat(): Promise<void> {
+  L.info("Starting migration of bookmarks to conat...");
+
+  const pool = getPool();
+  let totalMigrated = 0;
+  let batchCount = 0;
+
+  while (true) {
+    try {
+      // Query for a batch of bookmark entries
+      const { rows } = await pool.query(
+        `
+          SELECT id, account_id, project_id, stars
+          FROM bookmarks
+          WHERE type = $1
+          ORDER BY id
+          LIMIT $2
+        `,
+        [STARRED_FILES, BATCH_SIZE],
+      );
+
+      if (rows.length === 0) {
+        L.info(
+          `Migration completed. Total migrated: ${totalMigrated} bookmarks`,
+        );
+        break;
+      }
+
+      batchCount++;
+      L.info(`Processing batch ${batchCount} with ${rows.length} bookmarks...`);
+
+      // Process each bookmark in the batch
+      const processedIds: string[] = [];
+
+      for (const row of rows) {
+        try {
+          const { id, account_id, project_id, stars } = row;
+
+          if (!account_id || !project_id || !Array.isArray(stars)) {
+            L.warn(
+              `Skipping invalid bookmark ${id}: account_id=${account_id}, project_id=${project_id}, stars=${stars}`,
+            );
+            processedIds.push(id);
+            continue;
+          }
+
+          // Get or create conat DKV for this account
+          const bookmarks = await dkv<string[]>({
+            name: CONAT_BOOKMARKS_KEY,
+            account_id,
+            client: conat(),
+          });
+
+          // Set the starred files for this project
+          bookmarks.set(project_id, stars);
+          L.debug(
+            `Migrated bookmark ${id} for account ${account_id}, project ${project_id} with ${stars.length} stars`,
+          );
+
+          processedIds.push(id);
+          totalMigrated++;
+        } catch (err) {
+          L.error(`Failed to migrate bookmark ${row.id}: ${err}`);
+          // Still add to processedIds so we don't get stuck on this one
+          processedIds.push(row.id);
+        }
+      }
+
+      // Delete the processed bookmarks from the database
+      if (processedIds.length > 0) {
+        await pool.query(`DELETE FROM bookmarks WHERE id = ANY($1)`, [
+          processedIds,
+        ]);
+        L.debug(`Deleted ${processedIds.length} bookmarks from database`);
+      }
+
+      // Wait between batches to avoid database saturation
+      if (rows.length === BATCH_SIZE) {
+        L.debug(`Waiting ${MIGRATION_DELAY}ms before next batch...`);
+        await delay(MIGRATION_DELAY);
+      }
+    } catch (err) {
+      L.error(`Error in migration batch: ${err}`);
+      // Wait longer on error before retrying
+      await delay(MIGRATION_DELAY * 3);
+    }
+  }
+
+  L.info("Bookmark migration to conat completed successfully");
+}