bookmarks: migrate to conat

src/.claude/settings.json

@@ -1,32 +1,35 @@
 {
   "permissions": {
     "allow": [
+      "Bash(../node_modules/.bin/tsc:*)",
+      "Bash(NODE_OPTIONS=--max-old-space-size=8192 ../node_modules/.bin/tsc --noEmit)",
+      "Bash(curl:*)",
+      "Bash(docker run:*)",
       "Bash(find:*)",
       "Bash(gh pr view:*)",
       "Bash(gh:*)",
       "Bash(git add:*)",
+      "Bash(git checkout:*)",
       "Bash(git commit:*)",
+      "Bash(git push:*)",
       "Bash(grep:*)",
       "Bash(node:*)",
       "Bash(npm show:*)",
+      "Bash(npm view:*)",
       "Bash(npx tsc:*)",
       "Bash(pnpm build:*)",
+      "Bash(pnpm i18n:*)",
       "Bash(pnpm ts-build:*)",
       "Bash(pnpm tsc:*)",
       "Bash(prettier -w:*)",
+      "Bash(psql:*)",
       "WebFetch(domain:cocalc.com)",
       "WebFetch(domain:doc.cocalc.com)",
       "WebFetch(domain:docs.anthropic.com)",
       "WebFetch(domain:github.com)",
-      "Bash(git checkout:*)",
-      "Bash(git push:*)",
-      "Bash(NODE_OPTIONS=--max-old-space-size=8192 ../node_modules/.bin/tsc --noEmit)",
-      "Bash(docker run:*)",
-      "Bash(../node_modules/.bin/tsc:*)",
-      "Bash(npm view:*)",
-      "WebFetch(domain:www.anthropic.com)",
       "WebFetch(domain:mistral.ai)",
-      "Bash(pnpm i18n:*)",
+      "WebFetch(domain:www.anthropic.com)",
+      "mcp__cclsp__find_definition",
       "WebFetch(domain:simplelocalize.io)"
     ],
     "deny": []

src/CLAUDE.md

@@ -73,6 +73,8 @@ CoCalc is organized as a monorepo with key packages:
 - **TypeScript React Components**: All frontend code is TypeScript with proper typing
 - **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.)
 - **WebSocket Communication**: Real-time communication with backend via WebSocket messages
+- **Authentication Waiting**: When frontend code needs to wait for user authentication, use `redux.getStore("account").async_wait({ until: () => store.get_account_id() != null, timeout: 0 })` to wait indefinitely until authentication completes
+- **Conat DKV Usage**: For key-value storage with real-time sync, use `webapp_client.conat_client.dkv({ account_id, name: "store-name" })` to get a distributed key-value store that syncs across sessions
 
 #### Backend Architecture
 
@@ -81,6 +83,8 @@ CoCalc is organized as a monorepo with key packages:
 - **Conat System**: Container orchestration for compute servers
 - **Event-Driven Architecture**: Extensive use of EventEmitter patterns
 - **Microservice-like Packages**: Each package handles specific functionality
+- **Database Access**: Use `getPool()` from `@cocalc/database/pool` for direct database queries in hub/backend code. Example: `const pool = getPool(); const { rows } = await pool.query('SELECT * FROM table WHERE id = $1', [id]);`
+- **Hub Migration Functions**: Migration functions in hub should be designed to run once at startup, use batch processing with delays between batches to avoid database saturation
 
 #### Communication Patterns
 

src/packages/frontend/project/page/flyouts/body.tsx

@@ -14,10 +14,10 @@ import {
 } from "@cocalc/frontend/app-framework";
 import { Loading } from "@cocalc/frontend/components";
 import * as LS from "@cocalc/frontend/misc/local-storage-typed";
-import { useProjectContext } from "../../context";
+import { useProjectContext } from "@cocalc/frontend/project/context";
+import { FIXED_TABS_BG_COLOR } from "../activity-bar-tabs";
 import { FIX_BORDER } from "../common";
 import { FIXED_PROJECT_TABS, FixedTab } from "../file-tab";
-import { FIXED_TABS_BG_COLOR } from "../activity-bar-tabs";
 import { FLYOUT_PADDING } from "./consts";
 import { LSFlyout, lsKey, storeFlyoutState } from "./state";
 
@@ -56,7 +56,7 @@ export function FlyoutBody({ flyout, flyoutWidth }: FlyoutBodyProps) {
     { leading: false, trailing: true },
   );
 
-  // use this *once* around a vertically scollable content div in the component, e.g. results in a search.
+  // use this *once* around a vertically scrollable content div in the component, e.g. results in a search.
   // note: this won't work if the initial content is something else, e.g. a loading indicator.
   function wrap(content: React.ReactNode, style: CSS = {}) {
     return (

src/packages/frontend/project/page/flyouts/store.ts

@@ -3,115 +3,105 @@
  *  License: MS-RSL – see LICENSE.md for details
  */
 
-import { merge, sortBy, throttle, uniq, xor } from "lodash";
+/**
+// To debug starred files in the browser console:
+c = cc.client.conat_client
+bm = await c.dkv({account_id: cc.client.account_id, name: 'bookmark-starred-files'})
+// Check all bookmark data
+console.log('All bookmarks:', bm.getAll())
+// Check specific project bookmarks
+console.log('Project bookmarks (get):', bm.get("[project_id]"))
+// Set starred files for a project
+bm.set(project_id, ['file1.txt', 'folder/file2.md'])
+// Listen to changes
+bm.on('change', (e) => console.log('Bookmark change:', e))
+ */
+
+import { sortBy, uniq } from "lodash";
 import { useState } from "react";
 import useAsyncEffect from "use-async-effect";
 
-import api from "@cocalc/frontend/client/api";
-import { STARRED_FILES } from "@cocalc/util/consts/bookmarks";
-import {
-  GetStarredBookmarks,
-  GetStarredBookmarksPayload,
-  SetStarredBookmarks,
-} from "@cocalc/util/types/bookmarks";
-import {
-  FlyoutActiveStarred,
-  getFlyoutActiveStarred,
-  storeFlyoutState,
-} from "./state";
-
-// Additionally to local storage, we back the state of the starred files in the database.
-// Errors with the API are ignored, because we primarily rely on local storage.
-// The only really important situation to think of are when there is nothing in local storage but in the database,
-// or when there is
+import { redux } from "@cocalc/frontend/app-framework";
+import { webapp_client } from "@cocalc/frontend/webapp-client";
+import { CONAT_BOOKMARKS_KEY } from "@cocalc/util/consts/bookmarks";
+import type { FlyoutActiveStarred } from "./state";
+
+// Starred files are now managed entirely through conat with in-memory state.
+// No local storage dependency - conat handles synchronization and persistence.
 export function useStarredFilesManager(project_id: string) {
-  const [starred, setStarred] = useState<FlyoutActiveStarred>(
-    getFlyoutActiveStarred(project_id),
-  );
+  const [starred, setStarred] = useState<FlyoutActiveStarred>([]);
+  const [bookmarks, setBookmarks] = useState<any>(null);
+  const [isInitialized, setIsInitialized] = useState(false);
 
-  // once after mounting this, we update the starred bookmarks (which merges with what we have) and then stores it
+  // Initialize conat bookmarks once on mount, waiting for authentication
   useAsyncEffect(async () => {
-    await updateStarred();
+    // Wait until account is authenticated
+    const store = redux.getStore("account");
+    await store.async_wait({
+      until: () => store.get_account_id() != null,
+      timeout: 0, // indefinite timeout
+    });
+
+    const account_id = store.get_account_id();
+    await initializeConatBookmarks(account_id);
   }, []);
 
-  function setStarredLS(starred: string[]) {
-    setStarred(starred);
-    storeFlyoutState(project_id, "active", { starred: starred });
+  async function initializeConatBookmarks(account_id: string) {
+    try {
+      const conatBookmarks = await webapp_client.conat_client.dkv<string[]>({
+        account_id,
+        name: CONAT_BOOKMARKS_KEY,
+      });
+
+      setBookmarks(conatBookmarks);
+
+      // Listen for changes from other clients
+      conatBookmarks.on(
+        "change",
+        (changeEvent: { key: string; value?: string[]; prev?: string[] }) => {
+          if (changeEvent.key === project_id) {
+            const remoteStars = changeEvent.value || [];
+            setStarred(sortBy(uniq(remoteStars)));
+          }
+        },
+      );
+
+      // Load initial data from conat
+      const initialStars = conatBookmarks.get(project_id) || [];
+      if (Array.isArray(initialStars)) {
+        setStarred(sortBy(uniq(initialStars)));
+      }
+
+      setIsInitialized(true);
+    } catch (err) {
+      console.warn(`conat bookmark initialization warning -- ${err}`);
+      setIsInitialized(true); // Set initialized even on error to avoid infinite loading
+    }
   }
 
-  // TODO: there are also add/remove API endpoints, but for now we stick with set. Hardly worth optimizing.
   function setStarredPath(path: string, starState: boolean) {
+    if (!bookmarks || !isInitialized) {
+      console.warn("Conat bookmarks not yet initialized");
+      return;
+    }
+
     const next = starState
-      ? [...starred, path]
+      ? sortBy(uniq([...starred, path]))
       : starred.filter((p) => p !== path);
-    setStarredLS(next);
-    storeStarred(next);
-  }
 
-  async function storeStarred(stars: string[]) {
+    // Update local state immediately for responsive UI
+    setStarred(next);
+
+    // Store to conat (this will also trigger the change event for other clients)
     try {
-      const payload: SetStarredBookmarks = {
-        type: STARRED_FILES,
-        project_id,
-        stars,
-      };
-      await api("bookmarks/set", payload);
+      bookmarks.set(project_id, next);
     } catch (err) {
-      console.warn(`bookmark: warning -- ${err}`);
+      console.warn(`conat bookmark storage warning -- ${err}`);
+      // Revert local state on error
+      setStarred(starred);
     }
   }
 
-  // this is called once, when the flyout/tabs component is mounted
-  // throtteled, to usually take 1 sec from opening the panel to loading the stars
-  const updateStarred = throttle(
-    async () => {
-      try {
-        const payload: GetStarredBookmarksPayload = {
-          type: STARRED_FILES,
-          project_id,
-        };
-        const data: GetStarredBookmarks = await api("bookmarks/get", payload);
-
-        const { type, status } = data;
-
-        if (type !== STARRED_FILES) {
-          console.error(
-            `flyout/store/starred type must be ${STARRED_FILES} but we got`,
-            type,
-          );
-          return;
-        }
-
-        if (status === "success") {
-          const { stars } = data;
-          if (
-            Array.isArray(stars) &&
-            stars.every((x) => typeof x === "string")
-          ) {
-            stars.sort(); // sorted for the xor check below
-            const next = sortBy(uniq(merge(starred, stars)));
-            setStarredLS(next);
-            if (xor(stars, next).length > 0) {
-              // if there is a change (e.g. nothing in the database stored yet), store the stars
-              await storeStarred(next);
-            }
-          } else {
-            console.error("flyout/store/starred invalid payload", stars);
-          }
-        } else if (status === "error") {
-          const { error } = data;
-          console.error("flyout/store/starred error", error);
-        } else {
-          console.error("flyout/store/starred error: unknown status", status);
-        }
-      } catch (err) {
-        console.warn(`bookmark: warning -- ${err}`);
-      }
-    },
-    1000,
-    { trailing: true, leading: false },
-  );
-
   return {
     starred,
     setStarredPath,

src/packages/frontend/project/page/home-page/button.tsx

@@ -5,11 +5,7 @@
 
 import { Button } from "antd";
 
-import {
-  redux,
-  useActions,
-  useTypedRedux,
-} from "@cocalc/frontend/app-framework";
+import { redux, useActions } from "@cocalc/frontend/app-framework";
 import { Icon } from "@cocalc/frontend/components";
 import { getValidActivityBarOption } from "@cocalc/frontend/project/page/activity-bar";
 import { ACTIVITY_BAR_KEY } from "@cocalc/frontend/project/page/activity-bar-consts";
@@ -18,8 +14,6 @@ import { COLORS } from "@cocalc/util/theme";
 
 export default function HomePageButton({ project_id, active, width }) {
   const actions = useActions({ project_id });
-  const hideActionButtons = useTypedRedux(project_id, "hideActionButtons");
-  if (hideActionButtons) return <></>;
 
   return (
     <Button

src/packages/frontend/project_actions.ts

@@ -3447,7 +3447,7 @@ export class ProjectActions extends Actions<ProjectStoreState> {
   }
 
   /* NOTE!  Below we store the modal state *both* in a private
-  variabel *and* in the store.  The reason is because we need
+  variable *and* in the store.  The reason is because we need
   to know it immediately after it is set in order for
   wait_until_no_modals to work robustless, and setState can
   wait before changing the state.

src/packages/hub/hub.ts

@@ -11,6 +11,7 @@ import { callback } from "awaiting";
 import blocked from "blocked";
 import { spawn } from "child_process";
 import { program as commander, Option } from "commander";
+
 import basePath from "@cocalc/backend/base-path";
 import {
   pghost as DEFAULT_DB_HOST,
@@ -21,10 +22,18 @@ import { trimLogFileSize } from "@cocalc/backend/logger";
 import port from "@cocalc/backend/port";
 import { init_start_always_running_projects } from "@cocalc/database/postgres/always-running";
 import { load_server_settings_from_env } from "@cocalc/database/settings/server-settings";
+import {
+  initConatApi,
+  initConatChangefeedServer,
+  initConatPersist,
+  loadConatConfiguration,
+} from "@cocalc/server/conat";
+import { initConatServer } from "@cocalc/server/conat/socketio";
 import { init_passport } from "@cocalc/server/hub/auth";
 import { initialOnPremSetup } from "@cocalc/server/initial-onprem-setup";
 import initHandleMentions from "@cocalc/server/mentions/handle";
 import initMessageMaintenance from "@cocalc/server/messages/maintenance";
+import { start as startHubRegister } from "@cocalc/server/metrics/hub_register";
 import initProjectControl, {
   COCALC_MODES,
 } from "@cocalc/server/projects/control";
@@ -35,22 +44,15 @@ import initSalesloftMaintenance from "@cocalc/server/salesloft/init";
 import { stripe_sync } from "@cocalc/server/stripe/sync";
 import { callback2, retry_until_success } from "@cocalc/util/async-utils";
 import { set_agent_endpoint } from "./health-checks";
-import { start as startHubRegister } from "@cocalc/server/metrics/hub_register";
 import { getLogger } from "./logger";
 import initDatabase, { database } from "./servers/database";
 import initExpressApp from "./servers/express-app";
-import {
-  loadConatConfiguration,
-  initConatChangefeedServer,
-  initConatApi,
-  initConatPersist,
-} from "@cocalc/server/conat";
-import { initConatServer } from "@cocalc/server/conat/socketio";
 
 import initHttpRedirect from "./servers/http-redirect";
 
-import * as MetricsRecorder from "@cocalc/server/metrics/metrics-recorder";
 import { addErrorListeners } from "@cocalc/server/metrics/error-listener";
+import * as MetricsRecorder from "@cocalc/server/metrics/metrics-recorder";
+import { migrateBookmarksToConat } from "./migrate-bookmarks";
 
 // Logger tagged with 'hub' for this file.
 const logger = getLogger("hub");
@@ -291,6 +293,10 @@ async function startServer(): Promise<void> {
     // upgrades of projects.
     initPurchasesMaintenanceLoop();
     initSalesloftMaintenance();
+    // Migrate bookmarks from database to conat (runs once at startup)
+    migrateBookmarksToConat().catch((err) => {
+      logger.error("Failed to migrate bookmarks to conat:", err);
+    });
     setInterval(trimLogFileSize, 1000 * 60 * 3);
   }