Merge branch 'master' into news-expiration-date-8485

Author: williamstein

src/.claude/settings.json

@@ -1,36 +1,39 @@
 {
   "permissions": {
     "allow": [
+      "Bash(../node_modules/.bin/tsc:*)",
+      "Bash(NODE_OPTIONS=--max-old-space-size=8192 ../node_modules/.bin/tsc --noEmit)",
+      "Bash(curl:*)",
+      "Bash(docker run:*)",
       "Bash(find:*)",
       "Bash(gh pr view:*)",
       "Bash(gh:*)",
       "Bash(git add:*)",
+      "Bash(git checkout:*)",
       "Bash(git commit:*)",
+      "Bash(git push:*)",
       "Bash(grep:*)",
       "Bash(node:*)",
       "Bash(npm show:*)",
+      "Bash(npm view:*)",
       "Bash(npx tsc:*)",
       "Bash(pnpm build:*)",
+      "Bash(pnpm i18n:*)",
       "Bash(pnpm ts-build:*)",
       "Bash(pnpm tsc:*)",
       "Bash(prettier -w:*)",
+      "Bash(psql:*)",
       "WebFetch(domain:cocalc.com)",
       "WebFetch(domain:doc.cocalc.com)",
       "WebFetch(domain:docs.anthropic.com)",
       "WebFetch(domain:github.com)",
-      "Bash(git checkout:*)",
-      "Bash(git push:*)",
-      "Bash(NODE_OPTIONS=--max-old-space-size=8192 ../node_modules/.bin/tsc --noEmit)",
-      "Bash(docker run:*)",
-      "Bash(../node_modules/.bin/tsc:*)",
-      "Bash(npm view:*)",
-      "WebFetch(domain:www.anthropic.com)",
       "WebFetch(domain:mistral.ai)",
-      "Bash(pnpm i18n:*)",
       "WebFetch(domain:simplelocalize.io)",
       "Bash(pnpm list:*)",
       "Bash(pnpm why:*)",
-      "mcp__github__get_issue"
+      "mcp__github__get_issue",
+      "WebFetch(domain:www.anthropic.com)",
+      "mcp__cclsp__find_definition"
     ],
     "deny": []
   }

src/CLAUDE.md

@@ -73,6 +73,8 @@ CoCalc is organized as a monorepo with key packages:
 - **TypeScript React Components**: All frontend code is TypeScript with proper typing
 - **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.)
 - **WebSocket Communication**: Real-time communication with backend via WebSocket messages
+- **Authentication Waiting**: When frontend code needs to wait for user authentication, use `redux.getStore("account").async_wait({ until: () => store.get_account_id() != null, timeout: 0 })` to wait indefinitely until authentication completes
+- **Conat DKV Usage**: For key-value storage with real-time sync, use `webapp_client.conat_client.dkv({ account_id, name: "store-name" })` to get a distributed key-value store that syncs across sessions
 
 #### Backend Architecture
 
@@ -81,6 +83,8 @@ CoCalc is organized as a monorepo with key packages:
 - **Conat System**: Container orchestration for compute servers
 - **Event-Driven Architecture**: Extensive use of EventEmitter patterns
 - **Microservice-like Packages**: Each package handles specific functionality
+- **Database Access**: Use `getPool()` from `@cocalc/database/pool` for direct database queries in hub/backend code. Example: `const pool = getPool(); const { rows } = await pool.query('SELECT * FROM table WHERE id = $1', [id]);`
+- **Hub Migration Functions**: Migration functions in hub should be designed to run once at startup, use batch processing with delays between batches to avoid database saturation
 
 #### Communication Patterns
 

src/packages/conat/service/typed.ts

@@ -9,7 +9,7 @@ import type { Options, ServiceCall } from "./service";
 export type { ConatService };
 
 export interface Extra {
-  ping: typeof pingConatService;
+  ping: (opts?: { maxWait?: number }) => Promise<void>;
   waitFor: (opts?: { maxWait?: number }) => Promise<void>;
 }
 

src/packages/frontend/cspell.json

@@ -70,7 +70,11 @@
     "flyouts",
     "buttonbar",
     "noconf",
-    "flyoutdragbar"
+    "flyoutdragbar",
+    "ptree",
+    "pchildren",
+    "nprocs",
+    "pids"
   ],
   "flagWords": [],
   "ignorePaths": ["node_modules/**", "dist/**", "dist-ts/**", "build/**"],

src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts

@@ -12,6 +12,8 @@ import {
 } from "@cocalc/conat/service/terminal";
 import { until } from "@cocalc/util/async-utils";
 
+const DEFAULT_HEARTBEAT_INTERVAL = 15_000;
+
 type State = "disconnected" | "init" | "running" | "closed";
 
 export class ConatTerminal extends EventEmitter {
@@ -29,6 +31,7 @@ export class ConatTerminal extends EventEmitter {
   private writeQueue: string = "";
   private ephemeral?: boolean;
   private computeServers?;
+  private heartbeatInterval: number;
 
   constructor({
     project_id,
@@ -40,6 +43,7 @@ export class ConatTerminal extends EventEmitter {
     options,
     measureSize,
     ephemeral,
+    heartbeatInterval = DEFAULT_HEARTBEAT_INTERVAL,
   }: {
     project_id: string;
     path: string;
@@ -50,6 +54,7 @@ export class ConatTerminal extends EventEmitter {
     options?;
     measureSize?;
     ephemeral?: boolean;
+    heartbeatInterval?: number;
   }) {
     super();
     this.ephemeral = ephemeral;
@@ -63,13 +68,35 @@ export class ConatTerminal extends EventEmitter {
     this.terminalResize = terminalResize;
     this.openPaths = openPaths;
     this.closePaths = closePaths;
+    this.heartbeatInterval = heartbeatInterval;
     webapp_client.conat_client.on("connected", this.clearWriteQueue);
     this.computeServers = webapp_client.project_client.computeServers(
       this.project_id,
     );
     this.computeServers?.on("change", this.handleComputeServersChange);
   }
 
+  // ping server periodically -- if failure, closes conenction immediately
+  // so it can be fixed when user comes back, instead of having to react to
+  // a write failing (which also handles the same issue)
+  private heartbeat = reuseInFlight(async () => {
+    await until(
+      async () => {
+        if (this.isClosed()) {
+          return true;
+        }
+        try {
+          await this.api.conat.ping({ maxWait: 5000 });
+          return false;
+        } catch {
+          this.close();
+          return true;
+        }
+      },
+      { min: this.heartbeatInterval, max: this.heartbeatInterval },
+    );
+  });
+
   clearWriteQueue = () => {
     if (this.writeQueue) {
       this.write("");
@@ -147,6 +174,8 @@ export class ConatTerminal extends EventEmitter {
     }
   };
 
+  isClosed = () => this.state == "closed";
+
   close = () => {
     webapp_client.conat_client.removeListener(
       "connected",
@@ -239,6 +268,7 @@ export class ConatTerminal extends EventEmitter {
   init = reuseInFlight(async () => {
     await Promise.all([this.start(), this.getStream()]);
     await this.setReady();
+    this.heartbeat();
   });
 
   private handleStreamData = (data) => {

src/packages/frontend/project/context.tsx

@@ -32,6 +32,7 @@ export interface ProjectContextState {
   actions?: ProjectActions;
   active_project_tab?: string;
   compute_image: string | undefined;
+  contentSize: { width: number; height: number };
   enabledLLMs: LLMServicesAvailable;
   flipTabs: [number, React.Dispatch<React.SetStateAction<number>>];
   group?: UserGroup;
@@ -47,22 +48,15 @@ export interface ProjectContextState {
   onCoCalcDocker: boolean;
   project_id: string;
   project?: Project;
+  setContentSize: (size: { width: number; height: number }) => void;
   status: ProjectStatus;
 }
 
 export const emptyProjectContext = {
   actions: undefined,
   active_project_tab: undefined,
-  group: undefined,
-  project: undefined,
-  is_active: false,
-  project_id: "",
-  isRunning: undefined,
-  status: INIT_PROJECT_STATE,
-  hasInternet: undefined,
-  flipTabs: [0, () => {}],
-  onCoCalcCom: true,
-  onCoCalcDocker: false,
+  compute_image: undefined,
+  contentSize: { width: 0, height: 0 },
   enabledLLMs: {
     openai: false,
     google: false,
@@ -72,12 +66,22 @@ export const emptyProjectContext = {
     custom_openai: false,
     user: false,
   },
+  flipTabs: [0, () => {}],
+  group: undefined,
+  hasInternet: undefined,
+  is_active: false,
+  isRunning: undefined,
   mainWidthPx: 0,
   manageStarredFiles: {
     starred: [],
     setStarredPath: () => {},
   },
-  compute_image: undefined,
+  onCoCalcCom: true,
+  onCoCalcDocker: false,
+  project: undefined,
+  project_id: "",
+  setContentSize: () => {},
+  status: INIT_PROJECT_STATE,
 } as ProjectContextState;
 
 export const ProjectContext: Context<ProjectContextState> =
@@ -141,10 +145,13 @@ export function useProjectContextProvider({
     userDefinedLLM,
   ]);
 
+  const [contentSize, setContentSize] = useState({ width: 0, height: 0 });
+
   return {
     actions,
     active_project_tab,
     compute_image,
+    contentSize,
     enabledLLMs,
     flipTabs,
     group,
@@ -157,6 +164,7 @@ export function useProjectContextProvider({
     onCoCalcDocker,
     project_id,
     project,
+    setContentSize,
     status,
   };
 }