Merge branch 'master' into fs2

Author: williamstein

src/.claude/settings.local.json

@@ -0,0 +1,11 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(pnpm tsc:*)",
+      "Bash(pnpm build:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)"
+    ],
+    "deny": []
+  }
+}

src/CLAUDE.md

@@ -0,0 +1,130 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+# CoCalc Source Repository
+
+* This is the source code of CoCalc in a Git repository
+* It is a complex JavaScript/TypeScript SaaS application
+* CoCalc is organized as a monorepository (multi-packages) in the subdirectory "./packages"
+* The packages are managed as a pnpm workspace in "./packages/pnpm-workspace.yaml"
+
+## Code Style
+
+- Everything is written in TypeScript code
+- Indentation: 2-spaces
+- All .js and .ts files are formatted by the tool prettier
+- Add suitable types when you write code
+- Variable name styles are "camelCase" for local and "FOO_BAR" for global variables. If you edit older code not following these guidlines, adjust this rule to fit the files style.
+- Some older code is JavaScript or CoffeeScript, which will be translated to TypeScript
+- Use ES modules (import/export) syntax, not CommonJS (require)
+- Organize the list of imports in such a way: installed npm packages are on top, newline, then are imports from @cocalc's code base. Sorted alphabetically.
+
+## Development Commands
+
+### Essential Commands
+- `pnpm build-dev` - Build all packages for development
+- `pnpm clean` - Clean all node_modules and dist directories
+- `pnpm database` - Start PostgreSQL database server
+- `pnpm hub` - Start the main hub server
+- `pnpm psql` - Connect to the PostgreSQL database
+- `pnpm test` - Run full test suite
+- `pnpm test-parallel` - Run tests in parallel across packages
+- `pnpm depcheck` - Check for dependency issues
+
+### Package-Specific Commands
+- `cd packages/[package] && pnpm tsc` - Watch TypeScript compilation for a specific package
+- `cd packages/[package] && pnpm test` - Run tests for a specific package
+- `cd packages/[package] && pnpm build` - Build a specific package
+
+### Development Setup
+1. Start database: `pnpm database`
+2. Start hub: `pnpm hub`
+3. For TypeScript changes, run `pnpm tsc` in the relevant package directory
+
+## Architecture Overview
+
+### Package Structure
+CoCalc is organized as a monorepo with key packages:
+
+- **frontend** - React/TypeScript frontend application using Redux-style stores and actions
+- **backend** - Node.js backend services and utilities
+- **hub** - Main server orchestrating the entire system
+- **database** - PostgreSQL database layer with queries and schema
+- **util** - Shared utilities and types used across packages
+- **comm** - Communication layer including WebSocket types
+- **conat** - CoCalc's container/compute orchestration system
+- **sync** - Real-time synchronization system for collaborative editing
+- **project** - Project-level services and management
+- **static** - Static assets and build configuration
+- **next** - Next.js server components
+
+### Key Architectural Patterns
+
+#### Frontend Architecture
+- **Redux-style State Management**: Uses custom stores and actions pattern (see `packages/frontend/app-framework/actions-and-stores.ts`)
+- **TypeScript React Components**: All frontend code is TypeScript with proper typing
+- **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.)
+- **WebSocket Communication**: Real-time communication with backend via WebSocket messages
+
+#### Backend Architecture
+- **PostgreSQL Database**: Primary data store with sophisticated querying system
+- **WebSocket Messaging**: Real-time communication between frontend and backend
+- **Conat System**: Container orchestration for compute servers
+- **Event-Driven Architecture**: Extensive use of EventEmitter patterns
+- **Microservice-like Packages**: Each package handles specific functionality
+
+#### Communication Patterns
+- **WebSocket Messages**: Primary communication method (see `packages/comm/websocket/types.ts`)
+- **Database Queries**: Structured query system with typed interfaces
+- **Event Emitters**: Inter-service communication within backend
+- **REST-like APIs**: Some HTTP endpoints for specific operations
+
+### Key Technologies
+- **TypeScript**: Primary language for all new code
+- **React**: Frontend framework
+- **PostgreSQL**: Database
+- **Node.js**: Backend runtime
+- **WebSockets**: Real-time communication
+- **pnpm**: Package manager and workspace management
+- **Jest**: Testing framework
+- **SASS**: CSS preprocessing
+
+### Database Schema
+- Comprehensive schema in `packages/util/db-schema`
+- Query abstractions in `packages/database/postgres/`
+- Type-safe database operations with TypeScript interfaces
+
+### Testing
+- **Jest**: Primary testing framework
+- **ts-jest**: TypeScript support for Jest
+- **jsdom**: Browser environment simulation for frontend tests
+- Test files use `.test.ts` or `.spec.ts` extensions
+- Each package has its own jest.config.js
+
+### Import Patterns
+- Use absolute imports with `@cocalc/` prefix for cross-package imports
+- Example: `import { cmp } from "@cocalc/util/misc"`
+- Type imports: `import type { Foo } from "./bar"`
+- Destructure imports when possible
+
+### Development Workflow
+1. Changes to TypeScript require compilation (`pnpm tsc` in relevant package)
+2. Database must be running before starting hub
+3. Hub coordinates all services and should be restarted after changes
+4. Use `pnpm clean && pnpm build-dev` when switching branches or after major changes
+
+# Workflow
+- Be sure to typecheck when you're done making a series of code changes
+- Prefer running single tests, and not the whole test suite, for performance
+
+## Git Workflow
+
+- Prefix git commits with the package and general area. e.g. 'frontend/latex: ...' if it concerns latex editor changes in the packages/frontend/... code.
+- When pushing a new branch to Github, track it upstream. e.g. `git push --set-upstream origin feature-foo` for branch "feature-foo".
+
+# important-instruction-reminders
+- Do what has been asked; nothing more, nothing less.
+- NEVER create files unless they're absolutely necessary for achieving your goal.
+- ALWAYS prefer editing an existing file to creating a new one.
+- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.

src/packages/conat/core/server.ts

@@ -74,6 +74,7 @@ import { type SysConatServer, sysApiSubject, sysApi } from "./sys";
 import { forkedConatServer } from "./start-server";
 import { stickyChoice } from "./sticky";
 import { EventEmitter } from "events";
+import { Metrics } from "../types";
 
 const logger = getLogger("conat:core:server");
 
@@ -310,6 +311,10 @@ export class ConatServer extends EventEmitter {
     });
   };
 
+  public getUsage = (): Metrics => {
+    return this.usage.getMetrics();
+  };
+
   // this is for the Kubernetes health check -- I haven't
   // thought at all about what to do here, really.
   // Hopefully experience can teach us.
@@ -602,7 +607,9 @@ export class ConatServer extends EventEmitter {
       return;
     }
     if (!(await this.isAllowed({ user, subject, type: "sub" }))) {
-      const message = `permission denied subscribing to '${subject}' from ${JSON.stringify(user)}`;
+      const message = `permission denied subscribing to '${subject}' from ${JSON.stringify(
+        user,
+      )}`;
       this.log(message);
       throw new ConatError(message, {
         code: 403,
@@ -706,7 +713,9 @@ export class ConatServer extends EventEmitter {
     }
 
     if (!(await this.isAllowed({ user: from, subject, type: "pub" }))) {
-      const message = `permission denied publishing to '${subject}' from ${JSON.stringify(from)}`;
+      const message = `permission denied publishing to '${subject}' from ${JSON.stringify(
+        from,
+      )}`;
       this.log(message);
       throw new ConatError(message, {
         // this is the http code for permission denied, and having this
@@ -950,7 +959,9 @@ export class ConatServer extends EventEmitter {
           return;
         }
         if (!(await this.isAllowed({ user, subject, type: "pub" }))) {
-          const message = `permission denied waiting for interest in '${subject}' from ${JSON.stringify(user)}`;
+          const message = `permission denied waiting for interest in '${subject}' from ${JSON.stringify(
+            user,
+          )}`;
           this.log(message);
           respond({ error: message, code: 403 });
         }
@@ -1791,7 +1802,9 @@ export function updateSticky(update: StickyUpdate, sticky: Sticky): boolean {
 function getServerAddress(options: Options) {
   const port = options.port;
   const path = options.path?.slice(0, -"/conat".length) ?? "";
-  return `http${options.ssl || port == 443 ? "s" : ""}://${options.clusterIpAddress ?? "localhost"}:${port}${path}`;
+  return `http${options.ssl || port == 443 ? "s" : ""}://${
+    options.clusterIpAddress ?? "localhost"
+  }:${port}${path}`;
 }
 
 /*

src/packages/conat/monitor/usage.ts

@@ -1,8 +1,10 @@
-import json from "json-stable-stringify";
 import { EventEmitter } from "events";
-import type { JSONValue } from "@cocalc/util/types";
-import { ConatError } from "@cocalc/conat/core/client";
+import json from "json-stable-stringify";
+
 import { getLogger } from "@cocalc/conat/client";
+import { ConatError } from "@cocalc/conat/core/client";
+import type { JSONValue } from "@cocalc/util/types";
+import { Metrics } from "../types";
 
 const logger = getLogger("monitor:usage");
 
@@ -17,6 +19,9 @@ export class UsageMonitor extends EventEmitter {
   private options: Options;
   private total = 0;
   private perUser: { [user: string]: number } = {};
+  // metrics will be picked up periodically and exposed via e.g. prometheus
+  private countDeny = 0;
+  private metrics: Metrics = {};
 
   constructor(options: Options) {
     super();
@@ -38,27 +43,53 @@ export class UsageMonitor extends EventEmitter {
 
   private initLogging = () => {
     const { log } = this.options;
-    if (log == null) {
-      return;
-    }
+
+    // Record metrics for all events (even if logging is disabled)
     this.on("total", (total, limit) => {
-      log("usage", this.options.resource, { total, limit });
+      this.metrics["total:count"] = total;
+      this.metrics["total:limit"] = limit;
+      if (log) {
+        log("usage", this.options.resource, { total, limit });
+      }
     });
     this.on("add", (user, count, limit) => {
-      log("usage", this.options.resource, "add", { user, count, limit });
+      // this.metrics["add:count"] = count;
+      // this.metrics["add:limit"] = limit;
+      if (log) {
+        log("usage", this.options.resource, "add", { user, count, limit });
+      }
     });
     this.on("delete", (user, count, limit) => {
-      log("usage", this.options.resource, "delete", { user, count, limit });
+      // this.metrics["delete:count"] = count;
+      // this.metrics["delete:limit"] = limit;
+      if (log) {
+        log("usage", this.options.resource, "delete", { user, count, limit });
+      }
     });
     this.on("deny", (user, limit, type) => {
-      log("usage", this.options.resource, "not allowed due to hitting limit", {
-        type,
-        user,
-        limit,
-      });
+      this.countDeny += 1;
+      this.metrics["deny:count"] = this.countDeny;
+      this.metrics["deny:limit"] = limit;
+      if (log) {
+        log(
+          "usage",
+          this.options.resource,
+          "not allowed due to hitting limit",
+          {
+            type,
+            user,
+            limit,
+          },
+        );
+      }
     });
   };
 
+  // we return a copy
+  getMetrics = () => {
+    return { ...this.metrics };
+  };
+
   add = (user: JSONValue) => {
     const u = this.toJson(user);
     let count = this.perUser[u] ?? 0;

src/packages/conat/types.ts

@@ -9,3 +9,8 @@ export interface Location {
 
   path?: string;
 }
+
+type EventType = "total" | "add" | "delete" | "deny";
+type ValueType = "count" | "limit";
+type MetricKey = `${EventType}:${ValueType}`;
+export type Metrics = { [K in MetricKey]?: number };

src/packages/file-server/btrfs/subvolume.ts

@@ -159,7 +159,8 @@ export class Subvolume {
     await sudo({ command: "chmod", args: ["a-w", this.snapshotsDir] });
   };
 
-  createSnapshot = async (name: string) => {
+  createSnapshot = async (name?: string) => {
+    name ??= new Date().toISOString();
     logger.debug("createSnapshot", { name, subvolume: this.name });
     await this.makeSnapshotsDir();
     await sudo({
@@ -293,17 +294,26 @@ export class Subvolume {
   };
 
   bupRestore = async (path: string) => {
+    // path -- branch/revision/path/to/dir
+    if (path.startsWith("/")) {
+      path = path.slice(1);
+    }
     path = normalize(path);
-    // outdir -- path relative to subvolume
-    // path -- /branch/revision/path/to/dir
+    // ... but to avoid potential data loss, we make a snapshot before deleting it.
+    await this.createSnapshot();
+    const i = path.indexOf("/"); // remove the commit name
+    await sudo({
+      command: "rm",
+      args: ["-rf", join(this.path, path.slice(i + 1))],
+    });
     await sudo({
       command: "bup",
       args: [
         "-d",
         this.filesystem.bup,
         "restore",
         "-C",
-        this.path, //join(this.path, outdir),
+        this.path,
         join(`/${this.name}`, path),
         "--quiet",
       ],

src/packages/file-server/btrfs/test/setup.ts

@@ -7,14 +7,22 @@ import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "path";
 import { until } from "@cocalc/util/async-utils";
-export { sudo } from "../util";
+import { sudo } from "../util";
+export { sudo };
 export { delay } from "awaiting";
 
 export let fs: Filesystem;
 let tempDir;
 
+const TEMP_PREFIX = "cocalc-test-btrfs-";
+
 export async function before() {
-  tempDir = await mkdtemp(join(tmpdir(), "cocalc-test-btrfs-"));
+  try {
+    const command = `umount ${join(tmpdir(), TEMP_PREFIX)}*/mnt`;
+    // attempt to unmount any mounts left from previous runs
+    await sudo({ command, bash: true });
+  } catch {}
+  tempDir = await mkdtemp(join(tmpdir(), TEMP_PREFIX));
   // Set world read/write/execute
   await chmod(tempDir, 0o777);
   const mount = join(tempDir, "mnt");