Merge remote-tracking branch 'origin/master' into delete-project-data

Author: haraldschilly

src/compute/compute/lib/listings.ts

@@ -52,8 +52,8 @@ export async function initListings({
     project_id,
     compute_server_id,
     getListing,
-    createWatcher: (path: string, debounceMs: number) =>
-      new Watcher(path, debounceMs),
+    createWatcher: (path: string, debounce: number) =>
+      new Watcher(path, { debounce }),
     onDeletePath: (path) => {
       logger.debug("onDeletePath -- TODO:", { path });
     },

src/packages/api-client/README.md

@@ -10,7 +10,7 @@ the settings of your project.
 ```sh
 export PROJECT_ID=6640ddad-4bdd-4745-8e63-8db74686a20e
 export API_KEY=sk-FcnRs3NxsTZROgbF000001
-export API_SERVER=http://localhost:9001
+export API_SERVER=http://127.0.0.1:9001
 ```
 
 The run code in node from the shell in the current directory:

src/packages/backend/path-watcher.ts

@@ -4,6 +4,9 @@
  */
 
 /*
+Watch A DIRECTORY for changes.  Use ./watcher.ts for a single file.
+
+
 Slightly generalized fs.watch that works even when the directory doesn't exist,
 but also doesn't provide any information about what changed.
 
@@ -76,11 +79,18 @@ export class Watcher extends EventEmitter {
   private exists: boolean;
   private watchContents?: FSWatcher;
   private watchExistence?: FSWatcher;
+  private interval_ms: number;
   private debounce_ms: number;
   private debouncedChange: any;
   private log: Function;
 
-  constructor(path: string, debounce_ms: number) {
+  constructor(
+    path: string,
+    {
+      debounce: debounce_ms = 0,
+      interval: interval_ms,
+    }: { debounce?: number; interval?: number } = {},
+  ) {
     super();
     this.log = logger.extend(path).debug;
     this.log(`initializing: poll=${POLLING}`);
@@ -89,10 +99,13 @@ export class Watcher extends EventEmitter {
     }
     this.path = path.startsWith("/") ? path : join(process.env.HOME, path);
     this.debounce_ms = debounce_ms;
-    this.debouncedChange = debounce(this.change.bind(this), this.debounce_ms, {
-      leading: true,
-      trailing: true,
-    }).bind(this);
+    this.interval_ms = interval_ms ?? DEFAULT_POLL_MS;
+    this.debouncedChange = this.debounce_ms
+      ? debounce(this.change, this.debounce_ms, {
+          leading: true,
+          trailing: true,
+        }).bind(this)
+      : this.change;
     this.init();
   }
 
@@ -109,8 +122,16 @@ export class Watcher extends EventEmitter {
     }
   }
 
+  private chokidarOptions = () => {
+    return {
+      ...ChokidarOpts,
+      interval: this.interval_ms,
+      binaryInterval: this.interval_ms,
+    };
+  };
+
   private initWatchContents(): void {
-    this.watchContents = watch(this.path, ChokidarOpts);
+    this.watchContents = watch(this.path, this.chokidarOptions());
     this.watchContents.on("all", this.debouncedChange);
     this.watchContents.on("error", (err) => {
       this.log(`error watching listings -- ${err}`);
@@ -119,7 +140,7 @@ export class Watcher extends EventEmitter {
 
   private async initWatchExistence(): Promise<void> {
     const containing_path = path_split(this.path).head;
-    this.watchExistence = watch(containing_path, ChokidarOpts);
+    this.watchExistence = watch(containing_path, this.chokidarOptions());
     this.watchExistence.on("all", this.watchExistenceChange(containing_path));
     this.watchExistence.on("error", (err) => {
       this.log(`error watching for existence of ${this.path} -- ${err}`);
@@ -147,9 +168,9 @@ export class Watcher extends EventEmitter {
     }
   };
 
-  private change(): void {
+  private change = (): void => {
     this.emit("change");
-  }
+  };
 
   public close(): void {
     this.watchExistence?.close();

src/packages/backend/tcp/locked-socket.ts

@@ -67,7 +67,7 @@ It is not used in any other way.
 */
 export async function connectToLockedSocket({
   port,
-  host = "localhost",
+  host = "127.0.0.1",
   token,
   timeout = 5, // in seconds (not milliseconds)
 }: {

src/packages/backend/watcher.ts

@@ -4,78 +4,78 @@
  */
 
 /*
-Watch a file for changes
+Watch one SINGLE FILE for changes.   Use ./path-watcher.ts for a directory.
 
-Watch for changes to the given file.  Returns obj, which
+Watch for changes to the given file, which means the mtime changes or the
+mode changes (e.g., readonly versus readwrite).  Returns obj, which
 is an event emitter with events:
 
-   - 'change', ctime - when file changes or is created
+   - 'change', ctime, stats - when file changes or is created
    - 'delete' - when file is deleted
 
 and a method .close().
 
 Only fires after the file definitely has not had its
-ctime changed for at least debounce ms (this is the atomic
-option to chokidar).  Does NOT fire when the file first
-has ctime changed.
+ctime changed for at least debounce ms.  Does NOT
+fire when the file first has ctime changed.
+
+NOTE: for directories we use chokidar in path-watcher.  However,
+for a single file using polling, chokidar is horribly buggy and
+lacking in functionality (e.g., https://github.com/paulmillr/chokidar/issues/1132),
+and declared all bugs fixed, so we steer clear.  It had a lot of issues
+with just noticing actual file changes.
+
+We *always* use polling to fully support networked filesystems.
 */
 
 import { EventEmitter } from "node:events";
-import { watch, FSWatcher } from "chokidar";
+import { unwatchFile, watchFile } from "node:fs";
 import { getLogger } from "./logger";
 import { debounce as lodashDebounce } from "lodash";
 
-const L = getLogger("watcher");
+const logger = getLogger("backend:watcher");
 
 export class Watcher extends EventEmitter {
   private path: string;
-  private interval: number;
-  private watcher: FSWatcher;
 
-  constructor(path: string, interval: number = 300, debounce: number = 0) {
+  constructor(
+    path: string,
+    { debounce, interval = 300 }: { debounce?: number; interval?: number } = {},
+  ) {
     super();
     this.path = path;
-    this.interval = interval;
-
-    L.debug(`${path}: interval=${interval}, debounce=${debounce}`);
-    this.watcher = watch(this.path, {
-      interval: this.interval,
-      // polling is critical for network mounted file systems,
-      // and given architecture of cocalc there is no easy way around this.
-      // E.g., on compute servers, everything breaks involving sync or cloudfs,
-      // and in shared project s3/gcsfuse/sshfs would all break. So we
-      // use polling.
-      usePolling: true,
-      persistent: false,
-      alwaysStat: true,
-      atomic: true,
-    });
-    this.watcher.on("unlink", () => {
-      this.emit("delete");
-    });
-    this.watcher.on("unlinkDir", () => {
-      this.emit("delete");
-    });
-
-    const emitChange = lodashDebounce(
-      (ctime) => this.emit("change", ctime),
-      debounce,
-    );
-    this.watcher.on("error", (err) => {
-      L.debug("WATCHER error -- ", err);
-    });
-
-    this.watcher.on("change", (_, stats) => {
-      if (stats == null) {
-        L.debug("WATCHER change with no stats (shouldn't happen)", { path });
-        return;
-      }
-      emitChange(stats.ctime);
-    });
+
+    logger.debug("watchFile", { path, debounce, interval });
+    watchFile(this.path, { persistent: false, interval }, this.handleChange);
+
+    if (debounce) {
+      this.emitChange = lodashDebounce(this.emitChange, debounce);
+    }
   }
 
-  close = async () => {
+  private emitChange = (stats) => {
+    this.emit("change", stats.ctime, stats);
+  };
+
+  private handleChange = (curr, prev) => {
+    const path = this.path;
+    if (!curr.dev) {
+      logger.debug("handleChange: delete", { path });
+      this.emit("delete");
+      return;
+    }
+    if (curr.mtimeMs == prev.mtimeMs && curr.mode == prev.mode) {
+      logger.debug("handleChange: access but no change", { path });
+      // just *accessing* triggers watchFile (really StatWatcher), of course.
+      return;
+    }
+    logger.debug("handleChange: change", { path });
+    this.emitChange(curr);
+  };
+
+  close = () => {
+    logger.debug("close", this.path);
     this.removeAllListeners();
-    await this.watcher.close();
+    unwatchFile(this.path, this.handleChange);
   };
 }

src/packages/frontend/account/editor-settings/checkboxes.tsx

@@ -30,6 +30,8 @@ const EDITOR_SETTINGS_CHECKBOXES: { [setting: string]: string | Rendered } = {
   build_on_save: "build LaTex/Rmd files whenever it is saved to disk",
   show_exec_warning: "warn that certain files are not directly executable",
   ask_jupyter_kernel: "ask which kernel to use for a new Jupyter Notebook",
+  disable_jupyter_virtualization:
+    "render entire notebook instead of just visible part (slower and not recommended)",
   jupyter_classic: (
     <span>
       <A href="https://github.com/sagemathinc/cocalc/issues/7706">

src/packages/frontend/account/other-settings.tsx

@@ -402,7 +402,7 @@ export function OtherSettings(props: Readonly<Props>): JSX.Element {
 
   function render_language_model(): Rendered {
     return (
-      <LabeledRow label={<>Default Language Model</>}>
+      <LabeledRow label={<>Default AI Language Model</>}>
         <LLMSelector model={model} setModel={setModel} />
       </LabeledRow>
     );

src/packages/frontend/antd-bootstrap.tsx

@@ -35,9 +35,10 @@ import {
   Tooltip,
 } from "antd";
 import { MouseEventHandler } from "react";
+
 import { inDarkMode } from "@cocalc/frontend/account/dark-mode";
-import { r_join } from "@cocalc/frontend/components/r_join";
 import { Gap } from "@cocalc/frontend/components/gap";
+import { r_join } from "@cocalc/frontend/components/r_join";
 import { COLORS } from "@cocalc/util/theme";
 
 // Note regarding buttons -- there are 6 semantics meanings in bootstrap, but