widgets: work in progress

Author: williamstein

src/packages/frontend/jupyter/widgets/k3d/README.md

@@ -1,6 +1,6 @@
 The contents of this directory is just a rewrite of https://github.com/K3D-tools/K3D-jupyter/blob/main/js/src/k3d.js using Typescript and more modularity, and also using ES6 classes which is more "future proof" regarding Backbone.js and newer versions of Ipywidgets.
 
-The license is all code in this directory is the same as for upstream, namely the MIT license as given here:
+The license for all code in this directory is the same as for upstream, namely the MIT license as given here:
 
 https://github.com/K3D-tools/K3D-jupyter/blob/main/LICENSE.txt
 

src/packages/frontend/jupyter/widgets/manager.ts

@@ -16,7 +16,7 @@ import * as react_output from "./output";
 import * as react_controls from "./controls";
 import { size } from "lodash";
 import { delay } from "awaiting";
-//import * as k3d from "./k3d";
+import * as k3d from "./k3d";
 
 /*
 NOTES: Third party custom widgets:
@@ -590,9 +590,9 @@ export class WidgetManager extends base.ManagerBase<HTMLElement> {
     } else if (moduleName === "@jupyter-widgets/output") {
       module = react_output;
     } else if (moduleName === "k3d") {
-      // NOTE: I completely rewrote the entire k3d widget interface...
-      //module = k3d;
-      throw Error("k3d disabled");
+      // NOTE: I completely rewrote the entire k3d widget interface, since
+      // it made tons of assumptions that break RTC.
+      module = k3d;
     } else if (moduleName === "jupyter-matplotlib") {
       //module = await import("jupyter-matplotlib");
       throw Error(`custom widgets: ${moduleName} not installed`);

src/packages/frontend/jupyter/widgets/manager2.ts

@@ -17,7 +17,7 @@ import {
   ModelState,
 } from "@cocalc/sync/editor/generic/ipywidgets-state";
 import { once } from "@cocalc/util/async-utils";
-import { copy, is_array, is_object, len, uuid } from "@cocalc/util/misc";
+import { is_array, is_object, len, uuid } from "@cocalc/util/misc";
 import { fromJS } from "immutable";
 import { CellOutputMessage } from "@cocalc/frontend/jupyter/output-messages/message";
 import React from "react";
@@ -116,16 +116,17 @@ export class WidgetManager {
     if (state == null) {
       return;
     }
-    await this.updateModel(model_id, state!);
+    await this.updateModel(model_id, state!, false);
   };
 
   private updateModel = async (
     model_id: string,
     changed: ModelState,
+    merge: boolean,
   ): Promise<void> => {
     const model: base.DOMWidgetModel | undefined =
       await this.manager.get_model(model_id);
-    log("updateModel", { model, changed });
+    log("updateModel", { model_id, merge, changed });
     if (model == null) {
       return;
     }
@@ -140,17 +141,33 @@ export class WidgetManager {
       );
       return;
     }
-    const state = await this.deserializeState(model, changed);
-    if (state.hasOwnProperty("outputs") && state["outputs"] == null) {
+    changed = await this.deserializeState(model, changed);
+    if (changed.hasOwnProperty("outputs") && changed["outputs"] == null) {
       // It can definitely be 'undefined' but set, e.g., the 'out.clear_output()' example at
       // https://ipywidgets.readthedocs.io/en/latest/examples/Output%20Widget.html
       // causes this, which then totally breaks rendering (due to how the
       // upstream widget manager works).  This works around that.
-      state["outputs"] = [];
+      changed["outputs"] = [];
+    }
+    if (merge) {
+      const state = model.get_state(false);
+      const x: ModelState = {};
+      for (const k in changed) {
+        if (state[k] != null && is_object(state[k]) && is_object(changed[k])) {
+          x[k] = { ...state[k], ...changed[k] };
+        } else {
+          x[k] = changed[k];
+        }
+      }
+      changed = x;
+    }
+    log("updateModel -- doing set_state", { model_id, merge, changed });
+    try {
+      model.set(changed);
+    } catch (err) {
+      //window.z = { merge, model, model_id, changed };
+      console.error("saved to z", err);
     }
-
-    log("set_state", state);
-    model.set_state(state);
   };
 
   // ipywidgets_state_ValueChange is called when a value entry of the ipywidgets_state
@@ -192,7 +209,7 @@ export class WidgetManager {
     }
     this.state_lock.add(model_id);
     log("handleValueChange: got model and now making this change -- ", changed);
-    await this.updateModel(model_id, changed);
+    await this.updateModel(model_id, changed, true);
     const model = await this.manager.get_model(model_id);
     if (model != null) {
       await model.state_change;
@@ -239,20 +256,31 @@ export class WidgetManager {
     A simple example that uses buffers is this image one:
        https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#image
     */
-    const model = await this.manager.get_model(model_id);
     const { buffer_paths, buffers } =
       await this.ipywidgets_state.get_model_buffers(model_id);
     log("handleBuffersChange: ", { model_id, buffer_paths, buffers });
-    const deserialized_state = model.get_state(true);
+    if (buffer_paths.length == 0) {
+      return;
+    }
+    const state = this.ipywidgets_state.get_model_state(model_id);
+    if (state == null) {
+      return;
+    }
     const change: { [key: string]: any } = {};
     for (let i = 0; i < buffer_paths.length; i++) {
       const key = buffer_paths[i][0];
-      setInObject(deserialized_state[key], buffer_paths[i], buffers[i]);
-      change[key] = deserialized_state[key];
+      setInObject(state, buffer_paths[i], buffers[i]);
+      change[key] = state[key];
     }
-    log("handleBuffersChange: ", model_id, change);
+    log("handleBuffersChange: ", model_id, { change });
     if (len(change) > 0) {
-      model.set_state(change);
+      const model = await this.manager.get_model(model_id);
+      try {
+        model.set(change);
+      } catch (err) {
+        // window.y = { model_id, model, change, buffer_paths, buffers };
+        console.error("saved to y", err);
+      }
     }
   };
 
@@ -301,18 +329,20 @@ export class WidgetManager {
   // ipywidgets_state, so that it is gets sync'd to the backend
   // and any other clients.
   private handleModelChange = async (model): Promise<void> => {
-    log("handleModelChange", model);
     const { model_id } = model;
+    let changed = model.changed;
+    log("handleModelChange", model_id, changed);
     await model.state_change;
     if (this.state_lock.has(model_id)) {
-      // log("handleModelChange: ignoring change due to state lock");
+      log("handleModelChange: ignoring change due to state lock");
       return;
     }
-    const changed: any = copy(model.serialize(model.changed));
+    changed = model.serialize(changed);
     delete changed.children; // sometimes they are in there, but shouldn't be sync'ed.
     const { last_changed } = changed;
     delete changed.last_changed;
     if (len(changed) == 0) {
+      log("handleModelChange: nothing changed");
       return; // nothing
     }
     // increment sequence number.
@@ -543,6 +573,18 @@ class Environment implements WidgetEnvironment {
     this.manager = manager;
   }
 
+  async loadClass(
+    className: string,
+    moduleName: string,
+    _moduleVersion: string,
+  ): Promise<any> {
+    if (false && moduleName === "k3d") {
+      // NOTE: I completely rewrote the entire k3d widget interface...
+      console.log("using builtin k3d");
+      return await import("k3d")[className];
+    }
+  }
+
   async getModelState(model_id) {
     // log("getModelState", model_id);
     if (this.manager.ipywidgets_state.get_state() != "ready") {
@@ -556,6 +598,23 @@ class Environment implements WidgetEnvironment {
         state = this.manager.ipywidgets_state.get_model_state(model_id);
       }
     }
+    if (state == null) {
+      throw Error("bug");
+    }
+    if (state._model_module == "k3d" && state.type != null) {
+      while (!state?.type || !state?.id) {
+        log(
+          "getModelState",
+          model_id,
+          "k3d: waiting for state.type to be defined",
+        );
+        await once(this.manager.ipywidgets_state, "change");
+        state = this.manager.ipywidgets_state.get_model_state(model_id);
+      }
+    }
+    if (state == null) {
+      throw Error("bug");
+    }
     if (state.hasOwnProperty("outputs") && state["outputs"] == null) {
       // It can definitely be 'undefined' but set, e.g., the 'out.clear_output()' example at
       // https://ipywidgets.readthedocs.io/en/latest/examples/Output%20Widget.html
@@ -566,12 +625,15 @@ class Environment implements WidgetEnvironment {
     const { buffer_paths, buffers } =
       await this.manager.ipywidgets_state.get_model_buffers(model_id);
 
-    for (let i = 0; i < buffer_paths.length; i++) {
-      const buffer = buffers[i];
-      setInObject(state, buffer_paths[i], buffer);
+    if (buffers.length > 0) {
+      for (let i = 0; i < buffer_paths.length; i++) {
+        const buffer = buffers[i];
+        setInObject(state, buffer_paths[i], buffer);
+      }
     }
-
     setTimeout(() => this.manager.watchModel(model_id), 1);
+
+    log("getModelState", { model_id, state });
     return {
       modelName: state._model_name,
       modelModule: state._model_module,

src/packages/frontend/package.json

@@ -45,7 +45,7 @@
     "@cocalc/local-storage-lru": "^2.4.3",
     "@cocalc/sync": "workspace:*",
     "@cocalc/util": "workspace:*",
-    "@cocalc/widgets": "^1.1.0",
+    "@cocalc/widgets": "^1.1.1",
     "@cocalc/xpra-lz4": "^1.1.0",
     "@dnd-kit/core": "^6.0.7",
     "@dnd-kit/modifiers": "^6.0.1",

src/packages/pnpm-lock.yaml

@@ -14,6 +14,7 @@ import { Map as iMap } from "immutable";
 import {
   close,
   delete_null_fields,
+  is_object,
   len,
   auxFileToOriginal,
 } from "@cocalc/util/misc";
@@ -350,20 +351,28 @@ export class IpywidgetsState extends EventEmitter {
     fire_change_event: boolean = true,
     merge?: "none" | "shallow" | "deep",
   ): void => {
+    const dbg = this.dbg("set");
     const string_id = this.syncdoc.get_string_id();
     if (typeof data != "object") {
       throw Error("TypeError -- data must be a map");
     }
     let defaultMerge: "none" | "shallow" | "deep";
     if (type == "value") {
+      //defaultMerge = "shallow";
       // we manually do the shallow merge only on the data field.
-      const data0 = this.get_model_value(model_id);
-      if (data0 != null) {
+      const current = this.get_model_value(model_id);
+      dbg("value: before", { data, current });
+      if (current != null) {
         for (const k in data) {
-          data0[k] = data[k];
+          if (is_object(data[k]) && is_object(current[k])) {
+            current[k] = { ...current[k], ...data[k] };
+          } else {
+            current[k] = data[k];
+          }
         }
-        data = data0;
+        data = current;
       }
+      dbg("value -- after", { merged: data });
       defaultMerge = "none";
     } else if (type == "buffers") {
       // we keep around the buffers that were
@@ -646,8 +655,18 @@ export class IpywidgetsState extends EventEmitter {
         this.sendCustomMessage(model_id, message, false);
         break;
 
+      case "echo_update":
+        // just ignore echo_update -- it's a new ipywidgets 8 mechanism
+        // for some level of RTC sync between clients -- we don't need that
+        // since we have our own, obviously. Setting the env var
+        // JUPYTER_WIDGETS_ECHO to 0 will disable these messages to slightly
+        // reduce traffic.
+        return;
+
       case "update":
-        if (state == null) return;
+        if (state == null) {
+          return;
+        }
         dbg("method -- update");
         if (this.clear_output[model_id] && state.outputs != null) {
           // we are supposed to clear the output before inserting