widgets: get sending binary buffers to work, so now the official file upload widget works!

Author: williamstein

src/packages/jupyter/kernel/kernel.ts

@@ -999,20 +999,32 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface {
     target_name,
     data,
     buffers64,
+    buffers,
   }: {
     msg_id: string;
     comm_id: string;
     target_name: string;
     data: any;
     buffers64?: string[];
+    buffers?: Buffer[];
   }): void {
     const dbg = this.dbg("send_comm_message_to_kernel");
     dbg({ msg_id, comm_id, target_name, data, buffers64 });
-    const buffers = buffers64?.map((x) => new Buffer(base64ToBuffer(x))) ?? [];
-    dbg(
-      "buffers lengths = ",
-      buffers.map((x) => x.byteLength),
-    );
+    if (buffers64 != null && buffers64.length > 0) {
+      buffers = buffers64?.map((x) => new Buffer(base64ToBuffer(x))) ?? [];
+      dbg(
+        "buffers lengths = ",
+        buffers.map((x) => x.byteLength),
+      );
+      if (this._actions?.syncdb.ipywidgets_state != null) {
+        this._actions.syncdb.ipywidgets_state.setModelBuffers(
+          comm_id,
+          data.buffer_paths,
+          buffers,
+          false,
+        );
+      }
+    }
 
     const message = {
       parent_header: {},

src/packages/jupyter/redux/project-actions.ts

@@ -1335,16 +1335,28 @@ export class JupyterActions extends JupyterActions0 {
       let data: any;
       if (type === "value") {
         const state = this.syncdb.ipywidgets_state.get_model_value(model_id);
-        data = { method: "update", state };
+        // Saving the buffers on change is critical since otherwise this breaks:
+        //  https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload
+        // Note that stupidly the buffer (e.g., image upload) gets sent to the kernel twice.
+        // But it does work robustly, and the kernel and nodejs server processes next to each
+        // other so this isn't so bad.
+        const { buffer_paths, buffers } =
+          this.syncdb.ipywidgets_state.getKnownBuffers(model_id);
+        data = { method: "update", state, buffer_paths };
         this.jupyter_kernel.send_comm_message_to_kernel({
           msg_id: misc.uuid(),
           target_name: "jupyter.widget",
           comm_id: model_id,
           data,
+          buffers,
         });
       } else if (type === "buffers") {
-        // TODO: we will implement this soon.  A good example where this is required
-        // is by the file upload widget:
+        // TODO: we MIGHT need implement this... but MAYBE NOT.  An example where this seems like it might be
+        // required is by the file upload widget, but actually that just uses the value type above, since
+        // we explicitly fill in the widgets there; also there is an explicit comm upload message that
+        // the widget sends out that updates the buffer, and in send_comm_message_to_kernel in jupyter/kernel/kernel.ts
+        // when processing that message, we saves those buffers and make sure they are set in the
+        // value case above (otherwise they would get removed).
         //    https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload
         // which creates a buffer from the content of the file, then sends it to the backend,
         // which sees a change and has to write that buffer to the kernel (here) so that

src/packages/jupyter/types/project-interface.ts

@@ -146,6 +146,8 @@ export interface JupyterKernelInterface extends EventEmitterInterface {
     comm_id: string;
     target_name: string;
     data: any;
+    buffers?: any[];
+    buffers64?: any[];
   }): void;
   get_connection_file(): string | undefined;
 

src/packages/sync/editor/generic/ipywidgets-state.ts

@@ -281,6 +281,39 @@ export class IpywidgetsState extends EventEmitter {
     return { buffers, buffer_paths };
   };
 
+  // This is used on the backend when syncing changes from project nodejs *to*
+  // the jupyter kernel.
+  getKnownBuffers = (model_id: string) => {
+    let value: iMap<string, string> | undefined = this.get(model_id, "buffers");
+    if (value == null) {
+      return { buffer_paths: [], buffers: [] };
+    }
+    // value is an array from JSON of paths array to array buffers:
+    const buffer_paths: string[][] = [];
+    const buffers: ArrayBuffer[] = [];
+    if (this.buffers[model_id] == null) {
+      this.buffers[model_id] = {};
+    }
+    const f = (path: string) => {
+      const hash = value?.get(path);
+      if (!hash) {
+        return;
+      }
+      const cur = this.buffers[model_id][path];
+      if (cur?.hash == hash) {
+        buffer_paths.push(JSON.parse(path));
+        buffers.push(cur.buffer);
+        return;
+      }
+    };
+    value
+      .keySeq()
+      .toJS()
+      .filter((path) => path.startsWith("["))
+      .map(f);
+    return { buffers, buffer_paths };
+  };
+
   private clientGetBuffer = async (model_id: string, path: string) => {
     // async get of the buffer efficiently via HTTP:
     if (this.client.ipywidgetsGetBuffer == null) {
@@ -307,7 +340,7 @@ export class IpywidgetsState extends EventEmitter {
   };
 
   // returns the sha1 hashes of the buffers
-  private set_model_buffers = (
+  setModelBuffers = (
     // model that buffers are associated to:
     model_id: string,
     // if given, these are buffers with given paths; if not given, we
@@ -317,7 +350,7 @@ export class IpywidgetsState extends EventEmitter {
     buffers: Buffer[],
     fire_change_event: boolean = true,
   ): string[] => {
-    const dbg = this.dbg("set_model_buffers");
+    const dbg = this.dbg("setModelBuffers");
     dbg("buffer_paths = ", buffer_paths);
 
     const data: { [path: string]: boolean } = {};
@@ -728,7 +761,7 @@ scat.x, scat.y = np.random.rand(2, 50)
     if (content.data.buffer_paths?.length > 0) {
       // Deal with binary buffers:
       dbg("setting binary buffers");
-      this.set_model_buffers(
+      this.setModelBuffers(
         model_id,
         content.data.buffer_paths,
         msg.buffers,
@@ -752,7 +785,7 @@ scat.x, scat.y = np.random.rand(2, 50)
         ) {
           // TODO
           dbg("custom message  -- there are BUFFERS -- saving them");
-          buffer_hashes = this.set_model_buffers(
+          buffer_hashes = this.setModelBuffers(
             model_id,
             undefined,
             buffers,