widgets: first pass at comm message that sets binary buffers

- with file upload this works, then is immediately overwritten by empty
  data, thus breaking it.  So not done yet.

Author: williamstein

src/packages/frontend/jupyter/browser-actions.ts

@@ -35,6 +35,7 @@ import {
 } from "../misc/local-storage";
 import { parse_headings } from "./contents";
 import { webapp_client } from "@cocalc/frontend/webapp-client";
+import { bufferToBase64 } from "@cocalc/util/base64";
 
 export class JupyterActions extends JupyterActions0 {
   public widget_manager?: WidgetManager;
@@ -452,16 +453,24 @@ export class JupyterActions extends JupyterActions0 {
     comm_id,
     target_name,
     data,
+    buffers,
   }: {
     msg_id?: string;
     comm_id: string;
     target_name: string;
-    data: any;
+    data: unknown;
+    buffers?: ArrayBuffer[] | ArrayBufferView[];
   }): Promise<string> => {
     if (!msg_id) {
       msg_id = uuid();
     }
-    const msg = { msg_id, target_name, comm_id, data };
+    let buffers64;
+    if (buffers != null) {
+      buffers64 = buffers.map(bufferToBase64);
+    } else {
+      buffers64 = [];
+    }
+    const msg = { msg_id, target_name, comm_id, data, buffers64 };
     await this.api_call("comm", msg);
     // console.log("send_comm_message_to_kernel", "sent", msg);
     return msg_id;

src/packages/frontend/jupyter/widgets/manager.ts

@@ -30,6 +30,7 @@ type DeserializedModelState = { [key: string]: any };
 export type SendCommFunction = (string, data) => string;
 
 const log = console.log;
+// const log = (..._args) => {};
 
 export class WidgetManager {
   public ipywidgets_state: IpywidgetsState;
@@ -552,10 +553,15 @@ VBox([s1, s2])
         metadata?: JSONObject,
         buffers?: ArrayBuffer[] | ArrayBufferView[],
       ): string {
-        // TODO: buffers!  These need to get sent somehow.
         log("comm.send", { data, buffers, metadata, callbacks });
         const msg_id = uuid();
-        send_comm_message_to_kernel({ msg_id, comm_id, target_name, data });
+        send_comm_message_to_kernel({
+          msg_id,
+          comm_id,
+          target_name,
+          data,
+          buffers,
+        });
         return msg_id;
       },
 

src/packages/jupyter/kernel/kernel.ts

@@ -79,6 +79,7 @@ import { VERSION } from "@cocalc/jupyter/kernel/version";
 import type { NbconvertParams } from "@cocalc/jupyter/types/nbconvert";
 import type { Client } from "@cocalc/sync/client/types";
 import { getLogger } from "@cocalc/backend/logger";
+import { base64ToBuffer } from "@cocalc/util/base64";
 
 const MAX_KERNEL_SPAWN_TIME = 120 * 1000;
 
@@ -997,13 +998,21 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface {
     comm_id,
     target_name,
     data,
+    buffers64,
   }: {
     msg_id: string;
     comm_id: string;
     target_name: string;
     data: any;
+    buffers64?: string[];
   }): void {
     const dbg = this.dbg("send_comm_message_to_kernel");
+    dbg({ msg_id, comm_id, target_name, data, buffers64 });
+    const buffers = buffers64?.map((x) => new Buffer(base64ToBuffer(x))) ?? [];
+    dbg(
+      "buffers lengths = ",
+      buffers.map((x) => x.byteLength),
+    );
 
     const message = {
       parent_header: {},
@@ -1018,6 +1027,7 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface {
         version: VERSION,
         date: new Date().toISOString(),
       },
+      buffers,
     };
 
     dbg(message);

src/packages/jupyter/redux/project-actions.ts

@@ -1330,8 +1330,8 @@ export class JupyterActions extends JupyterActions0 {
       throw Error("syncdb's ipywidgets_state must be defined!");
     }
     for (const key of keys) {
-      dbg("key = ", key);
       const [, model_id, type] = JSON.parse(key);
+      dbg({ key, model_id, type });
       let data: any;
       if (type === "value") {
         const state = this.syncdb.ipywidgets_state.get_model_value(model_id);
@@ -1352,11 +1352,6 @@ export class JupyterActions extends JupyterActions0 {
         // process data, save file to disk, etc).
         // We need to be careful though to not send buffers to the kernel that the kernel sent us,
         // since that would be a waste.
-        // TODO
-        // I think the format for the send_comm_mmessage_to_kernel call here is easy to deduce
-        // from what process_comm_message_from_kernel unpacks.
-        //   ***
-        // TODO
       } else if (type === "state") {
         // TODO: currently ignoring this, since it seems chatty and pointless,
         // and could lead to race conditions probably with multiple users, etc.