From 12605254d3b82ac7ea750a9f759d0d68f090175e Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 19 Dec 2024 09:01:14 +1100 Subject: [PATCH 1/8] Support fallback of jupyter message serialization --- src/kernels/jupyter/jupyterUtils.ts | 36 ++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/kernels/jupyter/jupyterUtils.ts b/src/kernels/jupyter/jupyterUtils.ts index 3576e4be92b..bfda8ce64d8 100644 --- a/src/kernels/jupyter/jupyterUtils.ts +++ b/src/kernels/jupyter/jupyterUtils.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { ServerConnection } from '@jupyterlab/services'; +import type { KernelMessage, ServerConnection } from '@jupyterlab/services'; import * as path from '../../platform/vscode-path/path'; import { ConfigurationTarget, Uri, window } from 'vscode'; import { IJupyterConnection } from '../types'; @@ -137,7 +137,36 @@ export function createJupyterConnectionInfo( requestInit = { ...requestInit, agent: requestAgent }; } - const { ServerConnection } = require('@jupyterlab/services'); + const { ServerConnection } = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); + const { deserialize, serialize } = + require('@jupyterlab/services/lib/kernel/serialize') as typeof import('@jupyterlab/services/lib/kernel/serialize'); + const { supportedKernelWebSocketProtocols } = + require('@jupyterlab/services/lib/kernel/messages') as typeof import('@jupyterlab/services/lib/kernel/messages'); + + const serializer: import('@jupyterlab/services').ServerConnection.ISettings['serializer'] = { + deserialize: (data: ArrayBuffer, protocol?: string) => { + try { + return deserialize(data, protocol); + } catch (ex) { + if (protocol) { + return deserialize(data, ''); + } else { + return deserialize(data, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); + } + } + }, + serialize: (msg: KernelMessage.IMessage, protocol?: string) => { + try { + return serialize(msg, protocol); + } catch (ex) { + if (protocol) { + return serialize(msg, ''); + } else { + return serialize(msg, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); + } + } + } + }; // This replaces the WebSocket constructor in jupyter lab services with our own implementation // See _createSocket here: // https://github.com/jupyterlab/jupyterlab/blob/cfc8ebda95e882b4ed2eefd54863bb8cdb0ab763/packages/services/src/kernel/default.ts @@ -155,7 +184,8 @@ export function createJupyterConnectionInfo( ) as any), fetch: serverUri.fetch || requestCreator.getFetchMethod(), Request: requestCreator.getRequestCtor(undefined, allowUnauthorized, getAuthHeader), - Headers: requestCreator.getHeadersCtor() + Headers: requestCreator.getHeadersCtor(), + serializer }; const connection: IJupyterConnection = { From 21b14c9ee83ff8d791d24a7e1d0622b62f843856 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 6 Jan 2025 07:30:30 +1100 Subject: [PATCH 2/8] fixes --- src/kernels/jupyter/jupyterUtils.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/kernels/jupyter/jupyterUtils.ts b/src/kernels/jupyter/jupyterUtils.ts index bfda8ce64d8..2c6840adcfe 100644 --- a/src/kernels/jupyter/jupyterUtils.ts +++ b/src/kernels/jupyter/jupyterUtils.ts @@ -146,12 +146,16 @@ export function createJupyterConnectionInfo( const serializer: import('@jupyterlab/services').ServerConnection.ISettings['serializer'] = { deserialize: (data: ArrayBuffer, protocol?: string) => { try { + if (typeof data === 'string') { + return deserialize(data, ''); + } return deserialize(data, protocol); } catch (ex) { + logger.warn(`Failed to deserialize message protocol = ${protocol}`, ex); if (protocol) { - return deserialize(data, ''); - } else { return deserialize(data, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); + } else { + return deserialize(data, ''); } } }, @@ -159,10 +163,11 @@ export function createJupyterConnectionInfo( try { return serialize(msg, protocol); } catch (ex) { + logger.warn(`Failed to serialize message protocol = ${protocol}`, ex); if (protocol) { - return serialize(msg, ''); - } else { return serialize(msg, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); + } else { + return serialize(msg, ''); } } } From 6bd1f7c1b9d70d466e9dd5d60ec8ecbf6ae63772 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 22 Apr 2025 16:21:28 +1000 Subject: [PATCH 3/8] wip --- src/kernels/jupyter/jupyterUtils.ts | 37 ++++++++++++++++------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/kernels/jupyter/jupyterUtils.ts b/src/kernels/jupyter/jupyterUtils.ts index 2c6840adcfe..f3ea97e621d 100644 --- a/src/kernels/jupyter/jupyterUtils.ts +++ b/src/kernels/jupyter/jupyterUtils.ts @@ -140,11 +140,12 @@ export function createJupyterConnectionInfo( const { ServerConnection } = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); const { deserialize, serialize } = require('@jupyterlab/services/lib/kernel/serialize') as typeof import('@jupyterlab/services/lib/kernel/serialize'); - const { supportedKernelWebSocketProtocols } = - require('@jupyterlab/services/lib/kernel/messages') as typeof import('@jupyterlab/services/lib/kernel/messages'); + // const { supportedKernelWebSocketProtocols } = + // require('@jupyterlab/services/lib/kernel/messages') as typeof import('@jupyterlab/services/lib/kernel/messages'); const serializer: import('@jupyterlab/services').ServerConnection.ISettings['serializer'] = { deserialize: (data: ArrayBuffer, protocol?: string) => { + logger.trace(`Deserialize message ${typeof data} && ${data instanceof Buffer} with ${protocol}`); try { if (typeof data === 'string') { return deserialize(data, ''); @@ -152,24 +153,26 @@ export function createJupyterConnectionInfo( return deserialize(data, protocol); } catch (ex) { logger.warn(`Failed to deserialize message protocol = ${protocol}`, ex); - if (protocol) { - return deserialize(data, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); - } else { - return deserialize(data, ''); - } + throw ex; + // if (protocol) { + // return deserialize(data, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); + // } else { + // return deserialize(data, ''); + // } } }, serialize: (msg: KernelMessage.IMessage, protocol?: string) => { - try { - return serialize(msg, protocol); - } catch (ex) { - logger.warn(`Failed to serialize message protocol = ${protocol}`, ex); - if (protocol) { - return serialize(msg, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); - } else { - return serialize(msg, ''); - } - } + logger.trace(`Serialize message ${typeof msg} && ${msg instanceof Buffer} with ${protocol}`); + // try { + return serialize(msg, protocol); + // } catch (ex) { + // logger.warn(`Failed to serialize message protocol = ${protocol}`, ex); + // if (protocol) { + // return serialize(msg, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); + // } else { + // return serialize(msg, ''); + // } + // } } }; // This replaces the WebSocket constructor in jupyter lab services with our own implementation From 22690e5ddd3f86dc6dc3710248fa8866f2a881fd Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 22 Apr 2025 16:23:14 +1000 Subject: [PATCH 4/8] wip --- package-lock.json | 2 +- package.json | 2 +- src/kernels/jupyter/jupyterUtils.ts | 9 --------- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 301f05eb3e9..d8d001b215a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -229,7 +229,7 @@ "webpack-cli": "^5.1.4" }, "engines": { - "vscode": "^1.100.0" + "vscode": "^1.98.0" }, "optionalDependencies": { "fsevents": "^2.3.2" diff --git a/package.json b/package.json index 5b4a88a0e97..af945a28518 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "theme": "light" }, "engines": { - "vscode": "^1.100.0" + "vscode": "^1.98.0" }, "l10n": "./l10n", "extensionKind": [ diff --git a/src/kernels/jupyter/jupyterUtils.ts b/src/kernels/jupyter/jupyterUtils.ts index f3ea97e621d..c268efccb21 100644 --- a/src/kernels/jupyter/jupyterUtils.ts +++ b/src/kernels/jupyter/jupyterUtils.ts @@ -163,16 +163,7 @@ export function createJupyterConnectionInfo( }, serialize: (msg: KernelMessage.IMessage, protocol?: string) => { logger.trace(`Serialize message ${typeof msg} && ${msg instanceof Buffer} with ${protocol}`); - // try { return serialize(msg, protocol); - // } catch (ex) { - // logger.warn(`Failed to serialize message protocol = ${protocol}`, ex); - // if (protocol) { - // return serialize(msg, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); - // } else { - // return serialize(msg, ''); - // } - // } } }; // This replaces the WebSocket constructor in jupyter lab services with our own implementation From b04eaba599d4180e93bd22ea826b3bf38d193012 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 22 Apr 2025 20:14:34 +1000 Subject: [PATCH 5/8] More logging --- src/kernels/jupyter/jupyterUtils.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/kernels/jupyter/jupyterUtils.ts b/src/kernels/jupyter/jupyterUtils.ts index c268efccb21..f2291802478 100644 --- a/src/kernels/jupyter/jupyterUtils.ts +++ b/src/kernels/jupyter/jupyterUtils.ts @@ -145,12 +145,17 @@ export function createJupyterConnectionInfo( const serializer: import('@jupyterlab/services').ServerConnection.ISettings['serializer'] = { deserialize: (data: ArrayBuffer, protocol?: string) => { - logger.trace(`Deserialize message ${typeof data} && ${data instanceof Buffer} with ${protocol}`); + logger.trace(`Deserialize message type=${typeof data}, Buffer=${data instanceof Buffer}, ArrayBuffer=${data instanceof ArrayBuffer} with ${protocol}`); + logger.trace(`Deserialize data=${data.toString()}`); try { if (typeof data === 'string') { - return deserialize(data, ''); + const result = deserialize(data, ''); + logger.trace(`Deserialize message type=string with ${protocol}`); + return result; } - return deserialize(data, protocol); + const result = deserialize(data, protocol); + logger.trace(`Deserialize message type=ArrayBuffer with ${protocol}`); + return result; } catch (ex) { logger.warn(`Failed to deserialize message protocol = ${protocol}`, ex); throw ex; From 8204a9f54ae65d2447dd62e34da2c71fa4d84a57 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Sat, 26 Apr 2025 16:12:06 +1000 Subject: [PATCH 6/8] Updates --- src/kernels/common/kernelSocketWrapper.ts | 2 +- src/kernels/jupyter/jupyterUtils.ts | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/kernels/common/kernelSocketWrapper.ts b/src/kernels/common/kernelSocketWrapper.ts index f802faba792..8d477658607 100644 --- a/src/kernels/common/kernelSocketWrapper.ts +++ b/src/kernels/common/kernelSocketWrapper.ts @@ -101,7 +101,7 @@ export function KernelSocketWrapper>(SuperCl this.msgChain = this.msgChain .then(() => Promise.all(this.receiveHooks.map((p) => p(args[0])))) .then(() => superHandler(event, ...args)) - .catch((e) => logger.error(`Exception while handling messages: ${e}`)); + .catch((e) => logger.error(`Exception while handling messages: `, e, e.stack)); // True value indicates there were handlers. We definitely have 'message' handlers. return true; } else { diff --git a/src/kernels/jupyter/jupyterUtils.ts b/src/kernels/jupyter/jupyterUtils.ts index f2291802478..271875cacc6 100644 --- a/src/kernels/jupyter/jupyterUtils.ts +++ b/src/kernels/jupyter/jupyterUtils.ts @@ -145,7 +145,11 @@ export function createJupyterConnectionInfo( const serializer: import('@jupyterlab/services').ServerConnection.ISettings['serializer'] = { deserialize: (data: ArrayBuffer, protocol?: string) => { - logger.trace(`Deserialize message type=${typeof data}, Buffer=${data instanceof Buffer}, ArrayBuffer=${data instanceof ArrayBuffer} with ${protocol}`); + logger.trace( + `Deserialize message type=${typeof data}, Buffer=${data instanceof Buffer}, ArrayBuffer=${ + data instanceof ArrayBuffer + } with ${protocol}` + ); logger.trace(`Deserialize data=${data.toString()}`); try { if (typeof data === 'string') { @@ -157,7 +161,7 @@ export function createJupyterConnectionInfo( logger.trace(`Deserialize message type=ArrayBuffer with ${protocol}`); return result; } catch (ex) { - logger.warn(`Failed to deserialize message protocol = ${protocol}`, ex); + logger.error(`Failed to deserialize message protocol = ${protocol}`, ex); throw ex; // if (protocol) { // return deserialize(data, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); @@ -168,7 +172,17 @@ export function createJupyterConnectionInfo( }, serialize: (msg: KernelMessage.IMessage, protocol?: string) => { logger.trace(`Serialize message ${typeof msg} && ${msg instanceof Buffer} with ${protocol}`); - return serialize(msg, protocol); + try { + const data = serialize(msg, protocol); + logger.trace(`Serialized message ${typeof msg} && ${msg instanceof Buffer} with ${protocol}`); + return data; + } catch (ex) { + logger.error( + `Failed to serialize message ${typeof msg} && ${msg instanceof Buffer} with ${protocol}`, + ex + ); + throw ex; + } } }; // This replaces the WebSocket constructor in jupyter lab services with our own implementation From 3f0983e0dd481cff2761fb751d2922c8466e6b33 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 5 Jun 2025 10:32:32 +1000 Subject: [PATCH 7/8] Even more logging --- src/kernels/jupyter/jupyterUtils.ts | 10 +++ .../message/ipyWidgetMessageDispatcher.ts | 68 ++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/kernels/jupyter/jupyterUtils.ts b/src/kernels/jupyter/jupyterUtils.ts index 271875cacc6..40fa0b6f609 100644 --- a/src/kernels/jupyter/jupyterUtils.ts +++ b/src/kernels/jupyter/jupyterUtils.ts @@ -145,6 +145,15 @@ export function createJupyterConnectionInfo( const serializer: import('@jupyterlab/services').ServerConnection.ISettings['serializer'] = { deserialize: (data: ArrayBuffer, protocol?: string) => { + try { + logger.trace(`1.a UTILS. MESSAGE`, JSON.stringify(data)); + } catch (error) { + try { + logger.trace(`1.b UTILS. MESSAGE`, data, error); + } catch (error2) { + logger.error(`1.c UTILS. MESSAGE`, error, error2); + } + } logger.trace( `Deserialize message type=${typeof data}, Buffer=${data instanceof Buffer}, ArrayBuffer=${ data instanceof ArrayBuffer @@ -157,6 +166,7 @@ export function createJupyterConnectionInfo( logger.trace(`Deserialize message type=string with ${protocol}`); return result; } + logger.trace(`Deserialize message type=NOT string with ${protocol}`); const result = deserialize(data, protocol); logger.trace(`Deserialize message type=ArrayBuffer with ${protocol}`); return result; diff --git a/src/notebooks/controllers/ipywidgets/message/ipyWidgetMessageDispatcher.ts b/src/notebooks/controllers/ipywidgets/message/ipyWidgetMessageDispatcher.ts index 4e815dc6d41..db2ecaf4670 100644 --- a/src/notebooks/controllers/ipywidgets/message/ipyWidgetMessageDispatcher.ts +++ b/src/notebooks/controllers/ipywidgets/message/ipyWidgetMessageDispatcher.ts @@ -107,7 +107,73 @@ export class IPyWidgetMessageDispatcher implements IIPyWidgetMessageDispatcher { // eslint-disable-next-line @typescript-eslint/no-require-imports const jupyterLabSerialize = require('@jupyterlab/services/lib/kernel/serialize') as typeof import('@jupyterlab/services/lib/kernel/serialize'); // NOSONAR - this.deserialize = jupyterLabSerialize.deserialize; + this.deserialize = (data: ArrayBuffer, protocol?: string) => { + if ( + data && + typeof data === 'object' && + (data as unknown as KernelMessage.IMessage).header && + (data as unknown as KernelMessage.IMessage).channel + ) { + logger.trace( + `1.0 WIDGET. Deserialize message not required, has header and channel properties ${protocol}` + ); + } + try { + logger.trace(`1.a WIDGET. MESSAGE`, JSON.stringify(data)); + } catch (error) { + try { + logger.trace(`1.b WIDGET. MESSAGE`, data, error); + } catch (error2) { + logger.error(`1.c WIDGET. MESSAGE`, error, error2); + } + } + logger.trace( + `1. WIDGET. Deserialize message type=${typeof data}, Buffer=${data instanceof Buffer}, ArrayBuffer=${ + data instanceof ArrayBuffer + } with ${protocol}` + ); + logger.trace(`1. WIDGET. Deserialize data=${data.toString()}`); + try { + if (typeof data === 'string') { + const result = jupyterLabSerialize.deserialize(data, ''); + logger.trace(`1. WIDGET. Deserialize message type=string with ${protocol}`); + return result; + } + if ( + data && + typeof data === 'object' && + (data as unknown as KernelMessage.IMessage).header && + (data as unknown as KernelMessage.IMessage).channel + ) { + logger.trace( + `1.x WIDGET. Deserialize message not required, has header and channel properties ${protocol}` + ); + try { + logger.trace( + `1.y WIDGET. Deserialize message not required type=JSON object with ${protocol}`, + JSON.stringify(data) + ); + } catch { + logger.trace( + `1.z WIDGET. Deserialize message not required type=JSON object with ${protocol}`, + data + ); + } + return data as unknown as KernelMessage.IMessage; + } + const result = jupyterLabSerialize.deserialize(data, protocol); + logger.trace(`1. WIDGET. Deserialize message type=ArrayBuffer with ${protocol}`); + return result; + } catch (ex) { + logger.error(`1. WIDGET. Failed to deserialize message protocol = ${protocol}`, ex); + throw ex; + // if (protocol) { + // return deserialize(data, supportedKernelWebSocketProtocols.v1KernelWebsocketJupyterOrg); + // } else { + // return deserialize(data, ''); + // } + } + }; } public dispose() { this.disposed = true; From 63ef56a23185b55f7f38d049c2dac88302c18d54 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 5 Jun 2025 11:31:14 +1000 Subject: [PATCH 8/8] Updates --- .vscodeignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.vscodeignore b/.vscodeignore index da310b0a3ac..511aede2ddb 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -5,6 +5,7 @@ images/** docs/** *.vsix .appveyor.yml +.config/** .devcontainer*/** .editorconfig .env @@ -18,7 +19,9 @@ docs/** .npmrc .nvmrc .nycrc +.ruff_cache/** .travis.yml +.vscode-test.mjs api/** CODE_OF_CONDUCT.md CODING_STANDARDS.md @@ -83,6 +86,7 @@ obj/** logs/** out/** precommit.hook +pythonFiles/.env pythonFiles/**/*.pyc pythonFiles/lib/**/*.dist-info/** pythonFiles/lib/**/*.egg-info/**