Merged PR posit-dev/positron-python#101: Wrap Jupyter Kernel runtime with Python Language Runtime

Merge pull request #101 from posit-dev/feature/jupyter-adapter-api

Wrap Jupyter Kernel runtime with Python Language Runtime
--------------------
Commit message for posit-dev/positron-python@ca3b312:

Merge remote-tracking branch 'origin/main' into feature/jupyter-adapter-api

--------------------
Commit message for posit-dev/positron-python@8dd9758:

allow unresolved 'positron' import (TS lint)

--------------------
Commit message for posit-dev/positron-python@d446de6:

implement shutdown/restart/interrupt as async methods

--------------------
Commit message for posit-dev/positron-python@08e9e26:

register new PythonLanguageRuntime wrapping the kernel

--------------------
Commit message for posit-dev/positron-python@5f66208:

add wrapper class for language runtime kernel

--------------------
Commit message for posit-dev/positron-python@e0190c5:

add jupyter adapter types


Authored-by: Jonathan McPherson <[email protected]>
Signed-off-by: Jonathan McPherson <[email protected]>

Author: jmcphers

extensions/positron-python/positron-dts/positron.d.ts

@@ -85,6 +85,9 @@ declare module 'positron' {
 		/** The runtime is busy executing code. */
 		Busy = 'busy',
 
+		/** The runtime is in the process of restarting. */
+		Restarting = 'restarting',
+
 		/** The runtime is in the process of shutting down. */
 		Exiting = 'exiting',
 
@@ -314,15 +317,15 @@ declare module 'positron' {
 		/** The version of the language; e.g. "4.2" */
 		languageVersion: string;
 
-		/** The Base64-encoded icon SVG for the language. */
-		base64EncodedIconSvg: string | undefined;
-
 		/** The text the language's interpreter uses to prompt the user for input, e.g. ">" or ">>>" */
 		inputPrompt: string;
 
 		/** The text the language's interpreter uses to prompt the user for continued input, e.g. "+" or "..." */
 		continuationPrompt: string;
 
+		/** The Base64-encoded icon SVG for the language. */
+		base64EncodedIconSvg: string | undefined;
+
 		/** Whether the runtime should start up automatically or wait until explicitly requested */
 		startupBehavior: LanguageRuntimeStartupBehavior;
 	}
@@ -473,14 +476,25 @@ declare module 'positron' {
 		 */
 		start(): Thenable<LanguageRuntimeInfo>;
 
-		/** Interrupt the runtime */
-		interrupt(): void;
+		/**
+		 * Interrupt the runtime; returns a Thenable that resolves when the interrupt has been
+		 * successfully sent to the runtime (not necessarily when it has been processed)
+		 */
+		interrupt(): Thenable<void>;
 
-		/** Restart the runtime */
-		restart(): void;
+		/**
+		 * Restart the runtime; returns a Thenable that resolves when the runtime restart sequence
+		 * has been successfully started (not necessarily when it has completed). A restart will
+		 * cause the runtime to be shut down and then started again; its status will change from
+		 * `Restarting` => `Exited` => `Initializing` => `Starting` => `Ready`.
+		 */
+		restart(): Thenable<void>;
 
-		/** Shut down the runtime */
-		shutdown(): void;
+		/**
+		 * Shut down the runtime; returns a Thenable that resolves when the runtime shutdown
+		 * sequence has been successfully started (not necessarily when it has completed).
+		 */
+		shutdown(): Thenable<void>;
 	}
 
 

extensions/positron-python/src/client/activation/jedi/positronLanguageRuntimes.ts

@@ -11,6 +11,7 @@ import * as fs from 'fs';
 // eslint-disable-next-line import/no-unresolved
 import * as positron from 'positron';
 import * as vscode from 'vscode';
+import * as crypto from 'crypto';
 import { Disposable, DocumentFilter, LanguageClient, LanguageClientOptions, ServerOptions, StreamInfo } from 'vscode-languageclient/node';
 
 import { compare } from 'semver';
@@ -24,6 +25,8 @@ import { PythonEnvironment } from '../../pythonEnvironments/info';
 import { PythonVersion } from '../../pythonEnvironments/info/pythonVersion';
 import { ProgressReporting } from '../progress';
 import { ILanguageServerProxy } from '../types';
+import { PythonLanguageRuntime } from './pythonLanguageRuntime';
+import { JupyterAdapterApi } from '../../jupyter-adapter.d'
 
 // Load the Python icon.
 const base64EncodedIconSvg = fs.readFileSync(path.join(EXTENSION_ROOT_DIR, 'resources', 'branding', 'python-icon.svg')).toString('base64');
@@ -97,7 +100,11 @@ export class PositronJediLanguageServerProxy implements ILanguageServerProxy {
 
         // Register available python interpreters as language runtimes with our Jupyter Adapter
         this.withActiveExtention(ext, async () => {
-            await this.registerLanguageRuntimes(ext, resource, interpreter, options);
+            // Create typesafe reference to the Jupyter Adapter's API
+            const jupyterAdapterApi = ext.exports as JupyterAdapterApi;
+
+            // Register the language runtimes for each available interpreter
+            await this.registerLanguageRuntimes(jupyterAdapterApi, resource, interpreter, options);
         });
     }
 
@@ -139,7 +146,7 @@ export class PositronJediLanguageServerProxy implements ILanguageServerProxy {
      * Register available python environments as a language runtime with the Jupyter Adapter.
      */
     private async registerLanguageRuntimes(
-        ext: vscode.Extension<any>,
+        jupyterAdapterApi: JupyterAdapterApi,
         resource: Resource,
         preferredInterpreter: PythonEnvironment | undefined,
         options: LanguageClientOptions
@@ -167,7 +174,8 @@ export class PositronJediLanguageServerProxy implements ILanguageServerProxy {
                 debugPort = await portfinder.getPortPromise({ port: debugPort });
             }
 
-            const runtime: vscode.Disposable = await this.registerLanguageRuntime(ext, interpreter, debugPort, logLevel, options);
+            const runtime: vscode.Disposable = await this.registerLanguageRuntime(
+                jupyterAdapterApi, interpreter, debugPort, logLevel, options);
             this.disposables.push(runtime);
 
             if (debugPort !== undefined) {
@@ -182,7 +190,7 @@ export class PositronJediLanguageServerProxy implements ILanguageServerProxy {
      * IPyKernel with a connection file.
      */
     private async registerLanguageRuntime(
-        ext: vscode.Extension<any>,
+        jupyterAdapterApi: JupyterAdapterApi,
         interpreter: PythonEnvironment,
         debugPort: number | undefined,
         logLevel: string,
@@ -210,16 +218,40 @@ export class PositronJediLanguageServerProxy implements ILanguageServerProxy {
         };
         traceVerbose(`Configuring Jedi LSP with IPyKernel using args '${args}'`);
 
-        // Create an adapter for the kernel as our language runtime
-        const runtime: positron.LanguageRuntime = ext.exports.adaptKernel(kernelSpec,
-            PYTHON_LANGUAGE,
-            pythonVersion,
-            this.extensionVersion,
+        // Create a stable ID for the runtime based on the interpreter path and version.
+        const digest = crypto.createHash('sha256');
+        digest.update(JSON.stringify(kernelSpec));
+        digest.update(pythonVersion ?? '');
+        const runtimeId = digest.digest('hex').substring(0, 32);
+
+        // Create the metadata for the language runtime
+        const metadata: positron.LanguageRuntimeMetadata = {
+            runtimePath: interpreter.path,
+            runtimeId,
+            runtimeName: displayName,
+            runtimeVersion: this.extensionVersion ?? '0.0.0',
+            languageName: kernelSpec.language,
+            languageId: PYTHON_LANGUAGE,
+            languageVersion: pythonVersion ?? '0.0.0',
             base64EncodedIconSvg,
-            '>>>',
-            '...',
-            startupBehavior,
-            (port: number) => this.startClient(options, port));
+            inputPrompt: '>>>',
+            continuationPrompt: '...',
+            startupBehavior
+        }
+
+        // Create an adapter for the kernel as our language runtime
+        const runtime = new PythonLanguageRuntime(
+            kernelSpec, metadata, jupyterAdapterApi);
+
+        runtime.onDidChangeRuntimeState((state) => {
+            if (state === positron.RuntimeState.Ready) {
+                jupyterAdapterApi.findAvailablePort([], 25).then((port: number) => {
+                    runtime.emitJupyterLog(`Starting Positron LSP server on port ${port}`);
+                    runtime.startPositronLsp(`127.0.0.1:${port}`);
+                    this.startClient(options, port).ignoreErrors();
+                });
+            }
+        });
 
         // Register our language runtime provider
         return positron.runtime.registerLanguageRuntime(runtime);

extensions/positron-python/src/client/activation/jedi/pythonLanguageRuntime.ts

@@ -0,0 +1,94 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2023 Posit Software, PBC. All rights reserved.
+ *--------------------------------------------------------------------------------------------*/
+
+// eslint-disable-next-line import/no-unresolved
+import * as positron from 'positron';
+import * as vscode from 'vscode';
+import { JupyterAdapterApi, JupyterKernelSpec, JupyterLanguageRuntime } from '../../jupyter-adapter.d';
+
+/**
+ * A Positron Python language runtime that wraps a Jupyter kernel runtime.
+ * Fulfills most Language Runtime API calls by delegating to the wrapped kernel.
+ */
+export class PythonLanguageRuntime implements JupyterLanguageRuntime {
+
+    /** The Jupyter kernel-based implementation of the Language Runtime API */
+    private _kernel: JupyterLanguageRuntime;
+
+    /**
+     * Create a new PythonLanguageRuntime object to wrap a Jupyter kernel.
+     *
+     * @param kernelSpec The specification of the Jupyter kernel to wrap.
+     * @param metadata The metadata of the language runtime to create.
+     * @param adapterApi The API of the Jupyter Adapter extension.
+     */
+    constructor(
+        readonly kernelSpec: JupyterKernelSpec,
+        readonly metadata: positron.LanguageRuntimeMetadata,
+        readonly adapterApi: JupyterAdapterApi) {
+
+        this._kernel = adapterApi.adaptKernel(kernelSpec, metadata);
+
+        this.onDidChangeRuntimeState = this._kernel.onDidChangeRuntimeState;
+        this.onDidReceiveRuntimeMessage = this._kernel.onDidReceiveRuntimeMessage;
+    }
+
+    startPositronLsp(clientAddress: string): void {
+        this._kernel.startPositronLsp(clientAddress);
+    }
+
+    emitJupyterLog(message: string): void {
+        this._kernel.emitJupyterLog(message);
+    }
+
+    onDidReceiveRuntimeMessage: vscode.Event<positron.LanguageRuntimeMessage>;
+
+    onDidChangeRuntimeState: vscode.Event<positron.RuntimeState>;
+
+    execute(code: string, id: string, mode: positron.RuntimeCodeExecutionMode, errorBehavior: positron.RuntimeErrorBehavior): void {
+        this._kernel.execute(code, id, mode, errorBehavior);
+    }
+
+    isCodeFragmentComplete(code: string): Thenable<positron.RuntimeCodeFragmentStatus> {
+        return this._kernel.isCodeFragmentComplete(code);
+    }
+
+    // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any
+    createClient(id: string, type: positron.RuntimeClientType, params: any): Thenable<void> {
+        return this._kernel.createClient(id, type, params);
+    }
+
+    listClients(type?: positron.RuntimeClientType | undefined): Thenable<Record<string, string>> {
+        return this._kernel.listClients(type);
+    }
+
+    removeClient(id: string): void {
+        this._kernel.removeClient(id);
+    }
+
+    // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any
+    sendClientMessage(clientId: string, messageId: string, message: any): void {
+        this._kernel.sendClientMessage(clientId, messageId, message);
+    }
+
+    replyToPrompt(id: string, reply: string): void {
+        this._kernel.replyToPrompt(id, reply);
+    }
+
+    start(): Thenable<positron.LanguageRuntimeInfo> {
+        return this._kernel.start();
+    }
+
+    async interrupt(): Promise<void> {
+        return this._kernel.interrupt();
+    }
+
+    async restart(): Promise<void> {
+        return this._kernel.restart();
+    }
+
+    async shutdown(): Promise<void> {
+        return this._kernel.shutdown();
+    }
+}

extensions/positron-python/src/client/jupyter-adapter.d.ts

@@ -0,0 +1,85 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2023 Posit Software, PBC. All rights reserved.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+
+// eslint-disable-next-line import/no-unresolved
+import * as positron from 'positron';
+
+/**
+ * This set of type definitions defines the interfaces used by the Positron
+ * Jupyter Adapter extension.
+ */
+
+/**
+ * Represents a registered Jupyter Kernel. These types are defined in the
+ * Jupyter documentation at:
+ *
+ * https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs
+ */
+export interface JupyterKernelSpec {
+	/** Command used to start the kernel and an array of command line arguments */
+	argv: Array<string>;
+
+	/** The kernel's display name */
+	display_name: string;  // eslint-disable-line
+
+	/** The language the kernel executes */
+	language: string;
+
+	/** Interrupt mode (signal or message) */
+	interrupt_mode?: 'signal' | 'message'; // eslint-disable-line
+
+	/** Environment variables to set when starting the kernel */
+	env?: { [key: string]: string };
+}
+
+/**
+ * A language runtime that wraps a Jupyter kernel.
+ */
+export interface JupyterLanguageRuntime extends positron.LanguageRuntime {
+	/**
+	 * Convenience method for starting the Positron LSP server, if the
+	 * language runtime supports it.
+	 *
+	 * @param clientAddress The address of the client that will connect to the
+	 *  language server.
+	 */
+	startPositronLsp(clientAddress: string): void;
+
+	/**
+	 * Method for emitting a message to the language server's Jupyter output
+	 * channel.
+	 *
+	 * @param message A message to emit to the Jupyter log.
+	 */
+	emitJupyterLog(message: string): void;
+}
+
+/**
+ * The Jupyter Adapter API as exposed by the Jupyter Adapter extension.
+ */
+export interface JupyterAdapterApi extends vscode.Disposable {
+
+	/**
+	 * Create an adapter for a Jupyter-compatible kernel.
+	 *
+	 * @param kernel A Jupyter kernel spec containing the information needed to
+	 *   start the kernel.
+	 * @param metadata The metadata for the language runtime to be wrapped by the
+	 *   adapter.
+	 * @returns A LanguageRuntimeAdapter that wraps the kernel.
+	 */
+	adaptKernel(kernel: JupyterKernelSpec,
+		metadata: positron.LanguageRuntimeMetadata): JupyterLanguageRuntime;
+
+	/**
+	 * Finds an available TCP port for a server
+	 *
+	 * @param excluding A list of ports to exclude from the search
+	 * @param maxTries The maximum number of attempts
+	 * @returns An available TCP port
+	 */
+	findAvailablePort(excluding: Array<number>, maxTries: number): Promise<number>;
+}