Merge pull request #321 from krassowski/transclusion-refactor

Make code extractors and replacements modular

Author: krassowski

docs/Extending.ipynb

@@ -46,7 +46,7 @@
     "#### How to override the default implementation of a feature?\n",
     "\n",
     "You can specify a list of extensions to be disabled the the feature manager\n",
-    "passing their plugin identifiers in `superseds` field of `IFeatureOptions`."
+    "passing their plugin identifiers in `supersedes` field of `IFeatureOptions`."
    ]
   },
   {
@@ -64,7 +64,7 @@
     "\n",
     "#### Why virtual editor?\n",
     "\n",
-    "The vitual editor takes multiple instances of your editor (e.g. in a notebook)\n",
+    "The virtual editor takes multiple instances of your editor (e.g. in a notebook)\n",
     "and makes them act like a single editor. For example, when \"onKeyPress\" event is\n",
     "bound on the VirtualEditor instance, it should be bound onto each actual code\n",
     "editor; this allows the features to be implemented without the knowledge about\n",
@@ -89,7 +89,7 @@
     "`IDocumentWidget` interface. Each such widget has to adapted by a\n",
     "`WidgetAdapter` to enable its use with the LSP extension. The role of the\n",
     "`WidgetAdapter` is to extract the document metadata (language, mimetype) and the\n",
-    "underyling code editor (e.g. CodeMirror or Monaco) instances so that other parts\n",
+    "underlying code editor (e.g. CodeMirror or Monaco) instances so that other parts\n",
     "of the LSP extension can interface with them without knowing about the\n",
     "implementation details of the DocumentWidget (or even about the existence of a\n",
     "Notebook construct!).\n",
@@ -108,8 +108,18 @@
    "source": [
     "### How to add a custom magic or foreign extractor?\n",
     "\n",
-    "The ability to define custom magics or register foreign extractors is currently\n",
-    "a work-in-progress."
+    "It is now possible to register custom code replacements using\n",
+    "`ILSPCodeOverridesManager` token and to register custom foreign code extractors\n",
+    "using `ILSPCodeExtractorsManager` token, however this API is considered\n",
+    "provisional and subject to change.\n",
+    "\n",
+    "#### Future plans for transclusions handling\n",
+    "\n",
+    "We will strive to make it possible for kernels to register their custom\n",
+    "syntax/code transformations easily, but the frontend API will remain available\n",
+    "for the end-users who write their custom syntax modifications with actionable\n",
+    "side-effects (e.g. a custom IPython magic which copies a variable from the host\n",
+    "document to the embedded document)."
    ]
   },
   {
@@ -135,7 +145,7 @@
     "2. Prepare `CompletionKind` → `IconSvgString` mapping for the light (and\n",
     "   optionally dark) theme, implementing the `ICompletionIconSet` interface. We\n",
     "   have an additional `Kernel` completion kind that is used for completions\n",
-    "   provided by kernel that had no recognisable type provided.\n",
+    "   provided by kernel that had no recognizable type provided.\n",
     "\n",
     "3. Provide all other metadata required by the `ICompletionTheme` interface and\n",
     "   register it on `ILSPCompletionThemeManager` instance using `register_theme()`\n",
@@ -144,7 +154,7 @@
     "4. Provide any additional CSS styling targeting the JupyterLab completer\n",
     "   elements inside of `.lsp-completer-theme-{id}`, e.g.\n",
     "   `.lsp-completer-theme-material .jp-Completer-icon svg` for the material\n",
-    "   theme. Rememmber to include the styles by importing the in one of the source\n",
+    "   theme. Remember to include the styles by importing the in one of the source\n",
     "   files.\n",
     "\n",
     "For an example of a complete theme see\n",

packages/jupyterlab-lsp/src/adapters/adapter.ts

@@ -17,6 +17,7 @@ import { ILSPExtension } from '../index';
 import { IFeatureEditorIntegration, IFeature } from '../feature';
 import { EditorAdapter } from '../editor_integration/editor_adapter';
 import IEditor = CodeEditor.IEditor;
+import { LanguageIdentifier } from '../lsp';
 
 export class StatusMessage {
   /**
@@ -126,6 +127,14 @@ export abstract class WidgetAdapter<T extends IDocumentWidget> {
     this.widget.disposed.connect(this.dispose, this);
   }
 
+  protected get foreign_code_extractors() {
+    return this.extension.foreign_code_extractors;
+  }
+
+  protected get code_overrides() {
+    return this.extension.code_overrides;
+  }
+
   on_connection_closed(
     manager: DocumentConnectionManager,
     { virtual_document }: IDocumentConnectionData
@@ -187,8 +196,9 @@ export abstract class WidgetAdapter<T extends IDocumentWidget> {
     return this.widget.id;
   }
 
-  get language(): string {
-    // the values should follow https://microsoft.github.io/language-server-protocol/specification guidelines
+  get language(): LanguageIdentifier {
+    // the values should follow https://microsoft.github.io/language-server-protocol/specification guidelines,
+    // see the table in https://microsoft.github.io/language-server-protocol/specification#textDocumentItem
     if (mime_type_language_map.hasOwnProperty(this.mime_type)) {
       return mime_type_language_map[this.mime_type] as string;
     } else {

packages/jupyterlab-lsp/src/adapters/notebook/notebook.ts

@@ -1,8 +1,6 @@
 import { WidgetAdapter } from '../adapter';
 import { Notebook, NotebookPanel } from '@jupyterlab/notebook';
 import { until_ready } from '../../utils';
-import { language_specific_overrides } from '../../magics/defaults';
-import { foreign_code_extractors } from '../../extractors/defaults';
 import { Cell } from '@jupyterlab/cells';
 import * as nbformat from '@jupyterlab/nbformat';
 import ILanguageInfoMetadata = nbformat.ILanguageInfoMetadata;
@@ -165,8 +163,8 @@ export class NotebookAdapter extends WidgetAdapter<NotebookPanel> {
     return new VirtualDocument({
       language: this.language,
       path: this.document_path,
-      overrides_registry: language_specific_overrides,
-      foreign_code_extractors: foreign_code_extractors,
+      overrides_registry: this.code_overrides,
+      foreign_code_extractors: this.foreign_code_extractors,
       file_extension: this.language_file_extension,
       // notebooks are continuous, each cell is dependent on the previous one
       standalone: false,

packages/jupyterlab-lsp/src/editor_integration/feature.spec.ts

@@ -9,8 +9,8 @@ import {
 } from './testutils';
 import * as lsProtocol from 'vscode-languageserver-protocol';
 import * as nbformat from '@jupyterlab/nbformat';
-import { language_specific_overrides } from '../magics/defaults';
-import { foreign_code_extractors } from '../extractors/defaults';
+import { overrides } from '../transclusions/ipython/overrides';
+import { foreign_code_extractors } from '../transclusions/ipython/extractors';
 import { NotebookModel } from '@jupyterlab/notebook';
 import { PageConfig } from '@jupyterlab/coreutils';
 import { CodeMirrorIntegration } from './codemirror';
@@ -183,7 +183,12 @@ describe('Feature', () => {
 
       beforeEach(() => {
         environment = new NotebookFeatureTestEnvironment({
-          overrides_registry: language_specific_overrides,
+          overrides_registry: {
+            python: {
+              cell: overrides.filter(override => override.scope == 'cell'),
+              line: overrides.filter(override => override.scope == 'line')
+            }
+          },
           foreign_code_extractors: foreign_code_extractors
         });
 

packages/jupyterlab-lsp/src/editor_integration/testutils.ts

@@ -44,6 +44,8 @@ import { ServiceManager } from '@jupyterlab/services';
 import { FeatureManager, ILSPExtension } from '../index';
 import { JupyterFrontEnd } from '@jupyterlab/application';
 import { IFeatureSettings } from '../feature';
+import { IForeignCodeExtractorsRegistry } from '../extractors/types';
+import { ICodeOverridesRegistry } from '../overrides/tokens';
 import { Signal } from '@lumino/signaling';
 
 export interface ITestEnvironment {
@@ -94,6 +96,8 @@ export class MockExtension implements ILSPExtension {
   language_server_manager: LanguageServerManager;
   feature_manager: ILSPFeatureManager;
   editor_type_manager: ILSPVirtualEditorManager;
+  foreign_code_extractors: IForeignCodeExtractorsRegistry;
+  code_overrides: ICodeOverridesRegistry;
 
   constructor() {
     this.app = null;
@@ -108,6 +112,8 @@ export class MockExtension implements ILSPExtension {
       name: 'CodeMirrorEditor',
       supports: CodeMirrorEditor
     });
+    this.foreign_code_extractors = {};
+    this.code_overrides = {};
   }
 }
 

packages/jupyterlab-lsp/src/extractors/manager.ts

@@ -0,0 +1,31 @@
+import { JupyterFrontEndPlugin } from '@jupyterlab/application';
+import { ILSPCodeExtractorsManager } from '../tokens';
+import { IForeignCodeExtractor, IForeignCodeExtractorsRegistry } from './types';
+
+export class CodeExtractorsManager implements ILSPCodeExtractorsManager {
+  registry: IForeignCodeExtractorsRegistry;
+
+  constructor() {
+    this.registry = {};
+  }
+
+  /**
+   * Register an extractor to extract foreign code from host documents of specified language.
+   */
+  register(extractor: IForeignCodeExtractor, host_language: string): void {
+    if (!this.registry.hasOwnProperty(host_language)) {
+      this.registry[host_language] = [];
+    }
+    this.registry[host_language].push(extractor);
+  }
+}
+
+export const CODE_EXTRACTORS_MANAGER: JupyterFrontEndPlugin<ILSPCodeExtractorsManager> = {
+  id: ILSPCodeExtractorsManager.name,
+  requires: [],
+  activate: app => {
+    return new CodeExtractorsManager();
+  },
+  provides: ILSPCodeExtractorsManager,
+  autoStart: true
+};

packages/jupyterlab-lsp/src/extractors/regexp.ts

@@ -1,6 +1,6 @@
 import { IExtractedCode, IForeignCodeExtractor } from './types';
 import { position_at_offset } from '../positioning';
-import { replacer } from '../magics/overrides';
+import { replacer } from '../overrides/tokens';
 import { CodeEditor } from '@jupyterlab/codeeditor';
 
 export class RegExpForeignCodeExtractor implements IForeignCodeExtractor {

packages/jupyterlab-lsp/src/extractors/testutils.ts

@@ -0,0 +1,35 @@
+import { CodeEditor } from '@jupyterlab/codeeditor';
+import { IVirtualDocumentBlock, VirtualDocument } from '../virtual/document';
+import { expect } from 'chai';
+
+export function extract_code(document: VirtualDocument, code: string) {
+  return document.extract_foreign_code(
+    { value: code, ce_editor: null },
+    {
+      line: 0,
+      column: 0
+    }
+  );
+}
+
+export function get_the_only_pair(
+  foreign_document_map: Map<CodeEditor.IRange, IVirtualDocumentBlock>
+) {
+  expect(foreign_document_map.size).to.equal(1);
+
+  let range = foreign_document_map.keys().next().value;
+  let { virtual_document } = foreign_document_map.get(range);
+
+  return { range, virtual_document };
+}
+
+export function get_the_only_virtual(
+  foreign_document_map: Map<CodeEditor.IRange, IVirtualDocumentBlock>
+) {
+  let { virtual_document } = get_the_only_pair(foreign_document_map);
+  return virtual_document;
+}
+
+export function wrap_in_python_lines(line: string) {
+  return 'print("some code before")\n' + line + '\n' + 'print("and after")\n';
+}

packages/jupyterlab-lsp/src/extractors/types.ts

@@ -1,4 +1,5 @@
 import { CodeEditor } from '@jupyterlab/codeeditor';
+import { LanguageIdentifier } from '../lsp';
 
 export interface IExtractedCode {
   /**
@@ -42,7 +43,7 @@ export interface IForeignCodeExtractor {
   /**
    * The foreign language.
    */
-  language: string;
+  language: LanguageIdentifier;
 
   /**
    * Split the code into the host and foreign code (if any foreign code was detected)

packages/jupyterlab-lsp/src/features/diagnostics/diagnostics.spec.ts

@@ -11,8 +11,7 @@ import {
 } from '../../editor_integration/testutils';
 import { CodeMirrorEditor } from '@jupyterlab/codemirror';
 import { is_equal } from '../../positioning';
-import { language_specific_overrides } from '../../magics/defaults';
-import { foreign_code_extractors } from '../../extractors/defaults';
+import { foreign_code_extractors } from '../../transclusions/ipython/extractors';
 import * as lsProtocol from 'vscode-languageserver-protocol';
 import { CodeDiagnostics as LSPDiagnosticsSettings } from '../../_diagnostics';
 import { message_without_code } from './listing';
@@ -141,7 +140,7 @@ describe('Diagnostics', () => {
 
     beforeEach(() => {
       env = new NotebookFeatureTestEnvironment({
-        overrides_registry: language_specific_overrides,
+        overrides_registry: {},
         foreign_code_extractors
       });
       feature = env.init_integration({