Improve syntax highlighting in editors covered by foreign extractors

Author: krassowski

packages/jupyterlab-lsp/package.json

@@ -29,11 +29,12 @@
   },
   "scripts": {
     "build": "jlpm build:schema && tsc -b",
-    "build:schema": "jlpm build:schema-backend && jlpm build:schema-completion && jlpm build:schema-hover && jlpm build:schema-diagnostics",
+    "build:schema": "jlpm build:schema-backend && jlpm build:schema-completion && jlpm build:schema-hover && jlpm build:schema-diagnostics && jlpm build:schema-syntax_highlighting",
     "build:schema-backend": "json2ts ../../py_src/jupyter_lsp/schema/schema.json --unreachableDefinitions | prettier --stdin-filepath _schema.d.ts > src/_schema.d.ts",
     "build:schema-completion": "json2ts schema/completion.json | prettier --stdin-filepath _completion.d.ts > src/_completion.d.ts",
     "build:schema-hover": "json2ts schema/hover.json | prettier --stdin-filepath _hover.d.ts > src/_hover.d.ts",
     "build:schema-diagnostics": "json2ts schema/diagnostics.json | prettier --stdin-filepath _diagnostics.d.ts > src/_diagnostics.d.ts",
+    "build:schema-syntax_highlighting": "json2ts schema/syntax_highlighting.json | prettier --stdin-filepath _syntax_highlighting.d.ts > src/_syntax_highlighting.d.ts",
     "bundle": "npm pack .",
     "clean": "rimraf lib",
     "lab:link": "jupyter labextension link . --no-build",

packages/jupyterlab-lsp/schema/syntax_highlighting.json

@@ -0,0 +1,16 @@
+{
+  "jupyter.lab.setting-icon": "lsp:syntax-highlight",
+  "jupyter.lab.setting-icon-label": "Language integration",
+  "title": "Code Syntax",
+  "description": "LSP code syntax highlighting settings.",
+  "type": "object",
+  "properties": {
+    "foreignCodeThreshold": {
+      "title": "Threshold of foreign code coverage for changing the mode in an editor",
+      "type": "number",
+      "default": 0.5,
+      "description": "If a code editor includes a code fragment in another language (for example a %%markdown magic in IPython) with appropriate foreign code extractor defined , and the extend of this code (coverage of the editor) passes the threshold, the syntax highlighting (i.e. the mode) will change to provide highlighting for the language of the foreign code."
+    }
+  },
+  "jupyter.lab.shortcuts": []
+}

packages/jupyterlab-lsp/src/editor_integration/codemirror.ts

@@ -363,7 +363,10 @@ export abstract class CodeMirrorIntegration
     let raw_value = doc.getValue('\n');
     // extract foreign documents and substitute magics,
     // as it was done when the shadow virtual document was being created
-    let { lines } = document.prepare_code_block(raw_value, editor);
+    let { lines } = document.prepare_code_block({
+      value: raw_value,
+      ce_editor: editor
+    });
     let old_value = lines.join('\n');
 
     if (is_whole_document_edit) {

packages/jupyterlab-lsp/src/extractors/defaults.spec.ts

@@ -11,10 +11,13 @@ describe('Default extractors', () => {
   let document: VirtualDocument;
 
   function extract(code: string) {
-    return document.extract_foreign_code(code, null, {
-      line: 0,
-      column: 0
-    });
+    return document.extract_foreign_code(
+      { value: code, ce_editor: null },
+      {
+        line: 0,
+        column: 0
+      }
+    );
   }
 
   function get_the_only_pair(

packages/jupyterlab-lsp/src/extractors/types.ts

@@ -25,7 +25,7 @@ export interface IExtractedCode {
  *
  * The typical examples are:
  *  - (X=CSS< Y=HTML), or
- *  - (X=JavaScrip, Y=HTML),
+ *  - (X=JavaScript, Y=HTML),
  *
  * while in the data analysis realm, examples include:
  *   - (X=R, Y=IPython),

packages/jupyterlab-lsp/src/features/syntax_highlighting.ts

@@ -0,0 +1,113 @@
+import { CodeMirrorIntegration } from '../editor_integration/codemirror';
+import {
+  JupyterFrontEnd,
+  JupyterFrontEndPlugin
+} from '@jupyterlab/application';
+import { ILSPFeatureManager, PLUGIN_ID } from '../tokens';
+import { LabIcon } from '@jupyterlab/ui-components';
+import syntaxSvg from '../../style/icons/syntax-highlight.svg';
+import {
+  IEditorIntegrationOptions,
+  IFeatureLabIntegration,
+  IFeatureSettings
+} from '../feature';
+import { VirtualDocument } from '../virtual/document';
+import { CodeMirrorEditor } from '@jupyterlab/codemirror';
+import CodeMirror from 'codemirror';
+import { IEditorMimeTypeService } from '@jupyterlab/codeeditor/lib/mimetype';
+import { IEditorServices } from '@jupyterlab/codeeditor/lib/tokens';
+import { CodeSyntax as LSPSyntaxHighlightingSettings } from '../_syntax_highlighting';
+
+export const syntaxHighlightingIcon = new LabIcon({
+  name: 'lsp:syntax-highlighting',
+  svgstr: syntaxSvg
+});
+
+const FEATURE_ID = PLUGIN_ID + ':syntax-highlighting';
+
+export class CMSyntaxHighlighting extends CodeMirrorIntegration {
+  lab_integration: SyntaxLabIntegration;
+  settings: IFeatureSettings<LSPSyntaxHighlightingSettings>;
+
+  constructor(options: IEditorIntegrationOptions) {
+    super(options);
+    this.virtual_document.changed.connect(this.update_mode.bind(this), this);
+  }
+
+  private get_mode(language: string) {
+    let mimetype = this.lab_integration.mimeTypeService.getMimeTypeByLanguage({
+      name: language
+    });
+
+    if (!mimetype || mimetype == 'text/plain') {
+      // if a mimetype cannot be found it will be 'text/plain', therefore do
+      // not change mode to text/plain, as this could be a step backwards for
+      // the user experience
+      return;
+    }
+
+    return CodeMirror.findModeByMIME(mimetype);
+  }
+
+  update_mode(doc: VirtualDocument, changed_document: VirtualDocument) {
+    let root = this.virtual_document;
+    for (let map of root.foreign_document_maps) {
+      for (let [range, block] of map.entries()) {
+        let ce_editor = block.editor;
+        // get the ce_editor from block
+        let editor = (ce_editor as CodeMirrorEditor).editor;
+        let lines = editor.getValue('\n');
+        let total_area = lines.concat('').length;
+
+        let covered_area =
+          ce_editor.getOffsetAt(range.end) - ce_editor.getOffsetAt(range.start);
+
+        let coverage = covered_area / total_area;
+
+        let language = block.virtual_document.language;
+
+        let mode = this.get_mode(language);
+
+        // if not highlighting mode available, skip this editor
+        if (typeof mode === 'undefined') {
+          continue;
+        }
+
+        // change the mode if the majority of the code is the foreign code
+        if (coverage > this.settings.composite.foreignCodeThreshold) {
+          editor.setOption('mode', mode.mime);
+        }
+      }
+    }
+  }
+}
+
+class SyntaxLabIntegration implements IFeatureLabIntegration {
+  // TODO: we could accept custom mimetype mapping from settings
+  settings: IFeatureSettings<LSPSyntaxHighlightingSettings>;
+
+  constructor(public mimeTypeService: IEditorMimeTypeService) {}
+}
+
+export const SYNTAX_HIGHLIGHTING_PLUGIN: JupyterFrontEndPlugin<void> = {
+  id: FEATURE_ID,
+  requires: [ILSPFeatureManager, IEditorServices],
+  autoStart: true,
+  activate: (
+    app: JupyterFrontEnd,
+    featureManager: ILSPFeatureManager,
+    editorServices: IEditorServices
+  ) => {
+    featureManager.register({
+      feature: {
+        editorIntegrationFactory: new Map([
+          ['CodeMirrorEditor', CMSyntaxHighlighting]
+        ]),
+        commands: [],
+        id: FEATURE_ID,
+        name: 'Syntax highlighting',
+        labIntegration: new SyntaxLabIntegration(editorServices.mimeTypeService)
+      }
+    });
+  }
+};

packages/jupyterlab-lsp/src/index.ts

@@ -35,6 +35,7 @@ import { LabIcon } from '@jupyterlab/ui-components';
 import codeCheckSvg from '../style/icons/code-check.svg';
 import { DIAGNOSTICS_PLUGIN } from './features/diagnostics';
 import { COMPLETION_PLUGIN } from './features/completion';
+import { SYNTAX_HIGHLIGHTING_PLUGIN } from './features/syntax_highlighting';
 
 export const codeCheckIcon = new LabIcon({
   name: 'lsp:codeCheck',
@@ -214,7 +215,8 @@ const default_features: JupyterFrontEndPlugin<void>[] = [
   HOVER_PLUGIN,
   RENAME_PLUGIN,
   HIGHLIGHTS_PLUGIN,
-  DIAGNOSTICS_PLUGIN
+  DIAGNOSTICS_PLUGIN,
+  SYNTAX_HIGHLIGHTING_PLUGIN
 ];
 
 const plugins: JupyterFrontEndPlugin<any>[] = [

packages/jupyterlab-lsp/src/virtual/document.spec.ts

@@ -60,10 +60,13 @@ describe('VirtualDocument', () => {
       let {
         cell_code_kept,
         foreign_document_map
-      } = document.extract_foreign_code(R_LINE_MAGICS, null, {
-        line: 0,
-        column: 0
-      });
+      } = document.extract_foreign_code(
+        { value: R_LINE_MAGICS, ce_editor: null },
+        {
+          line: 0,
+          column: 0
+        }
+      );
 
       // note R cell lines are kept in code (keep_in_host=true)
       expect(cell_code_kept).to.equal(R_LINE_MAGICS);
@@ -87,30 +90,31 @@ describe('VirtualDocument', () => {
     let ce_editor_for_cell_3 = {} as CodeEditor.IEditor;
     let ce_editor_for_cell_4 = {} as CodeEditor.IEditor;
     // first block
-    document.append_code_block(
-      'test line in Python 1\n%R 1st test line in R line magic 1',
-      ce_editor_for_cell_1
-    );
+    document.append_code_block({
+      value: 'test line in Python 1\n%R 1st test line in R line magic 1',
+      ce_editor: ce_editor_for_cell_1
+    });
     // second block
-    document.append_code_block(
-      'test line in Python 2\n%R 1st test line in R line magic 2',
-      ce_editor_for_cell_2
-    );
+    document.append_code_block({
+      value: 'test line in Python 2\n%R 1st test line in R line magic 2',
+      ce_editor: ce_editor_for_cell_2
+    });
     // third block
-    document.append_code_block(
-      'test line in Python 3\n%R -i imported_variable 1st test line in R line magic 3',
-      ce_editor_for_cell_2
-    );
+    document.append_code_block({
+      value:
+        'test line in Python 3\n%R -i imported_variable 1st test line in R line magic 3',
+      ce_editor: ce_editor_for_cell_2
+    });
     // fourth block
-    document.append_code_block(
-      '%%R\n1st test line in R cell magic 1',
-      ce_editor_for_cell_3
-    );
+    document.append_code_block({
+      value: '%%R\n1st test line in R cell magic 1',
+      ce_editor: ce_editor_for_cell_3
+    });
     // fifth block
-    document.append_code_block(
-      '%%R -i imported_variable\n1st test line in R cell magic 2',
-      ce_editor_for_cell_4
-    );
+    document.append_code_block({
+      value: '%%R -i imported_variable\n1st test line in R cell magic 2',
+      ce_editor: ce_editor_for_cell_4
+    });
   };
 
   describe('transform_virtual_to_editor', () => {

packages/jupyterlab-lsp/src/virtual/document.ts

@@ -32,12 +32,18 @@ interface IVirtualLine {
   editor: CodeEditor.IEditor;
 }
 
+export interface ICodeBlockOptions {
+  ce_editor: CodeEditor.IEditor;
+  value: string;
+}
+
 export interface IVirtualDocumentBlock {
   /**
    * Line corresponding to the block in the entire foreign document
    */
   virtual_line: number;
   virtual_document: VirtualDocument;
+  editor: CodeEditor.IEditor;
 }
 
 export type ForeignDocumentsMap = Map<IRange, IVirtualDocumentBlock>;
@@ -480,15 +486,16 @@ export class VirtualDocument {
   }
 
   extract_foreign_code(
-    cell_code: string,
-    ce_editor: CodeEditor.IEditor,
+    block: ICodeBlockOptions,
     editor_shift: CodeEditor.IPosition
   ) {
     let foreign_document_map = new Map<
       CodeEditor.IRange,
       IVirtualDocumentBlock
     >();
 
+    let cell_code = block.value;
+
     for (let extractor of this.foreign_extractors) {
       // first, check if there is any foreign code:
 
@@ -506,15 +513,18 @@ export class VirtualDocument {
 
           foreign_document_map.set(result.range, {
             virtual_line: foreign_document.last_virtual_line,
-            virtual_document: foreign_document
+            virtual_document: foreign_document,
+            editor: block.ce_editor
           });
           let foreign_shift = {
             line: editor_shift.line + result.range.start.line,
             column: editor_shift.column + result.range.start.column
           };
           foreign_document.append_code_block(
-            result.foreign_code,
-            ce_editor,
+            {
+              value: result.foreign_code,
+              ce_editor: block.ce_editor
+            },
             foreign_shift,
             result.virtual_shift
           );
@@ -548,19 +558,17 @@ export class VirtualDocument {
   }
 
   prepare_code_block(
-    cell_code: string,
-    ce_editor: CodeEditor.IEditor,
+    block: ICodeBlockOptions,
     editor_shift: CodeEditor.IPosition = { line: 0, column: 0 }
   ) {
     let lines: Array<string>;
     let skip_inspect: Array<Array<VirtualDocument.id_path>>;
 
     let { cell_code_kept, foreign_document_map } = this.extract_foreign_code(
-      cell_code,
-      ce_editor,
+      block,
       editor_shift
     );
-    cell_code = cell_code_kept;
+    let cell_code = cell_code_kept;
 
     // cell magics are replaced if requested and matched
     let cell_override = this.cell_magics_overrides.override_for(cell_code);
@@ -590,16 +598,17 @@ export class VirtualDocument {
   }
 
   append_code_block(
-    cell_code: string,
-    ce_editor: CodeEditor.IEditor,
+    block: ICodeBlockOptions,
     editor_shift: CodeEditor.IPosition = { line: 0, column: 0 },
     virtual_shift?: CodeEditor.IPosition
   ) {
+    let cell_code = block.value;
+    let ce_editor = block.ce_editor;
+
     let source_cell_lines = cell_code.split('\n');
 
     let { lines, foreign_document_map, skip_inspect } = this.prepare_code_block(
-      cell_code,
-      ce_editor,
+      block,
       editor_shift
     );
 
@@ -909,10 +918,7 @@ export class UpdateManager {
               block: code_block,
               virtual_document: this.virtual_document
             });
-            this.virtual_document.append_code_block(
-              code_block.value,
-              code_block.ce_editor
-            );
+            this.virtual_document.append_code_block(code_block);
           }
 
           this.update_finished.emit(blocks);
@@ -935,8 +941,3 @@ export class UpdateManager {
     return update;
   }
 }
-
-export interface ICodeBlockOptions {
-  ce_editor: CodeEditor.IEditor;
-  value: string;
-}