Merge pull request #43 from lierdakil/various-tweaks

Author: aminya

.prettierignore

@@ -7,4 +7,5 @@ coverage
 build
 dist
 lib
+modules
 .rollup.cache

package.json

@@ -27,8 +27,8 @@
     "test": "atom --test spec",
     "clean": "shx rm -rf dist modules",
     "babel": "npm run clean && shx cp -r lib dist && cross-env NODE_ENV=development cross-env BABEL_KEEP_MODULES=false babel dist --out-dir dist",
-    "tsc.cjs": "tsc -p ./src/tsconfig.json --outDir dist || echo done",
-    "tsc.es": "tsc --module esnext -p ./src/tsconfig.json --outDir modules || echo done",
+    "tsc.cjs": "tsc -p ./src/tsconfig.json",
+    "tsc.es": "tsc --module esnext -p ./src/tsconfig.es.json",
     "tsc": "npm run tsc.cjs && npm run tsc.es",
     "dev": "npm run clean && npm run tsc && cross-env NODE_ENV=development cross-env BABEL_KEEP_MODULES=true rollup -c -w",
     "build": "npm run clean && npm run tsc && cross-env NODE_ENV=production cross-env BABEL_KEEP_MODULES=true rollup -c ",
@@ -43,31 +43,32 @@
   "devDependencies": {
     "@types/atom": "1.40.7",
     "@types/dompurify": "2.2.1",
+    "@types/jasmine": "^3.6.3",
     "@types/marked": "^1.2.2",
     "@types/node": "^14.14.25",
     "atom-ide-base": "^2.2.0",
-    "atom-languageclient": "^1.0.6",
-    "@types/jasmine": "^3.6.3",
     "atom-jasmine3-test-runner": "^5.1.8",
-    "typescript": "^4.1.3",
-    "tslib": "^2.1.0",
-    "prettier": "^2.2.1",
+    "atom-languageclient": "^1.0.6",
+    "build-commit": "0.1.4",
+    "cross-env": "^7.0.3",
     "eslint": "7.19.0",
     "eslint-config-atomic": "^1.9.0",
+    "npm-check-updates": "11.1.1",
+    "prettier": "^2.2.1",
     "rollup": "^2.38.4",
     "rollup-plugin-atomic": "^2.0.1",
     "shx": "^0.3.3",
-    "cross-env": "^7.0.3",
-    "npm-check-updates": "11.1.1",
-    "build-commit": "0.1.4"
+    "tslib": "^2.1.0",
+    "typescript": "^4.1.3"
   },
   "activationHooks": [
     "core:loaded-shell-environment"
   ],
   "providedServices": {
     "markdown-renderer": {
       "versions": {
-        "1.0.0": "provideMarkdownRenderer"
+        "1.0.0": "provideMarkdownRenderer",
+        "1.1.0": "provideMarkdownRenderer"
       }
     }
   },

rollup.config.js

@@ -1,6 +1,7 @@
 import { createPlugins } from "rollup-plugin-atomic"
 
-const plugins = createPlugins(["ts", "babel"])
+const plugins = createPlugins([["ts", { tsconfig: "./src/tsconfig.json" }, true], "babel"])
+const pluginsEs = createPlugins([["ts", { tsconfig: "./src/tsconfig.es.json" }, true], "babel"])
 
 export default [
   {
@@ -42,6 +43,6 @@ export default [
     ],
     // loaded externally
     external: ["atom"],
-    plugins: plugins,
+    plugins: pluginsEs,
   },
 ]

src/highlighter.ts

@@ -0,0 +1,89 @@
+import { TextBuffer, LanguageMode } from "atom"
+import { eventLoopYielder, maxTimeError } from "./utils/event-loop-yielder"
+
+declare module "atom" {
+  interface GrammarRegistry {
+    grammarForId(id: string): Grammar
+    languageModeForGrammarAndBuffer(g: Grammar, b: TextBuffer): LanguageMode
+  }
+  interface LanguageMode {
+    readonly fullyTokenized?: boolean
+    readonly tree?: boolean
+    onDidTokenize(cb: () => void): Disposable
+    buildHighlightIterator(): HighlightIterator
+    classNameForScopeId(id: ScopeId): string
+    startTokenizing?(): void
+  }
+  interface HighlightIterator {
+    seek(pos: { row: number; column: number }): void
+    getPosition(): { row: number; column: number }
+    getOpenScopeIds?(): ScopeId[]
+    getCloseScopeIds?(): ScopeId[]
+    moveToSuccessor(): void
+  }
+  interface ScopeId {}
+}
+
+export async function highlightTreeSitter(sourceCode: string, scopeName: string) {
+  const yielder = eventLoopYielder(100, 5000)
+  const buf = new TextBuffer()
+  try {
+    const grammar = atom.grammars.grammarForId(scopeName)
+    const lm = atom.grammars.languageModeForGrammarAndBuffer(grammar, buf)
+    buf.setLanguageMode(lm)
+    buf.setText(sourceCode)
+    const end = buf.getEndPosition()
+    if (lm.startTokenizing) lm.startTokenizing()
+    await tokenized(lm)
+    const iter = lm.buildHighlightIterator()
+    if (iter.getOpenScopeIds && iter.getCloseScopeIds) {
+      let pos = { row: 0, column: 0 }
+      iter.seek(pos)
+      const res = []
+      while (pos.row < end.row || (pos.row === end.row && pos.column <= end.column)) {
+        res.push(
+          ...iter.getCloseScopeIds().map(() => "</span>"),
+          ...iter.getOpenScopeIds().map((x) => `<span class="${lm.classNameForScopeId(x)}">`)
+        )
+        iter.moveToSuccessor()
+        const nextPos = iter.getPosition()
+        res.push(escapeHTML(buf.getTextInRange([pos, nextPos])))
+
+        if (!(await yielder())) {
+          console.error(maxTimeError("Atom-IDE-Markdown-Service: Highlighter", 5))
+          break
+        }
+        pos = nextPos
+      }
+      return res.join("")
+    } else {
+      return sourceCode
+    }
+  } finally {
+    buf.destroy()
+  }
+}
+
+async function tokenized(lm: LanguageMode) {
+  return new Promise((resolve) => {
+    if (lm.fullyTokenized || lm.tree) {
+      resolve(undefined)
+    } else if (lm.onDidTokenize) {
+      const disp = lm.onDidTokenize(() => {
+        disp.dispose()
+        resolve(undefined)
+      })
+    } else {
+      resolve(undefined) // null language mode
+    }
+  })
+}
+
+function escapeHTML(str: string) {
+  return str
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#039;")
+}

src/renderer.ts

@@ -1,4 +1,3 @@
-import { TextEditor } from "atom"
 import marked from "marked"
 
 /**
@@ -7,71 +6,34 @@ import marked from "marked"
  * @type {DOMPurify}
  */
 import DOMPurify from "dompurify"
-
-/**
- * A function that resolves once the given editor has tokenized
- * @param editor
- */
-export async function editorTokenized(editor: TextEditor) {
-  return new Promise((resolve) => {
-    const languageMode = editor.getBuffer().getLanguageMode()
-    const nextUpdatePromise = editor.component.getNextUpdatePromise()
-    if ("fullyTokenized" in languageMode || "tree" in languageMode) {
-      resolve(nextUpdatePromise)
-    } else {
-      const disp = editor.onDidTokenize(() => {
-        disp.dispose()
-        resolve(nextUpdatePromise)
-      })
-    }
-  })
-}
-
-/**
- * Highlights the given code with the given scope name (language)
- * @param code the given code as string
- * @param scopeName the language to highlight the code for
- */
-export async function highlight(code: string, scopeName: string) {
-  const ed = new TextEditor({
-    readonly: true,
-    keyboardInputEnabled: false,
-    showInvisibles: false,
-    tabLength: atom.config.get("editor.tabLength"),
-  })
-  const el = atom.views.getView(ed)
-  try {
-    el.setUpdatedSynchronously(true)
-    atom.grammars.assignLanguageMode(ed.getBuffer(), scopeName)
-    ed.setText(code)
-    ed.scrollToBufferPosition(ed.getBuffer().getEndPosition())
-    atom.views.getView(atom.workspace).appendChild(el)
-    await editorTokenized(ed)
-    return Array.from(el.querySelectorAll(".line:not(.dummy)")).map((x) => x.innerHTML)
-  } finally {
-    el.remove()
-  }
-}
+import { highlightTreeSitter } from "./highlighter"
 
 marked.setOptions({
   breaks: true,
 })
 
+export type DOMPurifyConfig = Omit<DOMPurify.Config, "RETURN_DOM" | "RETURN_DOM_FRAGMENT" | "RETURN_TRUSTED_TYPE">
+
 /**
  * renders markdown to safe HTML asynchronously
  * @param markdownText the markdown text to render
  * @param scopeName scope name used for highlighting the code
+ * @param purifyConfig (optional) configuration object for DOMPurify
  * @return the html string containing the result
  */
-function internalRender(markdownText: string, scopeName: string = "text.plain"): Promise<string> {
+export async function render(
+  markdownText: string,
+  scopeName: string = "text.plain",
+  domPurifyConfig?: DOMPurifyConfig
+): Promise<string> {
   return new Promise((resolve, reject) => {
     marked(
       markdownText,
       {
-        highlight: function (code, lang, callback) {
-          highlight(code, scopeName)
+        highlight: function (code, _lang, callback) {
+          highlightTreeSitter(code, scopeName)
             .then((codeResult) => {
-              callback!(null, codeResult.join("\n"))
+              callback!(null, codeResult)
             })
             .catch((e) => {
               callback!(e)
@@ -83,21 +45,10 @@ function internalRender(markdownText: string, scopeName: string = "text.plain"):
           reject(e)
         }
         // sanitization
-        html = DOMPurify.sanitize(html)
+        html = domPurifyConfig ? DOMPurify.sanitize(html, domPurifyConfig) : DOMPurify.sanitize(html)
 
         return resolve(html)
       }
     )
   })
 }
-
-/**
- * renders the markdown text to html
- * @param markdownText the markdown text to render
- * @param grammar the default grammar used in code sections that have no specific grammar set
- * @return the inner HTML text of the rendered section
- */
-export async function render(markdownText: string, grammar: string): Promise<string> {
-  const html = await internalRender(markdownText, grammar)
-  return html
-}

src/tsconfig.es.json

@@ -0,0 +1,7 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "esnext",
+    "outDir": "../modules"
+  }
+}

src/tsconfig.json

@@ -26,7 +26,8 @@
     "module": "commonjs",
     "moduleResolution": "node",
     "importHelpers": false,
-    "outDir": "../dist"
+    "outDir": "../dist",
+    "skipLibCheck": true
   },
   "compileOnSave": false
 }

src/utils/event-loop-yielder.ts

@@ -0,0 +1,31 @@
+/**
+ * A helper to allow the JavaScript event loop continue for a given interval between each
+ * iteration of a CPU intensive loop. If the time spent in the loop reaches the given
+ * maxTime, the operation is killed.
+ *
+ * @returns An async function to call inside your heavy loop. It will return `false` if
+ *     the operation has exceeded the given max time (`true` otherwise).
+ */
+export function eventLoopYielder(delayMs: number, maxTimeMs: number) {
+  const started = performance.now()
+  let lastYield = started
+  return async function (): Promise<boolean> {
+    const now = performance.now()
+    if (now - lastYield > delayMs) {
+      await new Promise(setImmediate)
+      lastYield = now
+    }
+    return now - started <= maxTimeMs
+  }
+}
+
+/** Throws maximum time reached error */
+export function maxTimeError(name: string, timeS: number) {
+  const err = new Error("Max time reached")
+  atom.notifications.addError(`${name} took more than ${timeS} seconds to complete`, {
+    dismissable: true,
+    description: `${name} took too long to complete and was terminated.`,
+    stack: err.stack,
+  })
+  return err
+}

src/utils/utils.ts

@@ -1,4 +1,4 @@
-const scopesByFenceName = {
+const scopesByFenceName: { [key: string]: string | undefined } = {
   bash: "source.shell",
   sh: "source.shell",
   powershell: "source.powershell",
@@ -42,11 +42,7 @@ const scopesByFenceName = {
 
 export function scopeForFenceName(fenceName: string): string {
   fenceName = fenceName.toLowerCase()
-  let result = `source.${fenceName}`
-  if (scopesByFenceName[fenceName] != null) {
-    result = scopesByFenceName[fenceName]
-  }
-  return result
+  return scopesByFenceName[fenceName] ?? `source.${fenceName}`
 }
 
 export function fenceNameForScope(scope: string): string {