refactor code around conversion to prosemirror to make it easier for me to understand

Author: vezwork

packages/editor/src/api/pandoc.ts

@@ -24,7 +24,7 @@ import { kLinkChildren } from './link';
 
 import { BibliographyResult, PandocServer, PandocApiVersion, PandocAst, PandocToken } from 'editor-types';
 import { PandocExtensions } from './pandoc-types';
-export type { BibliographyResult, PandocServer,PandocApiVersion, PandocAst, PandocToken, PandocExtensions };
+export type { BibliographyResult, PandocServer, PandocApiVersion, PandocAst, PandocToken, PandocExtensions };
 
 
 export function imageAttributesAvailable(pandocExtensions: PandocExtensions) {
@@ -257,52 +257,73 @@ export function stringifyTokens(c: PandocToken[], unemoji = false): string {
 }
 
 export function forEachToken(tokens: PandocToken[], f: (tok: PandocToken) => void) {
-  mapTokens(tokens, (tok: PandocToken) => {
+  mapTokensRecursive(tokens, (tok: PandocToken) => {
     f(tok);
     return tok;
   });
 }
 
-export function mapTokens(tokens: PandocToken[], f: (tok: PandocToken) => PandocToken) {
-  function isToken(val: unknown) {
-    if (val !== null && typeof val === 'object') {
-      return Object.prototype.hasOwnProperty.call(val, 't');
-    } else {
-      return false;
-    }
-  }
+const isObject = (val: unknown): val is object =>
+  val !== null &&
+  typeof val === 'object';
 
-  function tokenHasChildren(tok: PandocToken) {
-    return tok !== null && typeof tok === 'object' && Array.isArray(tok.c);
-  }
+const isToken = (val: unknown): val is PandocToken =>
+  isObject(val) &&
+  val.hasOwnProperty('t');
 
-  function mapValue(val: unknown): unknown {
-    if (isToken(val)) {
-      return mapToken(val as PandocToken);
-    } else if (Array.isArray(val)) {
-      return val.map(mapValue);
-    } else {
-      return val;
-    }
-  }
+/**
+ * @param f A function to be enhanced.
+ * @returns An enhanced function. If the input to the enhanced function is a
+ * non-array value, then `f` is simply applied to it.
+ * If the input to the enhanced function is an array, optionally of recursively nested arrays,
+ * then `f` is applied to all non-array values in the array and all recursively nested arrays.
+ */
+const mapRecursiveArray = (f: (v: unknown) => any) => {
+
+  const mappedF = (val: unknown): any =>
+    Array.isArray(val) ?
+      val.map(mappedF) : f(val);
+
+  return mappedF;
+};
 
-  function mapToken(tok: PandocToken): PandocToken {
+/**
+ * @param f A function from PandocToken to PandocToken to be enhanced.
+ * @returns An enhanced function that applies `f` to an input token, and
+ * then applies itself to any tokens in the output's content i.e. `mappedTok.c`,
+ * even if token is inside nested arrays in the output's content.
+ */
+const mapTokenAndContentTokensRecursively = (f: (tok: PandocToken) => PandocToken) => {
+  const mappedF = (tok: PandocToken) => {
     const mappedTok = f(tok);
-    if (tokenHasChildren(mappedTok)) {
-      mappedTok.c = mappedTok.c.map(mapValue);
-    }
+
+    const recursiveMappedF = mapRecursiveArray((v) => isToken(v) ? mappedF(v) : v);
+    mappedTok.c = recursiveMappedF(mappedTok.c);
+
     return mappedTok;
-  }
+  };
 
-  return tokens.map(mapToken);
+  return mappedF;
+};
+
+/**
+ *
+ * @param tokens An array of `PandocToken`s
+ * @param f A function to be applied to each token that returns a new or modified token
+ * @returns The array of tokens resulting from applying `f` to each token in `token`s, and
+ * also recursively applying `f` to any tokens in the resulting token's content (even if the content
+ * consists of recursively nested arrays, and the tokens are in the nested arrays).
+ */
+export function mapTokensRecursive(tokens: PandocToken[], f: (tok: PandocToken) => PandocToken) {
+  return tokens.map(mapTokenAndContentTokensRecursively(f));
 }
 
 export function tokenTextEscaped(t: PandocToken) {
   return t.c.replace(/\\/g, `\\\\`);
 }
 
 // sort marks by priority (in descending order)
-export function marksByPriority(marks: readonly Mark[], markWriters: { [key: string]: PandocMarkWriter }) {
+export function marksByPriority(marks: readonly Mark[], markWriters: { [key: string]: PandocMarkWriter; }) {
   return Array.prototype.sort.call(marks, (a: Mark, b: Mark) => {
     const aPriority = markWriters[a.type.name].priority;
     const bPriority = markWriters[b.type.name].priority;

packages/editor/src/api/pandoc_capsule.ts

@@ -17,7 +17,7 @@ import { Schema } from 'prosemirror-model';
 
 import { base64Encode, base64Decode } from './base64';
 
-import { PandocToken, ProsemirrorWriter, mapTokens, PandocTokenType } from './pandoc';
+import { PandocToken, ProsemirrorWriter, mapTokensRecursive, PandocTokenType } from './pandoc';
 
 // constants used for creating/consuming capsules
 const kFieldDelimiter = '\n';
@@ -65,7 +65,7 @@ export interface PandocBlockCapsuleFilter {
     p2: string,
     p3: string,
     p4: string,
-  ) => { prefix: string; source: string; suffix: string };
+  ) => { prefix: string; source: string; suffix: string; };
 
   // provide a (text) envelope around the capsule, e.g.
   //  - newlines to ensure that yaml is parsed as a standalone paragraph;
@@ -88,6 +88,15 @@ export interface PandocBlockCapsuleFilter {
   writeNode: (schema: Schema, writer: ProsemirrorWriter, capsule: PandocBlockCapsule) => void;
 }
 
+// default extractor
+const defaultExtractor = (_match: string, p1: string, p2: string, p3: string) => {
+  return {
+    prefix: p1,
+    source: p2,
+    suffix: p3,
+  };
+};
+
 // transform the passed markdown to include base64 encoded block capsules as specified by the
 // provided capsule filter. capsules are used to hoist block types that we don't want pandoc
 // to see (e.g. yaml metadata or Rmd chunks) out of the markdown, only to be re-inserted
@@ -99,15 +108,6 @@ export function pandocMarkdownWithBlockCapsules(
   markdown: string,
   capsuleFilter: PandocBlockCapsuleFilter,
 ) {
-  // default extractor
-  const defaultExtractor = (_match: string, p1: string, p2: string, p3: string) => {
-    return {
-      prefix: p1,
-      source: p2,
-      suffix: p3,
-    };
-  };
-
   // find the original position of all the matches
   const positions: number[] = [];
   let match = capsuleFilter.match.exec(original);
@@ -179,41 +179,27 @@ export function pandocMarkdownWithBlockCapsules(
   });
 }
 
-// block capsules can also end up not as block tokens, but rather as text within another
-// token (e.g. within a backtick code block or raw_block). this function takes a set
-// of pandoc tokens and recursively converts block capsules that aren't of type
-// PandocTokenType.Str (which is what we'd see in a paragraph) into their original form
-export function resolvePandocBlockCapsuleText(
-  tokens: PandocToken[],
-  filters: readonly PandocBlockCapsuleFilter[],
-): PandocToken[] {
-  // process all tokens
-  return mapTokens(tokens, token => {
-    // look for non-string pandoc tokens
-    if (token.t !== PandocTokenType.Str && token.c) {
-      if (typeof token.c === 'string') {
-        token.c = decodeBlockCapsuleText(token.c, token, filters);
-      } else if (Array.isArray(token.c)) {
-        const children = token.c.length;
-        for (let i = 0; i < children; i++) {
-          if (typeof token.c[i] === 'string') {
-            token.c[i] = decodeBlockCapsuleText(token.c[i], token, filters);
-          }
-        }
-      }
-    }
-
-    return token;
-  });
-}
-
 // decode the text capsule by running all of the filters (as there could be nesting)
 export function decodeBlockCapsuleText(text: string, tok: PandocToken, filters: readonly PandocBlockCapsuleFilter[]) {
   filters.forEach(filter => {
     text = filter.handleText(text, tok);
   });
   return text;
 }
+const resolveTokenBlockCapsuleText = (token: PandocToken, filters: readonly PandocBlockCapsuleFilter[]) => ({
+  t: token.t,
+  c: token.t !== PandocTokenType.Str && typeof token.c === 'string' ?
+    decodeBlockCapsuleText(token.c, token, filters) :
+    token.c
+});
+// block capsules can also end up not as block tokens, but rather as text within another
+// token (e.g. within a backtick code block or raw_block). this function takes a set
+// of pandoc tokens and recursively converts block capsules that aren't of type
+// PandocTokenType.Str (which is what we'd see in a paragraph) into their original form
+export const resolvePandocBlockCapsuleText = (
+  tokens: PandocToken[],
+  filters: readonly PandocBlockCapsuleFilter[],
+) => mapTokensRecursive(tokens, token => resolveTokenBlockCapsuleText(token, filters));
 
 export function blockCapsuleTextHandler(type: string, pattern: RegExp, textFilter?: (text: string) => string) {
   return (text: string, tok: PandocToken): string => {
@@ -263,12 +249,12 @@ export function blockCapsuleParagraphTokenHandler(type: string) {
 export function encodedBlockCapsuleRegex(prefix?: string, suffix?: string, flags?: string) {
   return new RegExp(
     (prefix || '') +
-      kBlockCapsuleSentinel +
-      kValueDelimiter +
-      '((?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?)' +
-      kValueDelimiter +
-      kBlockCapsuleSentinel +
-      (suffix || ''),
+    kBlockCapsuleSentinel +
+    kValueDelimiter +
+    '((?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?)' +
+    kValueDelimiter +
+    kBlockCapsuleSentinel +
+    (suffix || ''),
     flags,
   );
 }

packages/editor/src/nodes/list/list-checked.ts

@@ -20,15 +20,15 @@ import { findParentNodeOfType, NodeWithPos, setTextSelection } from 'prosemirror
 import { InputRule, wrappingInputRule } from 'prosemirror-inputrules';
 
 import { ProsemirrorCommand, EditorCommandId } from '../../api/command';
-import { PandocToken, mapTokens } from '../../api/pandoc';
+import { PandocToken, mapTokensRecursive } from '../../api/pandoc';
 
 // custom NodeView that accomodates display / interaction with item check boxes
 export class CheckedListItemNodeView implements NodeView {
   public readonly dom: HTMLElement;
   public readonly contentDOM: HTMLElement;
 
   constructor(node: ProsemirrorNode, view: EditorView, getPos: () => number) {
-    
+
     // create root li element
     this.dom = window.document.createElement('li');
     if (node.attrs.tight) {
@@ -167,7 +167,7 @@ export function checkedListItemInputRule() {
 }
 
 export interface InputRuleWithHandler extends InputRule {
-  handler: (state: EditorState, match: RegExpMatchArray, start: number, end: number) => Transaction
+  handler: (state: EditorState, match: RegExpMatchArray, start: number, end: number) => Transaction;
 }
 
 // allow users to begin a new checked list by typing [x] or [ ] at the beginning of a line
@@ -212,13 +212,13 @@ export function fragmentWithCheck(schema: Schema, fragment: Fragment, checked: b
 const kCheckedChar = '☒';
 const kUncheckedChar = '☐';
 
-export function tokensWithChecked(tokens: PandocToken[]): { checked: null | boolean; tokens: PandocToken[] } {
+export function tokensWithChecked(tokens: PandocToken[]): { checked: null | boolean; tokens: PandocToken[]; } {
   // will set this flag based on inspecting the first Str token
   let checked: null | boolean | undefined;
   let lastWasChecked = false;
 
   // map tokens
-  const mappedTokens = mapTokens(tokens, tok => {
+  const mappedTokens = mapTokensRecursive(tokens, tok => {
     // if the last token was checked then strip the next space
     if (tok.t === 'Space' && lastWasChecked) {
       lastWasChecked = false;