perf(linter/plugins): speed up initTokensWithComments (#16117)

Follow-on after #16071. Seeing that more performant implementation made me see another couple of optimizations.

1. Fast path for when file has no comments (probably not so uncommon).

2. Extend the "push in runs" approach. Ensure that in the main loop, there's always at least 1 token and 1 comment remaining, and you know which one it is, which reduces branches.

3. Avoid ever reading out of bounds of the arrays (expensive).

Author: overlookmotel

apps/oxlint/src-js/plugins/tokens.ts

@@ -4,7 +4,7 @@
 
 import { createRequire } from "node:module";
 import { sourceText, initSourceText } from "./source_code.js";
-import { debugAssertIsNonNull } from "../utils/asserts.js";
+import { debugAssert, debugAssertIsNonNull } from "../utils/asserts.js";
 
 import type { Comment, Node, NodeOrToken } from "./types.ts";
 import type { Span } from "./location.ts";
@@ -184,40 +184,90 @@ function initTokensWithComments() {
   debugAssertIsNonNull(tokens);
   debugAssertIsNonNull(comments);
 
-  // TODO: Replace `range[0]` with `start` once we have our own tokens which have `start` property.
+  // TODO: Replace `range[0]` with `start` throughout this function
+  // once we have our own tokens which have `start` property
+
+  // Fast paths for file with no comments, or file which is only comments
+  const commentsLength = comments.length;
+  if (commentsLength === 0) {
+    tokensWithComments = tokens;
+    return;
+  }
+
+  const tokensLength = tokens.length;
+  if (tokensLength === 0) {
+    tokensWithComments = comments;
+    return;
+  }
+
+  // File contains both tokens and comments.
+  // Fill `tokensWithComments` with the 2 arrays interleaved in source order.
   tokensWithComments = [];
 
-  const tokensLength = tokens.length,
-    commentsLength = comments.length;
+  let tokenIndex = 0,
+    commentIndex = 0,
+    token = tokens[0],
+    comment = comments[0],
+    tokenStart = token.range[0],
+    commentStart = comment.range[0];
+
+  // Push any leading comments
+  while (commentStart < tokenStart) {
+    // Push current comment
+    tokensWithComments.push(comment);
+
+    // If that was last comment, push all remaining tokens, and exit
+    if (++commentIndex === commentsLength) {
+      tokensWithComments.push(...tokens.slice(tokenIndex));
+      debugCheckTokensWithComments();
+      return;
+    }
 
-  let tokensIndex = 0,
-    commentsIndex = 0;
-  while (tokensIndex < tokensLength && commentsIndex < commentsLength) {
-    let token = tokens[tokensIndex],
-      comment = comments[commentsIndex];
+    // Get next comment
+    comment = comments[commentIndex];
+    commentStart = comment.range[0];
+  }
 
-    // TODO: Replace `range[0]` with `start` once we have our own tokens which have `start` property.
-    const nextTokenStart = token.range[0],
-      nextCommentStart = comment.range[0];
+  // Push a run of tokens, then a run of comments, and so on, until all tokens and comments are exhausted
+  while (true) {
+    // There's at least 1 token and 1 comment remaining, and token is first.
+    // Push tokens until we reach the next comment or the end.
+    do {
+      // Push current token
+      tokensWithComments.push(token);
 
-    if (nextTokenStart < nextCommentStart) {
-      while (tokensIndex < tokensLength && token.range[0] <= nextCommentStart) {
-        tokensWithComments.push(token);
-        token = tokens[++tokensIndex];
+      // If that was last token, push all remaining comments, and exit
+      if (++tokenIndex === tokensLength) {
+        tokensWithComments.push(...comments.slice(commentIndex));
+        debugCheckTokensWithComments();
+        return;
       }
-    } else {
-      while (commentsIndex < commentsLength && comment.range[0] <= nextTokenStart) {
-        tokensWithComments.push(comment);
-        comment = comments[++commentsIndex];
+
+      // Get next token
+      token = tokens[tokenIndex];
+      tokenStart = token.range[0];
+    } while (tokenStart < commentStart);
+
+    // There's at least 1 token and 1 comment remaining, and comment is first.
+    // Push comments until we reach the next token or the end.
+    do {
+      // Push current comment
+      tokensWithComments.push(comment);
+
+      // If that was last comment, push all remaining tokens, and exit
+      if (++commentIndex === commentsLength) {
+        tokensWithComments.push(...tokens.slice(tokenIndex));
+        debugCheckTokensWithComments();
+        return;
       }
-    }
-  }
 
-  // After one of `tokens` or `comments` is exhausted, directly push the other's elements
-  while (commentsIndex < commentsLength) tokensWithComments.push(comments[commentsIndex++]);
-  while (tokensIndex < tokensLength) tokensWithComments.push(tokens[tokensIndex++]);
+      // Get next comment
+      comment = comments[commentIndex];
+      commentStart = comment.range[0];
+    } while (commentStart < tokenStart);
+  }
 
-  debugCheckTokensWithComments();
+  debugAssert(false, "Unreachable");
 }
 
 /**