Speed up template migrations (#14679)

This PR does two things:
- Computes UTF-16 string positions in Rust rather than in JS —
eliminating a significant number of traversals of the input string
- Applies replacements to the content in ascending order so we only ever
move forward through the source string — this lets v8 optimize string
concatenation

Author: thecrypticace

crates/node/src/lib.rs

@@ -1,6 +1,10 @@
+use utf16::IndexConverter;
+
 #[macro_use]
 extern crate napi_derive;
 
+mod utf16;
+
 #[derive(Debug, Clone)]
 #[napi(object)]
 pub struct ChangedContent {
@@ -123,13 +127,25 @@ impl Scanner {
     &mut self,
     input: ChangedContent,
   ) -> Vec<CandidateWithPosition> {
+    let content = input.content.unwrap_or_else(|| {
+      std::fs::read_to_string(&input.file.unwrap()).expect("Failed to read file")
+    });
+
+    let input = ChangedContent {
+      file: None,
+      content: Some(content.clone()),
+      extension: input.extension,
+    };
+
+    let mut utf16_idx = IndexConverter::new(&content[..]);
+
     self
       .scanner
       .get_candidates_with_positions(input.into())
       .into_iter()
       .map(|(candidate, position)| CandidateWithPosition {
         candidate,
-        position: position as i64,
+        position: utf16_idx.get(position),
       })
       .collect()
   }

crates/node/src/utf16.rs

@@ -0,0 +1,100 @@
+/// The `IndexConverter` is used to convert UTF-8 *BYTE* indexes to UTF-16
+/// *character* indexes
+#[derive(Clone)]
+pub struct IndexConverter<'a> {
+  input: &'a str,
+  curr_utf8: usize,
+  curr_utf16: usize,
+}
+
+impl<'a> IndexConverter<'a> {
+  pub fn new(input: &'a str) -> Self {
+    Self {
+      input,
+      curr_utf8: 0,
+      curr_utf16: 0,
+    }
+  }
+
+  pub fn get(&mut self, pos: usize) -> i64 {
+    #[cfg(debug_assertions)]
+    if self.curr_utf8 > self.input.len() {
+      panic!("curr_utf8 points past the end of the input string");
+    }
+
+    if pos < self.curr_utf8 {
+      self.curr_utf8 = 0;
+      self.curr_utf16 = 0;
+    }
+
+    // SAFETY: No matter what `pos` is passed into this function `curr_utf8`
+    // will only ever be incremented up to the length of the input string.
+    //
+    // This eliminates a "potential" panic that cannot actually happen
+    let slice = unsafe {
+      self.input.get_unchecked(self.curr_utf8..)
+    };
+
+    for c in slice.chars() {
+      if self.curr_utf8 >= pos {
+        break
+      }
+
+      self.curr_utf8 += c.len_utf8();
+      self.curr_utf16 += c.len_utf16();
+    }
+
+    return self.curr_utf16 as i64;
+  }
+}
+
+#[cfg(test)]
+mod test {
+  use super::*;
+  use std::collections::HashMap;
+
+  #[test]
+  fn test_index_converter() {
+    let mut converter = IndexConverter::new("Hello 🔥🥳 world!");
+
+    let map = HashMap::from([
+      // hello<space>
+      (0, 0),
+      (1, 1),
+      (2, 2),
+      (3, 3),
+      (4, 4),
+      (5, 5),
+      (6, 6),
+
+      // inside the 🔥
+      (7, 8),
+      (8, 8),
+      (9, 8),
+      (10, 8),
+
+      // inside the 🥳
+      (11, 10),
+      (12, 10),
+      (13, 10),
+      (14, 10),
+
+      // <space>world!
+      (15, 11),
+      (16, 12),
+      (17, 13),
+      (18, 14),
+      (19, 15),
+      (20, 16),
+      (21, 17),
+
+      // Past the end should return the last utf-16 character index
+      (22, 17),
+      (100, 17),
+    ]);
+
+    for (idx_utf8, idx_utf16) in map {
+      assert_eq!(converter.get(idx_utf8), idx_utf16);
+    }
+  }
+}

packages/@tailwindcss-upgrade/package.json

@@ -38,7 +38,6 @@
     "postcss-import": "^16.1.0",
     "postcss-selector-parser": "^6.1.2",
     "prettier": "^3.3.3",
-    "string-byte-slice": "^3.0.0",
     "tailwindcss": "workspace:^",
     "tree-sitter": "^0.21.1",
     "tree-sitter-typescript": "^0.23.0"

packages/@tailwindcss-upgrade/src/template/candidates.test.ts

@@ -1,6 +1,7 @@
 import { __unstable__loadDesignSystem } from '@tailwindcss/node'
 import { describe, expect, test } from 'vitest'
-import { extractRawCandidates, printCandidate, replaceCandidateInContent } from './candidates'
+import { extractRawCandidates, printCandidate } from './candidates'
+import { spliceChangesIntoString } from './splice-changes-into-string'
 
 let html = String.raw
 
@@ -66,13 +67,20 @@ test('replaces the right positions for a candidate', async () => {
     ({ rawCandidate }) => designSystem.parseCandidate(rawCandidate).length > 0,
   )!
 
-  expect(replaceCandidateInContent(content, 'flex', candidate.start, candidate.end))
-    .toMatchInlineSnapshot(`
+  let migrated = spliceChangesIntoString(content, [
+    {
+      start: candidate.start,
+      end: candidate.end,
+      replacement: 'flex',
+    },
+  ])
+
+  expect(migrated).toMatchInlineSnapshot(`
+    "
+        <h1>🤠👋</h1>
+        <div class="flex" />
       "
-          <h1>🤠👋</h1>
-          <div class="flex" />
-        "
-    `)
+  `)
 })
 
 const candidates = [

packages/@tailwindcss-upgrade/src/template/candidates.ts

@@ -1,5 +1,4 @@
 import { Scanner } from '@tailwindcss/oxide'
-import stringByteSlice from 'string-byte-slice'
 import type { Candidate, Variant } from '../../../tailwindcss/src/candidate'
 import type { DesignSystem } from '../../../tailwindcss/src/design-system'
 
@@ -139,12 +138,3 @@ function escapeArbitrary(input: string) {
     .replaceAll('_', String.raw`\_`) // Escape underscores to keep them as-is
     .replaceAll(' ', '_') // Replace spaces with underscores
 }
-
-export function replaceCandidateInContent(
-  content: string,
-  replacement: string,
-  startByte: number,
-  endByte: number,
-) {
-  return stringByteSlice(content, 0, startByte) + replacement + stringByteSlice(content, endByte)
-}

packages/@tailwindcss-upgrade/src/template/migrate.ts

@@ -2,14 +2,15 @@ import fs from 'node:fs/promises'
 import path, { extname } from 'node:path'
 import type { Config } from 'tailwindcss'
 import type { DesignSystem } from '../../../tailwindcss/src/design-system'
-import { extractRawCandidates, replaceCandidateInContent } from './candidates'
+import { extractRawCandidates } from './candidates'
 import { arbitraryValueToBareValue } from './codemods/arbitrary-value-to-bare-value'
 import { automaticVarInjection } from './codemods/automatic-var-injection'
 import { bgGradient } from './codemods/bg-gradient'
 import { important } from './codemods/important'
 import { prefix } from './codemods/prefix'
 import { simpleLegacyClasses } from './codemods/simple-legacy-classes'
 import { variantOrder } from './codemods/variant-order'
+import { spliceChangesIntoString, type StringChange } from './splice-changes-into-string'
 
 export type Migration = (
   designSystem: DesignSystem,
@@ -46,19 +47,23 @@ export default async function migrateContents(
 ): Promise<string> {
   let candidates = await extractRawCandidates(contents, extension)
 
-  // Sort candidates by starting position desc
-  candidates.sort((a, z) => z.start - a.start)
+  let changes: StringChange[] = []
 
-  let output = contents
   for (let { rawCandidate, start, end } of candidates) {
     let migratedCandidate = migrateCandidate(designSystem, userConfig, rawCandidate)
 
-    if (migratedCandidate !== rawCandidate) {
-      output = replaceCandidateInContent(output, migratedCandidate, start, end)
+    if (migratedCandidate === rawCandidate) {
+      continue
     }
+
+    changes.push({
+      start,
+      end,
+      replacement: migratedCandidate,
+    })
   }
 
-  return output
+  return spliceChangesIntoString(contents, changes)
 }
 
 export async function migrate(designSystem: DesignSystem, userConfig: Config, file: string) {

packages/@tailwindcss-upgrade/src/template/splice-changes-into-string.ts

@@ -0,0 +1,42 @@
+export interface StringChange {
+  start: number
+  end: number
+  replacement: string
+}
+
+/**
+ * Apply the changes to the string such that a change in the length
+ * of the string does not break the indexes of the subsequent changes.
+ */
+export function spliceChangesIntoString(str: string, changes: StringChange[]) {
+  // If there are no changes, return the original string
+  if (!changes[0]) return str
+
+  // Sort all changes in order to make it easier to apply them
+  changes.sort((a, b) => {
+    return a.end - b.end || a.start - b.start
+  })
+
+  // Append original string between each chunk, and then the chunk itself
+  // This is sort of a String Builder pattern, thus creating less memory pressure
+  let result = ''
+
+  let previous = changes[0]
+
+  result += str.slice(0, previous.start)
+  result += previous.replacement
+
+  for (let i = 1; i < changes.length; ++i) {
+    let change = changes[i]
+
+    result += str.slice(previous.end, change.start)
+    result += change.replacement
+
+    previous = change
+  }
+
+  // Add leftover string from last chunk to end
+  result += str.slice(previous.end)
+
+  return result
+}