chore(monorepo): rehype-shiki (nodejs#7760)

avivkeller · web-flow · commit 259db02acd9a · 2025-05-21T10:47:02.000Z
* chore(monorepo): `rehype-shiki`

* fixup!

* add build to turbo
diff --git a/.github/workflows/lint-and-tests.yml b/.github/workflows/lint-and-tests.yml
@@ -149,10 +149,10 @@ jobs:
         if: ${{ !cancelled() && github.event_name != 'merge_group' }}
         uses: codecov/codecov-action@ad3126e916f78f00edff4ed0317cf185271ccc2d # v5.4.2
         with:
-          files: ./apps/site/lcov.info,./packages/ui-components/lcov.info
+          files: ./apps/site/lcov.info,./packages/*/lcov.info
 
       - name: Upload test results to Codecov
         if: ${{ !cancelled() && github.event_name != 'merge_group' }}
         uses: codecov/test-results-action@f2dba722c67b86c6caa034178c6e4d35335f6706 # v1.1.0
         with:
-          files: ./apps/site/junit.xml,./packages/ui-components/junit.xml
+          files: ./apps/site/junit.xml,./packages/*/junit.xml
diff --git a/apps/site/components/Downloads/Release/ReleaseCodeBox.tsx b/apps/site/components/Downloads/Release/ReleaseCodeBox.tsx
@@ -1,5 +1,6 @@
 'use client';
 
+import { highlightToHtml } from '@node-core/rehype-shiki';
 import AlertBox from '@node-core/ui-components/Common/AlertBox';
 import Skeleton from '@node-core/ui-components/Common/Skeleton';
 import { useTranslations } from 'next-intl';
@@ -16,7 +17,6 @@ import {
 } from '#site/providers/releaseProvider';
 import type { ReleaseContextType } from '#site/types/release';
 import { INSTALL_METHODS } from '#site/util/downloadUtils';
-import { highlightToHtml } from '#site/util/getHighlighter';
 
 // Creates a minimal JavaScript interpreter for parsing the JavaScript code from the snippets
 // Note: that the code runs inside a sandboxed environment and cannot interact with any code outside of the sandbox
diff --git a/apps/site/components/MDX/CodeBox/index.tsx b/apps/site/components/MDX/CodeBox/index.tsx
@@ -1,7 +1,7 @@
+import { getLanguageDisplayName } from '@node-core/rehype-shiki';
 import type { FC, PropsWithChildren } from 'react';
 
 import CodeBox from '#site/components/Common/CodeBox';
-import { getLanguageDisplayName } from '#site/util/getLanguageDisplayName';
 
 type CodeBoxProps = { className?: string; showCopyButton?: string };
 
diff --git a/apps/site/next.mdx.plugins.mjs b/apps/site/next.mdx.plugins.mjs
@@ -1,13 +1,12 @@
 'use strict';
 
+import rehypeShikiji from '@node-core/rehype-shiki';
 import remarkHeadings from '@vcarl/remark-headings';
 import rehypeAutolinkHeadings from 'rehype-autolink-headings';
 import rehypeSlug from 'rehype-slug';
 import remarkGfm from 'remark-gfm';
 import readingTime from 'remark-reading-time';
 
-import rehypeShikiji from './next.mdx.shiki.mjs';
-
 /**
  * Provides all our Rehype Plugins that are used within MDX
  */
diff --git a/apps/site/package.json b/apps/site/package.json
@@ -28,6 +28,7 @@
   "dependencies": {
     "@heroicons/react": "~2.2.0",
     "@mdx-js/mdx": "^3.1.0",
+    "@node-core/rehype-shiki": "workspace:*",
     "@node-core/ui-components": "workspace:*",
     "@node-core/website-i18n": "workspace:*",
     "@nodevu/core": "0.3.0",
@@ -41,8 +42,6 @@
     "@radix-ui/react-tabs": "^1.1.3",
     "@radix-ui/react-toast": "^1.2.6",
     "@radix-ui/react-tooltip": "^1.2.4",
-    "@shikijs/core": "^3.2.2",
-    "@shikijs/engine-javascript": "^3.2.2",
     "@tailwindcss/postcss": "~4.1.5",
     "@types/node": "22.15.3",
     "@types/react": "^19.1.0",
@@ -56,7 +55,6 @@
     "github-slugger": "~2.0.0",
     "glob": "~11.0.1",
     "gray-matter": "~4.0.3",
-    "hast-util-to-string": "~3.0.1",
     "next": "15.3.1",
     "next-intl": "~4.1.0",
     "next-themes": "~0.4.6",
@@ -69,10 +67,8 @@
     "remark-gfm": "~4.0.1",
     "remark-reading-time": "~2.0.1",
     "semver": "~7.7.1",
-    "shiki": "~3.3.0",
     "sval": "^0.6.3",
     "tailwindcss": "~4.0.17",
-    "unist-util-visit": "~5.0.0",
     "vfile": "~6.0.3",
     "vfile-matter": "~5.0.1"
   },
diff --git a/apps/site/util/getHighlighter.ts b/apps/site/util/getHighlighter.ts
diff --git a/apps/site/util/getLanguageDisplayName.ts b/apps/site/util/getLanguageDisplayName.ts
diff --git a/packages/rehype-shiki/eslint.config.js b/packages/rehype-shiki/eslint.config.js
@@ -0,0 +1,17 @@
+import baseConfig from '../../eslint.config.js';
+
+export default [
+  ...baseConfig,
+  {
+    languageOptions: {
+      parserOptions: {
+        // Allow nullish syntax (i.e. "?." or "??")
+        ecmaVersion: 2020,
+      },
+    },
+    rules: {
+      // Shiki's export isn't named, it's a re-export
+      'import-x/named': 'off',
+    },
+  },
+];
diff --git a/packages/rehype-shiki/package.json b/packages/rehype-shiki/package.json
@@ -0,0 +1,18 @@
+{
+  "name": "@node-core/rehype-shiki",
+  "type": "module",
+  "main": "./src/index.mjs",
+  "module": "./src/index.mjs",
+  "scripts": {
+    "lint:js": "eslint \"**/*.mjs\"",
+    "test": "node --test"
+  },
+  "dependencies": {
+    "@shikijs/core": "^3.3.0",
+    "@shikijs/engine-javascript": "^3.3.0",
+    "classnames": "~2.5.1",
+    "hast-util-to-string": "^3.0.1",
+    "shiki": "~3.3.0",
+    "unist-util-visit": "^5.0.0"
+  }
+}
diff --git a/packages/rehype-shiki/src/__tests__/languages.test.mjs b/packages/rehype-shiki/src/__tests__/languages.test.mjs
@@ -1,20 +1,16 @@
 import assert from 'node:assert/strict';
-import { it, describe, mock } from 'node:test';
+import { it, describe } from 'node:test';
 
-describe('getLanguageDisplayName', async () => {
-  mock.module('#site/shiki.config.mjs', {
-    namedExports: {
-      LANGUAGES: [
-        { name: 'javascript', aliases: ['js'], displayName: 'JavaScript' },
-        { name: 'typescript', aliases: ['ts'], displayName: 'TypeScript' },
-      ],
-    },
-  });
+import { getLanguageDisplayName, LANGUAGES } from '../languages.mjs';
 
-  const { getLanguageDisplayName } = await import(
-    '#site/util/getLanguageDisplayName'
-  );
+LANGUAGES.splice(
+  0,
+  LANGUAGES.length,
+  { name: 'javascript', aliases: ['js'], displayName: 'JavaScript' },
+  { name: 'typescript', aliases: ['ts'], displayName: 'TypeScript' }
+);
 
+describe('getLanguageDisplayName', async () => {
   it('should return the display name for a known language', () => {
     assert.equal(getLanguageDisplayName('javascript'), 'JavaScript');
     assert.equal(getLanguageDisplayName('js'), 'JavaScript');
diff --git a/packages/rehype-shiki/src/highlighter.mjs b/packages/rehype-shiki/src/highlighter.mjs
@@ -0,0 +1,47 @@
+import { createHighlighterCoreSync } from '@shikijs/core';
+import { createJavaScriptRegexEngine } from '@shikijs/engine-javascript';
+
+import { LANGUAGES, DEFAULT_THEME } from './languages.mjs';
+
+let _shiki;
+
+/**
+ * Lazy-load and memoize the minimal Shikiji Syntax Highlighter
+ * @returns {import('@shikijs/core').HighlighterCore}
+ */
+export const getShiki = () => {
+  if (!_shiki) {
+    _shiki = createHighlighterCoreSync({
+      themes: [DEFAULT_THEME],
+      langs: LANGUAGES,
+      // Let's use Shiki's new Experimental JavaScript-based regex engine!
+      engine: createJavaScriptRegexEngine(),
+    });
+  }
+  return _shiki;
+};
+
+/**
+ * Highlights code and returns the inner HTML inside the <code> tag
+ *
+ * @param {string} code - The code to highlight
+ * @param {string} language - The programming language to use for highlighting
+ * @returns {string} The inner HTML of the highlighted code
+ */
+export const highlightToHtml = (code, language) =>
+  getShiki()
+    .codeToHtml(code, { lang: language, theme: DEFAULT_THEME })
+    // Shiki will always return the Highlighted code encapsulated in a <pre> and <code> tag
+    // since our own CodeBox component handles the <code> tag, we just want to extract
+    // the inner highlighted code to the CodeBox
+    .match(/<code>(.+?)<\/code>/s)[1];
+
+/**
+ * Highlights code and returns a HAST tree
+ *
+ * @param {string} code - The code to highlight
+ * @param {string} language - The programming language to use for highlighting
+ * @returns {import('hast').Element} The HAST representation of the highlighted code
+ */
+export const highlightToHast = (code, language) =>
+  getShiki().codeToHast(code, { lang: language, theme: DEFAULT_THEME });
diff --git a/packages/rehype-shiki/src/index.mjs b/packages/rehype-shiki/src/index.mjs
@@ -0,0 +1,5 @@
+import { rehypeShikiji } from './plugin.mjs';
+export * from './highlighter.mjs';
+export * from './languages.mjs';
+
+export default rehypeShikiji;
diff --git a/packages/rehype-shiki/src/languages.mjs b/packages/rehype-shiki/src/languages.mjs
@@ -43,3 +43,18 @@ export const DEFAULT_THEME = {
   colorReplacements: { '#616e88': '#707e99' },
   ...shikiNordTheme,
 };
+
+/**
+ * Get the display name of a given language
+ * @param {string} language The language ID
+ * @returns {string} The display name of the language, or the input as a fallback
+ */
+export const getLanguageDisplayName = language => {
+  const languageByIdOrAlias = LANGUAGES.find(
+    ({ name, aliases }) =>
+      name.toLowerCase() === language.toLowerCase() ||
+      (aliases !== undefined && aliases.includes(language.toLowerCase()))
+  );
+
+  return languageByIdOrAlias?.displayName ?? language;
+};
diff --git a/packages/rehype-shiki/src/plugin.mjs b/packages/rehype-shiki/src/plugin.mjs
@@ -4,7 +4,7 @@ import classNames from 'classnames';
 import { toString } from 'hast-util-to-string';
 import { SKIP, visit } from 'unist-util-visit';
 
-import { highlightToHast } from './util/getHighlighter';
+import { highlightToHast } from './highlighter.mjs';
 
 // This is what Remark will use as prefix within a <pre> className
 // to attribute the current language of the <pre> element
@@ -53,7 +53,7 @@ function isCodeBlock(node) {
   );
 }
 
-export default function rehypeShikiji() {
+export function rehypeShikiji() {
   return function (tree) {
     visit(tree, 'element', (_, index, parent) => {
       const languages = [];
diff --git a/packages/rehype-shiki/turbo.json b/packages/rehype-shiki/turbo.json
@@ -0,0 +1,13 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "extends": ["//"],
+  "tasks": {
+    "build": {},
+    "lint:js": {
+      "inputs": ["src/**/*.mjs"]
+    },
+    "test": {
+      "inputs": ["src/**/*.mjs"]
+    }
+  }
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
