chore(shiki): only send required langs to client, and support more on server (#7787)

* chore(shiki): add languages used in core

* chore(shiki): only send required langs to client

* fixup! chore(shiki): only send required langs to client

* fix unit tests

* fixup! fix unit tests

* Update packages/rehype-shiki/src/highlighter.mjs

Co-authored-by: Copilot <[email protected]>
Signed-off-by: Aviv Keller <[email protected]>

* Update packages/rehype-shiki/src/highlighter.mjs

Co-authored-by: Claudio W. <[email protected]>
Signed-off-by: Aviv Keller <[email protected]>

---------

Signed-off-by: Aviv Keller <[email protected]>
Co-authored-by: Copilot <[email protected]>
Co-authored-by: Claudio W. <[email protected]>

Author: 3 people

apps/site/components/Downloads/Release/ReleaseCodeBox.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { highlightToHtml } from '@node-core/rehype-shiki';
+import { highlightToHtml } from '@node-core/rehype-shiki/minimal';
 import AlertBox from '@node-core/ui-components/Common/AlertBox';
 import Skeleton from '@node-core/ui-components/Common/Skeleton';
 import { useTranslations } from 'next-intl';

apps/site/next.mdx.plugins.mjs

@@ -1,6 +1,6 @@
 'use strict';
 
-import rehypeShikiji from '@node-core/rehype-shiki';
+import rehypeShikiji from '@node-core/rehype-shiki/plugin';
 import remarkHeadings from '@vcarl/remark-headings';
 import rehypeAutolinkHeadings from 'rehype-autolink-headings';
 import rehypeSlug from 'rehype-slug';

packages/rehype-shiki/package.json

@@ -1,11 +1,14 @@
 {
   "name": "@node-core/rehype-shiki",
   "type": "module",
-  "main": "./src/index.mjs",
-  "module": "./src/index.mjs",
+  "exports": {
+    ".": "./src/index.mjs",
+    "./*": "./src/*.mjs"
+  },
   "scripts": {
     "lint:js": "eslint \"**/*.mjs\"",
-    "test": "node --test"
+    "test": "turbo test:unit",
+    "test:unit": "node --experimental-test-coverage --experimental-test-module-mocks --test \"**/*.test.mjs\""
   },
   "dependencies": {
     "@shikijs/core": "^3.3.0",

packages/rehype-shiki/src/__tests__/highlighter.test.mjs

@@ -0,0 +1,75 @@
+import assert from 'node:assert/strict';
+import { describe, it, mock } from 'node:test';
+
+// Mock dependencies
+const mockShiki = {
+  codeToHtml: mock.fn(() => '<pre><code>highlighted code</code></pre>'),
+  codeToHast: mock.fn(() => ({ type: 'element', tagName: 'pre' })),
+};
+
+mock.module('@shikijs/core', {
+  namedExports: { createHighlighterCoreSync: () => mockShiki },
+});
+
+mock.module('@shikijs/engine-javascript', {
+  namedExports: { createJavaScriptRegexEngine: () => ({}) },
+});
+
+mock.module('shiki/themes/nord.mjs', {
+  defaultExport: { name: 'nord', colors: { 'editor.background': '#2e3440' } },
+});
+
+describe('createHighlighter', async () => {
+  const { createHighlighter } = await import('../highlighter.mjs');
+
+  describe('getLanguageDisplayName', () => {
+    it('returns display name for known languages', () => {
+      const langs = [
+        { name: 'javascript', displayName: 'JavaScript', aliases: ['js'] },
+      ];
+      const highlighter = createHighlighter({ langs });
+
+      assert.strictEqual(
+        highlighter.getLanguageDisplayName('javascript'),
+        'JavaScript'
+      );
+      assert.strictEqual(
+        highlighter.getLanguageDisplayName('js'),
+        'JavaScript'
+      );
+    });
+
+    it('returns original name for unknown languages', () => {
+      const highlighter = createHighlighter({ langs: [] });
+      assert.strictEqual(
+        highlighter.getLanguageDisplayName('unknown'),
+        'unknown'
+      );
+    });
+  });
+
+  describe('highlightToHtml', () => {
+    it('extracts inner HTML from code tag', () => {
+      mockShiki.codeToHtml.mock.mockImplementationOnce(
+        () => '<pre><code>const x = 1;</code></pre>'
+      );
+
+      const highlighter = createHighlighter({});
+      const result = highlighter.highlightToHtml('const x = 1;', 'javascript');
+
+      assert.strictEqual(result, 'const x = 1;');
+    });
+  });
+
+  describe('highlightToHast', () => {
+    it('returns HAST tree from shiki', () => {
+      const expectedHast = { type: 'element', tagName: 'pre' };
+      mockShiki.codeToHast.mock.mockImplementationOnce(() => expectedHast);
+
+      const highlighter = createHighlighter({});
+      const result = highlighter.highlightToHast('code', 'javascript');
+
+      assert.deepStrictEqual(result, expectedHast);
+    });
+  });
+});

packages/rehype-shiki/src/__tests__/languages.test.mjs

@@ -0,0 +1,67 @@
+import assert from 'node:assert/strict';
+import { describe, it, mock } from 'node:test';
+
+// Simplified mocks - only mock what's actually needed
+mock.module('../index.mjs', {
+  namedExports: { highlightToHast: mock.fn(() => ({ children: [] })) },
+});
+
+mock.module('classnames', {
+  defaultExport: (...args) => args.filter(Boolean).join(' '),
+});
+
+mock.module('hast-util-to-string', {
+  namedExports: { toString: () => 'code' },
+});
+
+const mockVisit = mock.fn();
+mock.module('unist-util-visit', {
+  namedExports: { visit: mockVisit, SKIP: Symbol() },
+});
+
+describe('rehypeShikiji', async () => {
+  const { default: rehypeShikiji } = await import('../plugin.mjs');
+  const mockTree = { type: 'root', children: [] };
+
+  it('calls visit twice', () => {
+    mockVisit.mock.resetCalls();
+    rehypeShikiji()(mockTree);
+    assert.strictEqual(mockVisit.mock.calls.length, 2);
+  });
+
+  it('creates CodeTabs for multiple code blocks', () => {
+    const parent = {
+      children: [
+        {
+          tagName: 'pre',
+          children: [
+            {
+              tagName: 'code',
+              data: { meta: 'displayName="JS"' },
+              properties: { className: ['language-js'] },
+            },
+          ],
+        },
+        {
+          tagName: 'pre',
+          children: [
+            {
+              tagName: 'code',
+              data: { meta: 'displayName="TS"' },
+              properties: { className: ['language-ts'] },
+            },
+          ],
+        },
+      ],
+    };
+
+    mockVisit.mock.mockImplementation((tree, selector, visitor) => {
+      if (selector === 'element') {
+        visitor(parent.children[0], 0, parent);
+      }
+    });
+
+    rehypeShikiji()(mockTree);
+    assert.ok(parent.children.some(child => child.tagName === 'CodeTabs'));
+  });
+});

packages/rehype-shiki/src/__tests__/plugin.test.mjs

@@ -1,47 +1,67 @@
 import { createHighlighterCoreSync } from '@shikijs/core';
 import { createJavaScriptRegexEngine } from '@shikijs/engine-javascript';
+import shikiNordTheme from 'shiki/themes/nord.mjs';
 
-import { LANGUAGES, DEFAULT_THEME } from './languages.mjs';
-
-let _shiki;
-
-/**
- * Lazy-load and memoize the minimal Shikiji Syntax Highlighter
- * @returns {import('@shikijs/core').HighlighterCore}
- */
-export const getShiki = () => {
-  if (!_shiki) {
-    _shiki = createHighlighterCoreSync({
-      themes: [DEFAULT_THEME],
-      langs: LANGUAGES,
-      // Let's use Shiki's new Experimental JavaScript-based regex engine!
-      engine: createJavaScriptRegexEngine(),
-    });
-  }
-  return _shiki;
+const DEFAULT_THEME = {
+  // We are updating this color because the background color and comment text color
+  // in the Codebox component do not comply with accessibility standards
+  // @see https://www.w3.org/WAI/WCAG21/Understanding/contrast-minimum.html
+  colorReplacements: { '#616e88': '#707e99' },
+  ...shikiNordTheme,
 };
 
 /**
- * Highlights code and returns the inner HTML inside the <code> tag
- *
- * @param {string} code - The code to highlight
- * @param {string} language - The programming language to use for highlighting
- * @returns {string} The inner HTML of the highlighted code
+ * Creates a syntax highlighter with utility functions
+ * @param {import('@shikijs/core').HighlighterCoreOptions} options - Configuration options for the highlighter
  */
-export const highlightToHtml = (code, language) =>
-  getShiki()
-    .codeToHtml(code, { lang: language, theme: DEFAULT_THEME })
-    // Shiki will always return the Highlighted code encapsulated in a <pre> and <code> tag
-    // since our own CodeBox component handles the <code> tag, we just want to extract
-    // the inner highlighted code to the CodeBox
-    .match(/<code>(.+?)<\/code>/s)[1];
+export const createHighlighter = options => {
+  const shiki = createHighlighterCoreSync({
+    themes: [DEFAULT_THEME],
+    engine: createJavaScriptRegexEngine(),
+    ...options,
+  });
 
-/**
- * Highlights code and returns a HAST tree
- *
- * @param {string} code - The code to highlight
- * @param {string} language - The programming language to use for highlighting
- * @returns {import('hast').Element} The HAST representation of the highlighted code
- */
-export const highlightToHast = (code, language) =>
-  getShiki().codeToHast(code, { lang: language, theme: DEFAULT_THEME });
+  const theme = options.themes?.[0] ?? DEFAULT_THEME;
+  const langs = options.langs ?? [];
+
+  const getLanguageDisplayName = language => {
+    const languageByIdOrAlias = langs.find(
+      ({ name, aliases }) =>
+        name.toLowerCase() === language.toLowerCase() ||
+        (aliases !== undefined && aliases.includes(language.toLowerCase()))
+    );
+
+    return languageByIdOrAlias?.displayName ?? language;
+  };
+
+  /**
+   * Highlights code and returns the inner HTML inside the <code> tag
+   *
+   * @param {string} code - The code to highlight
+   * @param {string} language - The programming language to use for highlighting
+   * @returns {string} The inner HTML of the highlighted code
+   */
+  const highlightToHtml = (code, language) =>
+    shiki
+      .codeToHtml(code, { lang: language, theme })
+      // Shiki will always return the Highlighted code encapsulated in a <pre> and <code> tag
+      // since our own CodeBox component handles the <code> tag, we just want to extract
+      // the inner highlighted code to the CodeBox
+      .match(/<code>(.+?)<\/code>/s)[1];
+
+  /**
+   * Highlights code and returns a HAST tree
+   *
+   * @param {string} code - The code to highlight
+   * @param {string} language - The programming language to use for highlighting
+   */
+  const highlightToHast = (code, language) =>
+    shiki.codeToHast(code, { lang: language, theme });
+
+  return {
+    shiki,
+    getLanguageDisplayName,
+    highlightToHtml,
+    highlightToHast,
+  };
+};

packages/rehype-shiki/src/highlighter.mjs

@@ -1,5 +1,43 @@
-import { rehypeShikiji } from './plugin.mjs';
-export * from './highlighter.mjs';
-export * from './languages.mjs';
+import cLanguage from 'shiki/langs/c.mjs';
+import coffeeScriptLanguage from 'shiki/langs/coffeescript.mjs';
+import cPlusPlusLanguage from 'shiki/langs/cpp.mjs';
+import diffLanguage from 'shiki/langs/diff.mjs';
+import dockerLanguage from 'shiki/langs/docker.mjs';
+import httpLanguage from 'shiki/langs/http.mjs';
+import iniLanguage from 'shiki/langs/ini.mjs';
+import javaScriptLanguage from 'shiki/langs/javascript.mjs';
+import jsonLanguage from 'shiki/langs/json.mjs';
+import powershellLanguage from 'shiki/langs/powershell.mjs';
+import shellScriptLanguage from 'shiki/langs/shellscript.mjs';
+import shellSessionLanguage from 'shiki/langs/shellsession.mjs';
+import typeScriptLanguage from 'shiki/langs/typescript.mjs';
+import yamlLanguage from 'shiki/langs/yaml.mjs';
 
-export default rehypeShikiji;
+import { createHighlighter } from './highlighter.mjs';
+
+const { shiki, getLanguageDisplayName, highlightToHast, highlightToHtml } =
+  createHighlighter({
+    langs: [
+      ...cLanguage,
+      ...coffeeScriptLanguage,
+      ...cPlusPlusLanguage,
+      ...diffLanguage,
+      ...dockerLanguage,
+      ...httpLanguage,
+      ...iniLanguage,
+      {
+        ...javaScriptLanguage[0],
+        // We patch the JavaScript language to include the CommonJS and ES Module aliases
+        // that are commonly used (non-standard aliases) within our API docs and Blog posts
+        aliases: javaScriptLanguage[0].aliases.concat('cjs', 'mjs'),
+      },
+      ...jsonLanguage,
+      ...powershellLanguage,
+      ...shellScriptLanguage,
+      ...shellSessionLanguage,
+      ...typeScriptLanguage,
+      ...yamlLanguage,
+    ],
+  });
+
+export { shiki, getLanguageDisplayName, highlightToHast, highlightToHtml };