feat: Code tokenize add

Author: 01-binary

packages/notion-to-jsx/src/components/Renderer/components/Code/CodeBlock.tsx

@@ -1,9 +1,13 @@
 import React, { useMemo } from 'react';
 import { codeBlock } from './styles.css';
-import Prism, { Grammar } from 'prismjs';
+import Prism, { Grammar, Token } from 'prismjs';
 import { MemoizedRichText } from '../MemoizedComponents';
 import { RichTextItem } from '../RichText/RichTexts';
 
+import 'prismjs/components/prism-typescript';
+import 'prismjs/components/prism-jsx';
+import 'prismjs/components/prism-tsx';
+
 if (typeof window !== 'undefined') {
   window.Prism = Prism;
 }
@@ -14,17 +18,42 @@ export interface Props {
   caption?: RichTextItem[];
 }
 
+const renderToken = (token: string | Token, i: number): React.ReactNode => {
+  if (typeof token === 'string') {
+    return <span key={i}>{token}</span>;
+  }
+
+  const content = token.content;
+  let tokenContent: React.ReactNode;
+
+  if (Array.isArray(content)) {
+    tokenContent = content.map((subToken, j) => renderToken(subToken, j));
+  } else if (typeof content === 'object' && content !== null) {
+    tokenContent = renderToken(content as Token, 0);
+  } else {
+    tokenContent = content;
+  }
+
+  return (
+    <span key={i} className={`token ${token.type}`}>
+      {tokenContent}
+    </span>
+  );
+};
+
 const CodeBlock: React.FC<Props> = ({ code, language, caption }) => {
-  const highlightedCode = useMemo(() => {
+  const tokens = useMemo(() => {
     const prismLanguage =
       Prism.languages[language] || Prism.languages.plaintext;
-    return Prism.highlight(code, prismLanguage as Grammar, language);
+    return Prism.tokenize(code, prismLanguage as Grammar);
   }, [code, language]);
 
   return (
     <>
-      <pre className={codeBlock}>
-        <code dangerouslySetInnerHTML={{ __html: highlightedCode }} />
+      <pre className={`${codeBlock} language-${language}`}>
+        <code className={`language-${language}`}>
+          {tokens.map((token, i) => renderToken(token, i))}
+        </code>
       </pre>
       {caption && (
         <figcaption>

packages/notion-to-jsx/src/components/Renderer/components/Code/styles.css.ts

@@ -1,4 +1,4 @@
-import { style, globalStyle } from '@vanilla-extract/css';
+import { style } from '@vanilla-extract/css';
 import { vars } from '../../../../styles/theme.css';
 
 export const codeBlock = style({
@@ -8,62 +8,7 @@ export const codeBlock = style({
   borderRadius: vars.borderRadius.md,
   overflow: 'auto',
   fontFamily: vars.typography.fontFamily.code,
-  fontSize: '0.85em',
+  fontSize: '0.85em !important',
   lineHeight: vars.typography.lineHeight.relaxed,
   margin: `${vars.spacing.sm} 0`,
 });
-
-globalStyle(`${codeBlock} code`, {
-  fontFamily: 'inherit',
-});
-
-globalStyle(
-  `${codeBlock} .token.comment, ${codeBlock} .token.prolog, ${codeBlock} .token.doctype, ${codeBlock} .token.cdata`,
-  {
-    color: '#666666',
-    fontStyle: 'italic',
-  }
-);
-
-globalStyle(`${codeBlock} .token.punctuation`, {
-  color: '#999999',
-});
-
-globalStyle(
-  `${codeBlock} .token.property, ${codeBlock} .token.tag, ${codeBlock} .token.boolean, ${codeBlock} .token.number, ${codeBlock} .token.constant, ${codeBlock} .token.symbol`,
-  {
-    color: '#905',
-  }
-);
-
-globalStyle(
-  `${codeBlock} .token.selector, ${codeBlock} .token.attr-name, ${codeBlock} .token.string, ${codeBlock} .token.char, ${codeBlock} .token.builtin`,
-  {
-    color: '#690',
-  }
-);
-
-globalStyle(
-  `${codeBlock} .token.operator, ${codeBlock} .token.entity, ${codeBlock} .token.url, ${codeBlock} .language-css .token.string, ${codeBlock} .style .token.string`,
-  {
-    color: '#9a6e3a',
-  }
-);
-
-globalStyle(
-  `${codeBlock} .token.atrule, ${codeBlock} .token.attr-value, ${codeBlock} .token.keyword`,
-  {
-    color: '#07a',
-  }
-);
-
-globalStyle(`${codeBlock} .token.function, ${codeBlock} .token.class-name`, {
-  color: '#dd4a68',
-});
-
-globalStyle(
-  `${codeBlock} .token.regex, ${codeBlock} .token.important, ${codeBlock} .token.variable`,
-  {
-    color: '#e90',
-  }
-);