feat: add general tree-sitter parser, and Rust, Python support

Author: sxzz

app/parser/TreeSitterGui.vue

@@ -0,0 +1,35 @@
+<script lang="ts">
+import type { TreeSitterOptions } from './tree-sitter'
+
+const useOption = makeUseOption<TreeSitterOptions>()
+</script>
+
+<script setup lang="ts">
+const wasmUrl = useOption('wasmUrl')
+</script>
+
+<template>
+  <div flex="~ col" gap2 text-sm font-mono>
+    <label flex flex-col gap1>
+      <span>WASM:</span>
+      <textarea
+        v-model="wasmUrl"
+        text-md
+        h-32
+        border
+        rounded-md
+        px1
+        py0.5
+        accent-current
+      />
+    </label>
+    <div text-right>
+      <a
+        href="https://github.com/tree-sitter/tree-sitter/wiki/List-of-parsers"
+        target="_blank"
+      >
+        Parser List
+      </a>
+    </div>
+  </div>
+</template>

app/parser/custom.ts app/parser/general.tsapp/parser/custom.ts renamed to app/parser/general.ts

@@ -1,12 +1,11 @@
 import json5 from 'json5'
 import type { LanguageOption, Parser } from './index'
 
-// @unocss-include
-
-const customParser: Parser = {
-  id: 'custom-parser',
-  label: 'custom',
-  icon: 'i-ri:magic-fill',
+const jsonAst: Parser = {
+  id: 'json-ast',
+  label: 'JSON AST',
+  // @unocss-include
+  icon: 'i-ri:braces-line',
   editorLanguage: 'json',
   options: {
     configurable: false,
@@ -20,15 +19,14 @@ const customParser: Parser = {
   },
 }
 
-export const custom: LanguageOption = {
-  label: 'JSON AST',
-  icon: 'i-ri:magic-line',
-  parsers: [customParser],
+export const general: LanguageOption = {
+  label: 'General',
+  icon: 'i-ri:code-line',
+  parsers: [jsonAst, treeSitter],
   codeTemplate: JSON.stringify(
     {
       type: 'Program',
-      body: [],
-      sourceType: 'module',
+      body: [{ type: 'ExpressionStatement' }],
     },
     undefined,
     2,

app/parser/index.ts

@@ -2,7 +2,7 @@ import { angular } from './angular'
 import { astro } from './astro'
 import { csharp } from './csharp'
 import { css } from './css'
-import { custom } from './custom'
+import { general } from './general'
 import { graphql } from './graphql'
 import { html } from './html'
 import { java } from './java'
@@ -116,6 +116,6 @@ export const LANGUAGES = {
   wxml,
   graphql,
   protobuf,
-  custom,
+  general,
 }
 export type Language = keyof typeof LANGUAGES

app/parser/python.ts

@@ -99,6 +99,6 @@ def parse_code(code, options):
 export const python: LanguageOption = {
   label: 'Python',
   icon: 'i-vscode-icons:file-type-python',
-  parsers: [pyodide],
+  parsers: [pyodide, treeSitterPython],
   codeTemplate: pythonTemplate,
 }

app/parser/rust.ts

@@ -89,6 +89,6 @@ const jinxRust: Parser<typeof JinxRust, JinxRust.rs.ParserOptions> = {
 export const rust: LanguageOption = {
   label: 'Rust',
   icon: 'i-vscode-icons:file-type-rust',
-  parsers: [syn, jinxRust],
+  parsers: [syn, jinxRust, treeSitterRust],
   codeTemplate: rustTemplate,
 }

app/parser/tree-sitter.ts

@@ -1,12 +1,13 @@
 import type { Parser } from './index'
 import type * as TreeSitter from 'web-tree-sitter'
 
-const common = {
+const shared = {
   options: {
     configurable: false,
     defaultValue: {},
     editorLanguage: 'json',
   },
+  icon: 'https://cdn.jsdelivr.net/gh/tree-sitter/tree-sitter/docs/src/assets/images/tree-sitter-small.png',
   init: (moduleUrl: string) => initTreeSitterParser(moduleUrl),
   parse(this: TreeSitter.Parser, code: string) {
     const parsed = this.parse(code)
@@ -15,24 +16,54 @@ const common = {
   getNodeLocation: genGetNodeLocation('treeSitter'),
 } as const
 
+export interface TreeSitterOptions {
+  wasmUrl: string
+}
+
+export const treeSitter: Parser<void, TreeSitterOptions> = {
+  id: 'tree-sitter',
+  label: 'Tree Sitter',
+  icon: shared.icon,
+  editorLanguage: 'text',
+  options: {
+    configurable: true,
+    defaultValue: {
+      wasmUrl: getJsdelivrUrl(
+        'tree-sitter-javascript',
+        '/tree-sitter-javascript.wasm',
+      ),
+    },
+    editorLanguage: 'json',
+  },
+  pkgName: '',
+  init: () => {},
+  async parse(code, options) {
+    if (!options.wasmUrl) {
+      throw new Error('WASM URL is required in options')
+    }
+
+    const parser = await initTreeSitterParser(options.wasmUrl)
+    const parsed = parser.parse(code)
+    return parsed && convertNode(parsed.rootNode)
+  },
+  getNodeLocation: shared.getNodeLocation,
+  gui: () => import('./TreeSitterGui.vue'),
+}
+
 export const treeSitterJavascript: Parser<TreeSitter.Parser> = {
-  ...common,
+  ...shared,
   id: 'tree-sitter-javascript',
   label: 'tree-sitter-javascript',
-  // @unocss-include
-  icon: 'i-vscode-icons:file-type-js',
   link: 'https://github.com/tree-sitter/tree-sitter-javascript',
   editorLanguage: 'javascript',
   pkgName: 'tree-sitter-javascript',
   getModuleUrl: (pkg) => getJsdelivrUrl(pkg, '/tree-sitter-javascript.wasm'),
 }
 
 export const treeSitterTypescript: Parser<TreeSitter.Parser> = {
-  ...common,
+  ...shared,
   id: 'tree-sitter-typescript',
   label: 'tree-sitter-typescript',
-  // @unocss-include
-  icon: 'i-vscode-icons:file-type-typescript',
   link: 'https://github.com/tree-sitter/tree-sitter-typescript',
   editorLanguage: 'typescript',
   pkgName: 'tree-sitter-typescript',
@@ -48,30 +79,36 @@ export const treeSitterTsx: Parser<TreeSitter.Parser> = {
 }
 
 export const treeSitterCSharp: Parser<TreeSitter.Parser> = {
+  ...shared,
   id: 'tree-sitter-c-sharp',
   label: 'tree-sitter',
-  // @unocss-include
-  icon: 'i-vscode-icons:file-type-csharp',
   link: 'https://github.com/tree-sitter/tree-sitter-c-sharp',
   editorLanguage: 'csharp',
-  options: {
-    configurable: false,
-    defaultValue: {},
-    editorLanguage: 'json',
-  },
   pkgName: 'tree-sitter-c-sharp',
   getModuleUrl: (pkg) => getJsdelivrUrl(pkg, '/tree-sitter-c_sharp.wasm'),
-  init(moduleUrl) {
-    return initTreeSitterParser(moduleUrl)
-  },
-  parse(code) {
-    const parsed = this.parse(code)
-    return parsed && convertNode(parsed.rootNode)
-  },
-  getNodeLocation: genGetNodeLocation('treeSitter'),
 }
 
-export function convertNode(node: TreeSitter.Node): any {
+export const treeSitterPython: Parser<TreeSitter.Parser> = {
+  ...shared,
+  id: 'tree-sitter-python',
+  label: 'tree-sitter-python',
+  link: 'https://github.com/tree-sitter/tree-sitter-python',
+  editorLanguage: 'python',
+  pkgName: 'tree-sitter-python',
+  getModuleUrl: (pkg) => getJsdelivrUrl(pkg, '/tree-sitter-python.wasm'),
+}
+
+export const treeSitterRust: Parser<TreeSitter.Parser> = {
+  ...shared,
+  id: 'tree-sitter-rust',
+  label: 'tree-sitter-rust',
+  link: 'https://github.com/tree-sitter/tree-sitter-rust',
+  editorLanguage: 'rust',
+  pkgName: 'tree-sitter-rust',
+  getModuleUrl: (pkg) => getJsdelivrUrl(pkg, '/tree-sitter-rust.wasm'),
+}
+
+function convertNode(node: TreeSitter.Node): any {
   const {
     startIndex,
     endIndex,
@@ -109,12 +146,17 @@ async function loadTreeSitter() {
 async function initTreeSitterParser(url: string) {
   const [{ Parser, Language }, response] = await Promise.all([
     loadTreeSitter(),
-    fetch(url).then((res) => res.arrayBuffer()),
+    fetch(url).then((res) => {
+      if (!res.ok) {
+        throw new Error(`Failed to load WASM: ${res.status} ${res.statusText}`)
+      }
+      return res.arrayBuffer()
+    }),
   ])
-  const CSharpLanguage = await Language.load(new Uint8Array(response))
+  const language = await Language.load(new Uint8Array(response))
 
   const parser = new Parser()
-  parser.setLanguage(CSharpLanguage)
+  parser.setLanguage(language)
 
   return parser
 }