Merge pull request #1403 from dodona-edu/fix/tree-sitter-detection

Fix detecting tree-sitter parsers

Author: rien

lib/package.json

@@ -34,7 +34,8 @@
     "ava": "5.3.1",
     "eslint": "8.53.0",
     "node-gyp": "^9.4.1",
-    "typescript": "5.2.2"
+    "typescript": "5.2.2",
+    "tree-sitter-json": "0.20.2"
   },
   "dependencies": {
     "@dodona/dolos-parsers": "1.0.0",

lib/src/lib/language.ts

@@ -43,8 +43,12 @@ export class ProgrammingLanguage extends Language {
     if (this.languageModule === undefined) {
       // @ts-ignore
       this.languageModule = (await import("@dodona/dolos-parsers")).default[this.name];
+      this.languageModule ||= (await import(`tree-sitter-${this.name}`)).default;
       if (this.languageModule === undefined) {
-        throw new LanguageError("Could not find language module for: " + this.name);
+        throw new LanguageError(
+          `Could not find language module for ${this.name}, ` +
+          `searched in @dodona/dolos-parsers and tree-sitter-${this.name}`
+        );
       }
     }
     return this.languageModule;

lib/src/test/tokenizer.test.ts

@@ -62,3 +62,14 @@ test("language picker should detect most common language", t => {
   const detected = new LanguagePicker().detectLanguage(files);
   t.deepEqual(detected.name, "python");
 });
+
+test("should be able to use external tree-sitter parsers (tree-sitter-json)", async t => {
+  const file = (await readPath("./package.json")).ok();
+  const language = await (new LanguagePicker().findLanguage("json"));
+
+  const tokenizer = await language.createTokenizer();
+  t.truthy(tokenizer);
+
+  const { tokens } = tokenizer.tokenizeFile(file);
+  t.truthy(tokens);
+});

yarn.lock

@@ -4114,6 +4114,13 @@ tree-sitter-cli@^0.20.8:
   resolved "https://registry.yarnpkg.com/tree-sitter-cli/-/tree-sitter-cli-0.20.8.tgz#06a81cea8d6d82f93d67eed7d28b6bc04a4a8916"
   integrity sha512-XjTcS3wdTy/2cc/ptMLc/WRyOLECRYcMTrSWyhZnj1oGSOWbHLTklgsgRICU3cPfb0vy+oZCC33M43u6R1HSCA==
 
+tree-sitter-json@0.20.2:
+  version "0.20.2"
+  resolved "https://registry.yarnpkg.com/tree-sitter-json/-/tree-sitter-json-0.20.2.tgz#8909ffb7149120daa72f9cadb63e8a214f1e5aba"
+  integrity sha512-eUxrowp4F1QEGk/i7Sa+Xl8Crlfp7J0AXxX1QdJEQKQYMWhgMbCIgyQvpO3Q0P9oyTrNQxRLlRipDS44a8EtRw==
+  dependencies:
+    nan "^2.18.0"
+
 tree-sitter@^0.20.6:
   version "0.20.6"
   resolved "https://registry.yarnpkg.com/tree-sitter/-/tree-sitter-0.20.6.tgz#fec52e5d7cc6c583135756479f2440dd89b25cbe"