Skip to content

Commit 8c7e3d5

Browse files
swilly22swilly22
authored
Merge pull request #102 from FalkorDB/more_parsers
take python out of route.ts
2 parents 544be26 + 278d5de commit 8c7e3d5

File tree

7 files changed

+89
-52
lines changed

7 files changed

+89
-52
lines changed

‎app/api/repo/route.ts‎

Lines changed: 13 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -5,41 +5,21 @@ import http from 'isomorphic-git/http/node';
55
import Parser from 'web-tree-sitter';
66

77
import { promises as fs } from 'fs';
8-
import { Language, SyntaxNode } from 'web-tree-sitter';
8+
import { SyntaxNode } from 'web-tree-sitter';
99
import { NextRequest, NextResponse } from "next/server";
1010
import { FalkorDB, Graph } from 'falkordb';
1111
import { RESPOSITORIES } from './repositories';
12+
import Language from '@/lib/languages/language';
13+
import Python from '@/lib/languages/python'
1214

1315
const GraphOps = require('./graph_ops');
1416
const LIMITED_MODE = process.env.NEXT_PUBLIC_MODE?.toLowerCase() === 'limited';
1517

1618
//-----------------------------------------------------------------------------
1719
// Tree-Sitter queries
1820
//-----------------------------------------------------------------------------
19-
2021
let parser: Parser;
21-
let Python: Language;
22-
23-
// class definition tree-sitter query
24-
// responsible for matching class definition, in addition to extracting the class name
25-
let class_definition_query: Parser.Query;
26-
27-
// function definition tree-sitter query
28-
// responsible for matching function definition, in addition to extracting the function name
29-
let function_definition_query: Parser.Query;
30-
31-
// function call tree-sitter query
32-
// responsible for matching function calls, in addition to extracting the callee function name
33-
let function_call_query: Parser.Query;
34-
35-
// function call tree-sitter query
36-
// responsible for matching function calls of type self.f()
37-
// in addition to extracting the callee function name
38-
let function_attr_call_query: Parser.Query;
39-
40-
// identifier tree-sitter query
41-
// responsible for matching Identifier nodes
42-
let identifier_query: Parser.Query;
22+
let language: Language;
4323

4424
// Process Class declaration
4525
async function processClassDeclaration
@@ -84,7 +64,7 @@ async function processFunctionDeclaration
8464
const file_components = source_file.split("/");
8565
parent = file_components[file_components.length - 1];
8666
} else {
87-
let identifier_matches = identifier_query.matches(parent)[0].captures;
67+
let identifier_matches = language.identifier_query.matches(parent)[0].captures;
8868
const identifierNode = identifier_matches[0].node;
8969
parent = identifierNode.text;
9070
}
@@ -101,7 +81,7 @@ async function processFunctionDeclaration
10181
if (child_node.type == 'identifier') {
10282
args.push(child_node.text);
10383
} else {
104-
let identifier_matches = identifier_query.matches(child_node)
84+
let identifier_matches = language.identifier_query.matches(child_node)
10585
if (identifier_matches.length == 0) {
10686
console.log("Investigate!");
10787
continue;
@@ -162,15 +142,15 @@ async function processFirstPass
162142
const tree = parser.parse(src);
163143

164144
// Match all Class definitions
165-
let class_matches = class_definition_query.matches(tree.rootNode);
145+
let class_matches = language.class_definition_query.matches(tree.rootNode);
166146

167147
// Iterate over each matched Class
168148
for (let class_match of class_matches) {
169149
await processClassDeclaration(source_file, graph, class_match);
170150
}
171151

172152
// Match all function definition within the current class
173-
let function_matches = function_definition_query.matches(tree.rootNode);
153+
let function_matches = language.function_definition_query.matches(tree.rootNode);
174154
for (let function_match of function_matches) {
175155
await processFunctionDeclaration(source_file, graph, function_match);
176156
}
@@ -193,7 +173,7 @@ async function processSecondPass
193173
const tree = parser.parse(src);
194174

195175
// Match all Function definitions
196-
let function_matches = function_definition_query.matches(tree.rootNode);
176+
let function_matches = language.function_definition_query.matches(tree.rootNode);
197177

198178
// Iterate over each matched Function
199179
for (let function_match of function_matches) {
@@ -203,14 +183,14 @@ async function processSecondPass
203183
let function_src_end = function_node.endPosition.row;
204184

205185
// Match all function calls: `f()` within the current function
206-
let function_call_matches = function_call_query.matches(function_node);
186+
let function_call_matches = language.function_call_query.matches(function_node);
207187
for (let function_call_match of function_call_matches) {
208188
await processFunctionCall(source_file, graph, function_name,
209189
function_src_start, function_src_end, function_call_match);
210190
}
211191

212192
// Match all function calls: `Obj.foo()` within the current function
213-
function_call_matches = function_attr_call_query.matches(function_node);
193+
function_call_matches = language.function_attr_call_query.matches(function_node);
214194
for (let function_call_match of function_call_matches) {
215195
await processFunctionCall(source_file, graph, function_name,
216196
function_src_start, function_src_end, function_call_match);
@@ -267,27 +247,9 @@ async function BuildGraph
267247
}
268248

269249
async function InitializeTreeSitter() {
270-
// Initialize Tree-Sitter parser
271-
await Parser.init({
272-
locateFile(scriptName: string, scriptDirectory: string) {
273-
return path.join(process.cwd(), 'app/parsers', scriptName);
274-
},
275-
});
276-
277250
parser = new Parser();
278-
Python = await Parser.Language.load(path.join(process.cwd(), 'app/parsers/tree-sitter-python.wasm'));
279-
280-
parser.setLanguage(Python);
281-
282-
//-------------------------------------------------------------------------
283-
// Tree-Sitter AST queries
284-
//-------------------------------------------------------------------------
285-
286-
identifier_query = Python.query(`((identifier) @identifier)`);
287-
function_call_query = Python.query(`((call function: (identifier) @function-name) @function-call)`);
288-
function_attr_call_query = Python.query(`((call function: (attribute object: (identifier) attribute: (identifier) @function-name )) @function-call)`);
289-
class_definition_query = Python.query(`(class_definition name: (identifier) @class-name) @class-definition`);
290-
function_definition_query = Python.query(`((function_definition name: (identifier) @function-name parameters: (parameters) @parameters) @function-definition)`);
251+
language = new Python();
252+
parser.setLanguage(language.language);
291253
}
292254

293255
export async function POST(request: NextRequest) {

‎app/parsers/tree-sitter-java.wasm‎

435 KB
Binary file not shown.

‎app/parsers/tree-sitter-rust.wasm‎

1000 KB
Binary file not shown.

‎app/parsers/tree-sitter-typescript.wasm‎

1.33 MB
Binary file not shown.

‎lib/languages/language.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import path from 'path';
2+
import Parser from 'web-tree-sitter';
3+
4+
// Initialize Tree-Sitter parser
5+
await Parser.init({
6+
locateFile(scriptName: string, scriptDirectory: string) {
7+
return path.join(process.cwd(), 'app/parsers', scriptName);
8+
},
9+
});
10+
11+
//-----------------------------------------------------------------------------
12+
// Tree-Sitter queries
13+
//-----------------------------------------------------------------------------
14+
export default class Language {
15+
16+
public language: Parser.Language;
17+
18+
// class definition tree-sitter query
19+
// responsible for matching class definition, in addition to extracting the class name
20+
public class_definition_query: Parser.Query;
21+
22+
// function definition tree-sitter query
23+
// responsible for matching function definition, in addition to extracting the function name
24+
public function_definition_query: Parser.Query;
25+
26+
// function call tree-sitter query
27+
// responsible for matching function calls, in addition to extracting the callee function name
28+
public function_call_query: Parser.Query;
29+
30+
// function call tree-sitter query
31+
// responsible for matching function calls of type self.f()
32+
// in addition to extracting the callee function name
33+
public function_attr_call_query: Parser.Query;
34+
35+
// identifier tree-sitter query
36+
// responsible for matching Identifier nodes
37+
public identifier_query: Parser.Query;
38+
39+
constructor(language: Parser.Language,
40+
class_definition_query: Parser.Query,
41+
function_definition_query: Parser.Query,
42+
function_call_query: Parser.Query,
43+
function_attr_call_query: Parser.Query,
44+
identifier_query: Parser.Query) {
45+
46+
this.language = language;
47+
this.class_definition_query = class_definition_query;
48+
this.function_definition_query = function_definition_query;
49+
this.function_call_query = function_call_query;
50+
this.function_attr_call_query = function_attr_call_query;
51+
this.identifier_query = identifier_query;
52+
}
53+
54+
}

‎lib/languages/python.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import path from 'path';
2+
import Parser from 'web-tree-sitter';
3+
import Language from './language';
4+
5+
const PYTHON_LANG = await Parser.Language.load(path.join(process.cwd(), 'app/parsers/tree-sitter-python.wasm'));
6+
7+
//-----------------------------------------------------------------------------
8+
// Tree-Sitter queries
9+
//-----------------------------------------------------------------------------
10+
export default class Python extends Language {
11+
12+
constructor() {
13+
super(
14+
PYTHON_LANG,
15+
PYTHON_LANG.query(`(class_definition name: (identifier) @class-name) @class-definition`),
16+
PYTHON_LANG.query(`((function_definition name: (identifier) @function-name parameters: (parameters) @parameters) @function-definition)`),
17+
PYTHON_LANG.query(`((call function: (identifier) @function-name) @function-call)`),
18+
PYTHON_LANG.query(`((call function: (attribute object: (identifier) attribute: (identifier) @function-name )) @function-call)`),
19+
PYTHON_LANG.query(`((identifier) @identifier)`))
20+
}
21+
}

‎tsconfig.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"compilerOptions": {
3-
"target": "es5",
3+
"target": "es2017",
44
"lib": ["dom", "dom.iterable", "esnext"],
55
"allowJs": true,
66
"skipLibCheck": true,

0 commit comments

Comments
 (0)