Skip to content

Commit 016fb86

Browse files
committed
fix(split): set a limit for tree sitter recursion depth
1 parent 53244e4 commit 016fb86

File tree

1 file changed

+5
-1
lines changed

1 file changed

+5
-1
lines changed

src/ops/functions/split_recursively.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ struct Spec {
2424
custom_languages: Vec<CustomLanguageSpec>,
2525
}
2626

27+
const TREESITTER_MAX_RECURSION_DEPTH: usize = 128;
28+
2729
const SYNTAX_LEVEL_GAP_COST: usize = 512;
2830
const MISSING_OVERLAP_COST: usize = 512;
2931
const PER_LINE_BREAK_LEVEL_GAP_COST: usize = 64;
@@ -542,7 +544,9 @@ impl<'t, 's: 't> RecursiveChunker<'s> {
542544
) -> Result<()> {
543545
match chunk.kind {
544546
ChunkKind::TreeSitterNode { lang_config, node } => {
545-
if !lang_config.terminal_node_kind_ids.contains(&node.kind_id()) {
547+
if !lang_config.terminal_node_kind_ids.contains(&node.kind_id())
548+
&& atom_collector.curr_level < TREESITTER_MAX_RECURSION_DEPTH
549+
{
546550
let mut cursor = node.walk();
547551
if cursor.goto_first_child() {
548552
return self.collect_atom_chunks_from_iter(

0 commit comments

Comments
 (0)