Skip to content

Commit f2bfa34

Browse files
claudeclaude
committed
Enter nullable multi-token RuleRefs in sll_advance for deeper prediction
Modify sll_advance_inner to enter nullable elements containing multi-token RuleRefs (e.g., with_clause?) via the return stack, instead of treating them as single-token consumers. This fixes the depth mismatch that caused try_expand_opaque to skip rules starting with nullable elements. When sll_advance encounters a nullable Repeat(Optional/Star, RuleRef): - If the RuleRef is single-token: advance past it (unchanged) - If multi-token and return_stack depth < 1: push continuation, enter the rule's alternatives, advance inside - Otherwise: fall back to pos+1 (legacy behavior) Guards: return_stack depth < 1, alt count <= 8, FIRST pre-filter. Results for SQLite: 298 → 253 backtracking sites (-15%). All 1301 tests pass. No correctness regressions. https://claude.ai/code/session_01ACVN5Rr7waUZWXtv8MFN2C
1 parent fc9d864 commit f2bfa34

File tree

2 files changed

+121
-116
lines changed

2 files changed

+121
-116
lines changed

‎package-gale/src/parser_gen.wado‎

Lines changed: 55 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,52 @@ fn sll_advance_inner(c: &SllConfig, token: &String, all_rules: &Array<ParserRule
188188
let mut results: Array<SllConfig> = [];
189189
let mut visiting: Array<String> = [];
190190
if array_contains_str(&first_of_element(elem, all_rules, lit_tokens, &mut visiting), token) {
191-
results.append(SllConfig { alt_index: c.alt_index, elements: c.elements, pos: c.pos + 1, return_stack: c.return_stack });
191+
// Check if the inner element is a multi-token RuleRef.
192+
// If so, enter it via return stack instead of skipping to pos+1.
193+
let mut entered = false;
194+
if let Repeat(rep) = *elem {
195+
if let RuleRef(inner_name) = rep.element {
196+
let mut stv: Array<String> = [];
197+
if !rule_is_single_token(&inner_name, all_rules, &mut stv) && c.return_stack.len() < 1 {
198+
let mut inner_alt_count = 0;
199+
for let rule of all_rules {
200+
if rule.name == inner_name {
201+
inner_alt_count = rule.alternatives.len();
202+
break;
203+
}
204+
}
205+
if inner_alt_count <= 8 {
206+
entered = true;
207+
let new_stack = push_return(&c.return_stack, c.elements, c.pos + 1);
208+
for let rule of all_rules {
209+
if rule.name != inner_name {
210+
continue;
211+
}
212+
for let alt of rule.alternatives {
213+
let mut v2: Array<String> = [];
214+
if !array_contains_str(&first_of_alt(&alt, all_rules, lit_tokens, &mut v2), token) {
215+
continue;
216+
}
217+
let exp = SllConfig {
218+
alt_index: c.alt_index,
219+
elements: alt.elements,
220+
pos: 0,
221+
return_stack: new_stack,
222+
};
223+
let advanced = sll_advance_inner(&exp, token, all_rules, lit_tokens, inline_depth + 1);
224+
for let a of advanced {
225+
results.append(a);
226+
}
227+
}
228+
break;
229+
}
230+
}
231+
}
232+
}
233+
}
234+
if !entered {
235+
results.append(SllConfig { alt_index: c.alt_index, elements: c.elements, pos: c.pos + 1, return_stack: c.return_stack });
236+
}
192237
}
193238
// Also try skipping: advance past all consecutive nullables non-recursively.
194239
let mut skip_pos = c.pos + 1;
@@ -521,22 +566,13 @@ fn try_expand_opaque(original_configs: &Array<SllConfig>, token: &String, non_op
521566
}
522567
if let RuleRef(name) = c.elements[c.pos] {
523568
let mut alt_count = 0;
524-
let mut has_nullable_start = false;
525569
for let rule of all_rules {
526570
if rule.name == name {
527571
alt_count = rule.alternatives.len();
528-
// Check if any alternative starts with a nullable element.
529-
// Nullable-start rules cause depth mismatch: sll_advance skips
530-
// the nullable, producing FIRST at a different input depth.
531-
for let alt of rule.alternatives {
532-
if !alt.elements.is_empty() && is_nullable(&alt.elements[0]) {
533-
has_nullable_start = true;
534-
}
535-
}
536572
break;
537573
}
538574
}
539-
if alt_count > 8 || has_nullable_start {
575+
if alt_count > 8 {
540576
continue;
541577
}
542578
let new_stack = push_return(&c.return_stack, c.elements, c.pos + 1);
@@ -625,6 +661,14 @@ fn try_expand_opaque(original_configs: &Array<SllConfig>, token: &String, non_op
625661
if !has_improvement || branches.is_empty() {
626662
return null;
627663
}
664+
// Only use if ALL branches are Leaf — any Backtrack branch means expansion
665+
// didn't fully resolve, and the code generator's Dispatch has no fallback
666+
// for unmatched tokens, which would silently drop valid inputs.
667+
for let b of branches {
668+
if let Backtrack(_) = b.child {
669+
return null;
670+
}
671+
}
628672
// Safety: verify all original opaque alts appear in at least one branch.
629673
let mut covered_alts: Array<i32> = [];
630674
for let b of branches {
@@ -633,13 +677,6 @@ fn try_expand_opaque(original_configs: &Array<SllConfig>, token: &String, non_op
633677
covered_alts.append(idx);
634678
}
635679
}
636-
if let Backtrack(idxs) = b.child {
637-
for let idx of idxs {
638-
if !array_contains_i32(&covered_alts, idx) {
639-
covered_alts.append(idx);
640-
}
641-
}
642-
}
643680
}
644681
for let oa of opaque_alts {
645682
if !array_contains_i32(&covered_alts, oa) {

‎package-gale/tests/golden/sqlite.wado‎

Lines changed: 66 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -6244,64 +6244,66 @@ fn parse_sql_stmt(p: &mut Parser) -> Result<SqlStmtNode, ParseError> {
62446244
}
62456245
} else if grp_kind == TK_K_CREATE {
62466246
if p.peek_kind() == TK_K_CREATE {
6247-
if p.peek_at(1) == TK_K_UNIQUE || p.peek_at(1) == TK_K_INDEX {
6248-
let tok_11 = parse_create_index_stmt(p)?;
6249-
} else if p.peek_at(1) == TK_K_TEMP || p.peek_at(1) == TK_K_TEMPORARY {
6250-
let saved_pos_5 = p.pos;
6251-
let mut bt_done_5 = false;
6252-
if !bt_done_5 {
6253-
bt_try_14: {
6254-
let opt_r = parse_create_table_stmt(p);
6255-
if let Err(_) = opt_r { p.pos = saved_pos_5; break bt_try_14; }
6256-
bt_done_5 = true;
6257-
}
6247+
let saved_pos_5 = p.pos;
6248+
let mut bt_done_5 = false;
6249+
if !bt_done_5 {
6250+
bt_try_14: {
6251+
let opt_r = parse_create_index_stmt(p);
6252+
if let Err(_) = opt_r { p.pos = saved_pos_5; break bt_try_14; }
6253+
bt_done_5 = true;
62586254
}
6259-
if !bt_done_5 {
6260-
bt_try_15: {
6261-
let opt_r = parse_create_trigger_stmt(p);
6262-
if let Err(_) = opt_r { p.pos = saved_pos_5; break bt_try_15; }
6263-
bt_done_5 = true;
6264-
}
6255+
}
6256+
if !bt_done_5 {
6257+
bt_try_15: {
6258+
let opt_r = parse_create_table_stmt(p);
6259+
if let Err(_) = opt_r { p.pos = saved_pos_5; break bt_try_15; }
6260+
bt_done_5 = true;
62656261
}
6266-
if !bt_done_5 {
6267-
let tok = parse_create_view_stmt(p)?;
6262+
}
6263+
if !bt_done_5 {
6264+
bt_try_16: {
6265+
let opt_r = parse_create_trigger_stmt(p);
6266+
if let Err(_) = opt_r { p.pos = saved_pos_5; break bt_try_16; }
6267+
bt_done_5 = true;
62686268
}
6269-
} else if p.peek_at(1) == TK_K_TABLE {
6270-
let tok_12 = parse_create_table_stmt(p)?;
6271-
} else if p.peek_at(1) == TK_K_TRIGGER {
6272-
let tok_13 = parse_create_trigger_stmt(p)?;
6273-
} else if p.peek_at(1) == TK_K_VIEW {
6274-
let tok_14 = parse_create_view_stmt(p)?;
6275-
} else if p.peek_at(1) == TK_K_VIRTUAL {
6276-
let tok_15 = parse_create_virtual_table_stmt(p)?;
6269+
}
6270+
if !bt_done_5 {
6271+
bt_try_17: {
6272+
let opt_r = parse_create_view_stmt(p);
6273+
if let Err(_) = opt_r { p.pos = saved_pos_5; break bt_try_17; }
6274+
bt_done_5 = true;
6275+
}
6276+
}
6277+
if !bt_done_5 {
6278+
let tok = parse_create_virtual_table_stmt(p)?;
62776279
}
62786280
}
62796281
} else if grp_kind == TK_K_DETACH {
6280-
let tok_16 = parse_detach_stmt(p)?;
6282+
let tok_11 = parse_detach_stmt(p)?;
62816283
} else if grp_kind == TK_K_DROP {
62826284
if p.peek_kind() == TK_K_DROP {
62836285
if p.peek_at(1) == TK_K_INDEX {
6284-
let tok_17 = parse_drop_index_stmt(p)?;
6286+
let tok_12 = parse_drop_index_stmt(p)?;
62856287
} else if p.peek_at(1) == TK_K_TABLE {
6286-
let tok_18 = parse_drop_table_stmt(p)?;
6288+
let tok_13 = parse_drop_table_stmt(p)?;
62876289
} else if p.peek_at(1) == TK_K_TRIGGER {
6288-
let tok_19 = parse_drop_trigger_stmt(p)?;
6290+
let tok_14 = parse_drop_trigger_stmt(p)?;
62896291
} else if p.peek_at(1) == TK_K_VIEW {
6290-
let tok_20 = parse_drop_view_stmt(p)?;
6292+
let tok_15 = parse_drop_view_stmt(p)?;
62916293
}
62926294
}
62936295
} else if grp_kind == TK_K_PRAGMA {
6294-
let tok_21 = parse_pragma_stmt(p)?;
6296+
let tok_16 = parse_pragma_stmt(p)?;
62956297
} else if grp_kind == TK_K_REINDEX {
6296-
let tok_22 = parse_reindex_stmt(p)?;
6298+
let tok_17 = parse_reindex_stmt(p)?;
62976299
} else if grp_kind == TK_K_RELEASE {
6298-
let tok_23 = parse_release_stmt(p)?;
6300+
let tok_18 = parse_release_stmt(p)?;
62996301
} else if grp_kind == TK_K_ROLLBACK {
6300-
let tok_24 = parse_rollback_stmt(p)?;
6302+
let tok_19 = parse_rollback_stmt(p)?;
63016303
} else if grp_kind == TK_K_SAVEPOINT {
6302-
let tok_25 = parse_savepoint_stmt(p)?;
6304+
let tok_20 = parse_savepoint_stmt(p)?;
63036305
} else if grp_kind == TK_K_VACUUM {
6304-
let tok_26 = parse_vacuum_stmt(p)?;
6306+
let tok_21 = parse_vacuum_stmt(p)?;
63056307
}
63066308
return Result::<SqlStmtNode, ParseError>::Ok(SqlStmtNode {
63076309
span: Span::new(start, p.last_end()),
@@ -6720,84 +6722,50 @@ fn parse_create_trigger_stmt(p: &mut Parser) -> Result<CreateTriggerStmtNode, Pa
67206722
let grp_kind_2 = p.peek_kind();
67216723
if grp_kind_2 == TK_K_WITH || grp_kind_2 == TK_K_UPDATE || grp_kind_2 == TK_K_INSERT || grp_kind_2 == TK_K_REPLACE || grp_kind_2 == TK_K_DELETE || grp_kind_2 == TK_K_SELECT || grp_kind_2 == TK_K_VALUES {
67226724
if p.peek_kind() == TK_K_WITH {
6723-
let saved_pos = p.pos;
6724-
let mut bt_done = false;
6725-
if !bt_done {
6726-
bt_try: {
6727-
let opt_r = parse_update_stmt(p);
6728-
if let Err(_) = opt_r { p.pos = saved_pos; break bt_try; }
6729-
bt_done = true;
6730-
}
6731-
}
6732-
if !bt_done {
6733-
bt_try_2: {
6734-
let opt_r = parse_insert_stmt(p);
6735-
if let Err(_) = opt_r { p.pos = saved_pos; break bt_try_2; }
6736-
bt_done = true;
6737-
}
6738-
}
6739-
if !bt_done {
6740-
bt_try_3: {
6741-
let opt_r = parse_delete_stmt(p);
6742-
if let Err(_) = opt_r { p.pos = saved_pos; break bt_try_3; }
6743-
bt_done = true;
6744-
}
6745-
}
6746-
if !bt_done {
6747-
let tok = parse_select_stmt(p)?;
6725+
if p.peek_at(1) == TK_K_UPDATE {
6726+
let tok_14 = parse_update_stmt(p)?;
6727+
} else if p.peek_at(1) == TK_K_INSERT || p.peek_at(1) == TK_K_REPLACE {
6728+
let tok_15 = parse_insert_stmt(p)?;
6729+
} else if p.peek_at(1) == TK_K_DELETE {
6730+
let tok_16 = parse_delete_stmt(p)?;
6731+
} else if p.peek_at(1) == TK_K_SELECT || p.peek_at(1) == TK_K_VALUES {
6732+
let tok_17 = parse_select_stmt(p)?;
67486733
}
67496734
} else if p.peek_kind() == TK_K_UPDATE {
6750-
let tok_14 = parse_update_stmt(p)?;
6735+
let tok_18 = parse_update_stmt(p)?;
67516736
} else if p.peek_kind() == TK_K_INSERT || p.peek_kind() == TK_K_REPLACE {
6752-
let tok_15 = parse_insert_stmt(p)?;
6737+
let tok_19 = parse_insert_stmt(p)?;
67536738
} else if p.peek_kind() == TK_K_DELETE {
6754-
let tok_16 = parse_delete_stmt(p)?;
6739+
let tok_20 = parse_delete_stmt(p)?;
67556740
} else if p.peek_kind() == TK_K_SELECT || p.peek_kind() == TK_K_VALUES {
6756-
let tok_17 = parse_select_stmt(p)?;
6741+
let tok_21 = parse_select_stmt(p)?;
67576742
}
67586743
}
6759-
let tok_18 = p.expect(TK_LIT_SEMI, "';'")?;
6744+
let tok_22 = p.expect(TK_LIT_SEMI, "';'")?;
67606745
while p.peek_kind() == TK_K_WITH || p.peek_kind() == TK_K_UPDATE || p.peek_kind() == TK_K_INSERT || p.peek_kind() == TK_K_REPLACE || p.peek_kind() == TK_K_DELETE || p.peek_kind() == TK_K_SELECT || p.peek_kind() == TK_K_VALUES {
67616746
let grp_kind = p.peek_kind();
67626747
if grp_kind == TK_K_WITH || grp_kind == TK_K_UPDATE || grp_kind == TK_K_INSERT || grp_kind == TK_K_REPLACE || grp_kind == TK_K_DELETE || grp_kind == TK_K_SELECT || grp_kind == TK_K_VALUES {
67636748
if p.peek_kind() == TK_K_WITH {
6764-
let saved_pos = p.pos;
6765-
let mut bt_done = false;
6766-
if !bt_done {
6767-
bt_try: {
6768-
let opt_r = parse_update_stmt(p);
6769-
if let Err(_) = opt_r { p.pos = saved_pos; break bt_try; }
6770-
bt_done = true;
6771-
}
6772-
}
6773-
if !bt_done {
6774-
bt_try_2: {
6775-
let opt_r = parse_insert_stmt(p);
6776-
if let Err(_) = opt_r { p.pos = saved_pos; break bt_try_2; }
6777-
bt_done = true;
6778-
}
6779-
}
6780-
if !bt_done {
6781-
bt_try_3: {
6782-
let opt_r = parse_delete_stmt(p);
6783-
if let Err(_) = opt_r { p.pos = saved_pos; break bt_try_3; }
6784-
bt_done = true;
6785-
}
6786-
}
6787-
if !bt_done {
6788-
let tok = parse_select_stmt(p)?;
6749+
if p.peek_at(1) == TK_K_UPDATE {
6750+
let tok = parse_update_stmt(p)?;
6751+
} else if p.peek_at(1) == TK_K_INSERT || p.peek_at(1) == TK_K_REPLACE {
6752+
let tok_2 = parse_insert_stmt(p)?;
6753+
} else if p.peek_at(1) == TK_K_DELETE {
6754+
let tok_3 = parse_delete_stmt(p)?;
6755+
} else if p.peek_at(1) == TK_K_SELECT || p.peek_at(1) == TK_K_VALUES {
6756+
let tok_4 = parse_select_stmt(p)?;
67896757
}
67906758
} else if p.peek_kind() == TK_K_UPDATE {
6791-
let tok = parse_update_stmt(p)?;
6759+
let tok_5 = parse_update_stmt(p)?;
67926760
} else if p.peek_kind() == TK_K_INSERT || p.peek_kind() == TK_K_REPLACE {
6793-
let tok_2 = parse_insert_stmt(p)?;
6761+
let tok_6 = parse_insert_stmt(p)?;
67946762
} else if p.peek_kind() == TK_K_DELETE {
6795-
let tok_3 = parse_delete_stmt(p)?;
6763+
let tok_7 = parse_delete_stmt(p)?;
67966764
} else if p.peek_kind() == TK_K_SELECT || p.peek_kind() == TK_K_VALUES {
6797-
let tok_4 = parse_select_stmt(p)?;
6765+
let tok_8 = parse_select_stmt(p)?;
67986766
}
67996767
}
6800-
let tok_5 = p.expect(TK_LIT_SEMI, "';'")?;
6768+
let tok_9 = p.expect(TK_LIT_SEMI, "';'")?;
68016769
}
68026770
let k_end = p.expect(TK_K_END, "K_END")?;
68036771
return Result::<CreateTriggerStmtNode, ParseError>::Ok(CreateTriggerStmtNode {

0 commit comments

Comments
 (0)