Skip to content

Commit 0f7933d

Browse files
giladgdgiladgd
committed
feat(JSON Schema grammar): prefixItems, minItems, maxItems support
1 parent 97abbca commit 0f7933d

21 files changed

+1210
-105
lines changed

‎src/evaluator/LlamaJsonSchemaGrammar.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ export class LlamaJsonSchemaGrammar<const T extends Readonly<GbnfJsonSchema>> ex
1111
/**
1212
* Prefer to create a new instance of this class by using `llama.createGrammarForJsonSchema(...)`.
1313
*/
14-
public constructor(llama: Llama, schema: T) {
14+
public constructor(llama: Llama, schema: T & Readonly<GbnfJsonSchema>) {
1515
const grammar = getGbnfGrammarForGbnfJsonSchema(schema);
1616

1717
super(llama, {

‎src/utils/gbnfJson/GbnfGrammarGenerator.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
export class GbnfGrammarGenerator {
22
public rules = new Map<string, string | null>();
3+
public ruleContentToRuleName = new Map<string, string>();
4+
public literalValueRuleNames = new Map<string | number, string>();
35
private ruleId: number = 0;
6+
private valueRuleId: number = 0;
47

58
public generateRuleName() {
69
const ruleId = this.ruleId;
@@ -9,6 +12,19 @@ export class GbnfGrammarGenerator {
912
return `rule${ruleId}`;
1013
}
1114

15+
public generateRuleNameForLiteralValue(value: string | number) {
16+
const existingRuleName = this.literalValueRuleNames.get(value);
17+
if (existingRuleName != null)
18+
return existingRuleName;
19+
20+
const ruleName = `val${this.valueRuleId}`;
21+
this.valueRuleId++;
22+
23+
this.literalValueRuleNames.set(value, ruleName);
24+
25+
return ruleName;
26+
}
27+
1228
public generateGbnfFile(rootGrammar: string) {
1329
const rules: {name: string, grammar: string}[] = [{
1430
name: "root",
@@ -30,4 +46,8 @@ export class GbnfGrammarGenerator {
3046

3147
return gbnf;
3248
}
49+
50+
public getProposedLiteralValueRuleNameLength() {
51+
return `val${this.valueRuleId}`.length;
52+
}
3353
}

‎src/utils/gbnfJson/GbnfTerminal.ts‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,16 @@ import {GbnfGrammarGenerator} from "./GbnfGrammarGenerator.js";
44
export abstract class GbnfTerminal {
55
private _ruleName: string | null = null;
66

7+
/** To be used only by `getRuleName` */
8+
protected generateRuleName(grammarGenerator: GbnfGrammarGenerator): string {
9+
return grammarGenerator.generateRuleName();
10+
}
11+
712
protected getRuleName(grammarGenerator: GbnfGrammarGenerator): string {
813
if (this._ruleName != null)
914
return this._ruleName;
1015

11-
const ruleName = grammarGenerator.generateRuleName();
16+
const ruleName = this.generateRuleName(grammarGenerator);
1217
this._ruleName = ruleName;
1318

1419
return ruleName;
@@ -17,11 +22,30 @@ export abstract class GbnfTerminal {
1722
public abstract getGrammar(grammarGenerator: GbnfGrammarGenerator): string;
1823

1924
public resolve(grammarGenerator: GbnfGrammarGenerator): string {
25+
if (this._ruleName != null)
26+
return this._ruleName;
27+
28+
const grammar = this.getGrammar(grammarGenerator);
29+
30+
const existingRuleName = grammarGenerator.ruleContentToRuleName.get(grammar);
31+
if (existingRuleName != null) {
32+
this._ruleName = existingRuleName;
33+
return existingRuleName;
34+
}
35+
2036
const ruleName = this.getRuleName(grammarGenerator);
2137

22-
if (!grammarGenerator.rules.has(ruleName))
23-
grammarGenerator.rules.set(ruleName, this.getGrammar(grammarGenerator));
38+
if (grammar === ruleName) {
39+
this._ruleName = ruleName;
40+
return ruleName;
41+
}
2442

43+
if (!grammarGenerator.rules.has(ruleName)) {
44+
grammarGenerator.rules.set(ruleName, grammar);
45+
grammarGenerator.ruleContentToRuleName.set(grammar, ruleName);
46+
}
47+
48+
this._ruleName = ruleName;
2549
return ruleName;
2650
}
2751
}

‎src/utils/gbnfJson/terminals/GbnfAnyJson.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import {GbnfTerminal} from "../GbnfTerminal.js";
2+
import {GbnfGrammarGenerator} from "../GbnfGrammarGenerator.js";
3+
import {GbnfJsonScopeState} from "../utils/GbnfJsonScopeState.js";
4+
import {GbnfString} from "./GbnfString.js";
5+
import {GbnfOr} from "./GbnfOr.js";
6+
import {GbnfNumber} from "./GbnfNumber.js";
7+
import {GbnfBoolean} from "./GbnfBoolean.js";
8+
import {GbnfNull} from "./GbnfNull.js";
9+
import {GbnfArray} from "./GbnfArray.js";
10+
import {reservedRuleNames} from "./gbnfConsts.js";
11+
12+
13+
export class GbnfAnyJson extends GbnfTerminal {
14+
public readonly scopeState: GbnfJsonScopeState;
15+
16+
public constructor(scopeState: GbnfJsonScopeState = new GbnfJsonScopeState()) {
17+
super();
18+
19+
this.scopeState = scopeState;
20+
}
21+
22+
public getGrammar(grammarGenerator: GbnfGrammarGenerator): string {
23+
const subAnyJsonScopeItem = this.scopeState.settings.allowNewLines
24+
? new GbnfAnyJson(
25+
new GbnfJsonScopeState({
26+
allowNewLines: false,
27+
scopePadSpaces: this.scopeState.settings.scopePadSpaces
28+
}, this.scopeState.currentNestingScope)
29+
)
30+
: new GbnfSubAnyJson(this.scopeState);
31+
32+
return new GbnfOr([
33+
new GbnfString(),
34+
new GbnfNumber({allowFractional: true}),
35+
new GbnfBoolean(),
36+
new GbnfNull(),
37+
new GbnfArray({
38+
items: subAnyJsonScopeItem,
39+
scopeState: this.scopeState
40+
})
41+
// TODO: Add support for object maps
42+
]).getGrammar(grammarGenerator);
43+
}
44+
45+
protected override getRuleName(): string {
46+
return reservedRuleNames.anyJson({
47+
allowNewLines: this.scopeState.settings.allowNewLines,
48+
scopeSpaces: this.scopeState.settings.scopePadSpaces,
49+
nestingScope: this.scopeState.currentNestingScope
50+
});
51+
}
52+
}
53+
54+
class GbnfSubAnyJson extends GbnfAnyJson {
55+
public override getGrammar(): string {
56+
return this.getRuleName();
57+
}
58+
}

‎src/utils/gbnfJson/terminals/GbnfArray.ts‎

Lines changed: 75 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,39 +3,99 @@ import {GbnfGrammarGenerator} from "../GbnfGrammarGenerator.js";
33
import {GbnfJsonScopeState} from "../utils/GbnfJsonScopeState.js";
44
import {GbnfWhitespace} from "./GbnfWhitespace.js";
55
import {GbnfGrammar} from "./GbnfGrammar.js";
6-
import {GbnfOr} from "./GbnfOr.js";
6+
import {GbnfRepetition} from "./GbnfRepetition.js";
7+
import {GbnfCommaWhitespace} from "./GbnfCommaWhitespace.js";
8+
import {GbnfAnyJson} from "./GbnfAnyJson.js";
79

810

911
export class GbnfArray extends GbnfTerminal {
10-
public readonly items: GbnfTerminal;
12+
public readonly items?: GbnfTerminal;
13+
public readonly prefixItems?: GbnfTerminal[];
14+
public readonly minItems: number;
15+
public readonly maxItems?: number;
1116
public readonly scopeState: GbnfJsonScopeState;
1217

13-
public constructor(items: GbnfTerminal, scopeState: GbnfJsonScopeState = new GbnfJsonScopeState()) {
18+
public constructor({
19+
items, prefixItems, minItems = 0, maxItems,
20+
scopeState = new GbnfJsonScopeState()
21+
}: {
22+
items?: GbnfTerminal, prefixItems?: GbnfTerminal[], minItems?: number, maxItems?: number,
23+
scopeState: GbnfJsonScopeState
24+
}) {
1425
super();
1526

1627
this.items = items;
28+
this.prefixItems = prefixItems;
29+
this.minItems = Math.floor(minItems);
30+
this.maxItems = maxItems == null ? undefined : Math.floor(maxItems);
1731
this.scopeState = scopeState;
32+
33+
if (this.prefixItems != null && this.minItems < this.prefixItems.length)
34+
this.minItems = this.prefixItems.length;
35+
else if (this.minItems < 0)
36+
this.minItems = 0;
37+
38+
if (this.maxItems != null && this.maxItems < this.minItems)
39+
this.maxItems = this.minItems;
40+
else if (this.maxItems != null && this.maxItems < 0)
41+
this.maxItems = 0;
1842
}
1943

2044
public getGrammar(grammarGenerator: GbnfGrammarGenerator): string {
45+
const getWhitespaceRule = (newScope: boolean, newLine: "before" | "after" | false) => (
46+
newScope
47+
? new GbnfWhitespace(this.scopeState.getForNewScope(), {newLine})
48+
: new GbnfWhitespace(this.scopeState, {newLine})
49+
);
2150
const getWhitespaceRuleName = (newScope: boolean, newLine: "before" | "after" | false) => (
51+
getWhitespaceRule(newScope, newLine).resolve(grammarGenerator)
52+
);
53+
54+
const getCommaWhitespaceRule = (newScope: boolean, newLine: "before" | "after" | false) => (
2255
newScope
23-
? new GbnfWhitespace(this.scopeState.getForNewScope(), {newLine}).resolve(grammarGenerator)
24-
: new GbnfWhitespace(this.scopeState, {newLine}).resolve(grammarGenerator)
56+
? new GbnfCommaWhitespace(this.scopeState.getForNewScope(), {newLine})
57+
: new GbnfCommaWhitespace(this.scopeState, {newLine})
58+
);
59+
const getCommaWhitespaceRuleName = (newScope: boolean, newLine: "before" | "after" | false) => (
60+
getCommaWhitespaceRule(newScope, newLine).resolve(grammarGenerator)
2561
);
26-
const itemsGrammarRuleName = this.items.resolve(grammarGenerator);
62+
63+
const arrayItemsGrammar: string[] = [];
64+
if (this.prefixItems != null && this.prefixItems.length > 0) {
65+
for (const item of this.prefixItems) {
66+
if (arrayItemsGrammar.length > 0) {
67+
arrayItemsGrammar.push(getCommaWhitespaceRuleName(true, "before"));
68+
}
69+
70+
arrayItemsGrammar.push(item.resolve(grammarGenerator));
71+
}
72+
73+
if (this.minItems > this.prefixItems.length || this.maxItems == null || this.maxItems > this.prefixItems.length) {
74+
arrayItemsGrammar.push(getCommaWhitespaceRuleName(true, "before"));
75+
arrayItemsGrammar.push(
76+
new GbnfRepetition({
77+
value: this.items ?? new GbnfAnyJson(),
78+
separator: getCommaWhitespaceRule(true, "before"),
79+
minRepetitions: this.minItems - this.prefixItems.length,
80+
maxRepetitions: this.maxItems == null
81+
? undefined
82+
: this.maxItems - this.prefixItems.length
83+
}).getGrammar(grammarGenerator)
84+
);
85+
}
86+
} else
87+
arrayItemsGrammar.push(
88+
new GbnfRepetition({
89+
value: this.items ?? new GbnfAnyJson(),
90+
separator: getCommaWhitespaceRule(true, "before"),
91+
minRepetitions: this.minItems,
92+
maxRepetitions: this.maxItems
93+
}).getGrammar(grammarGenerator)
94+
);
2795

2896
return new GbnfGrammar([
2997
'"["', getWhitespaceRuleName(true, "before"),
30-
new GbnfOr([
31-
new GbnfGrammar([
32-
"(", itemsGrammarRuleName, ")",
33-
"(", '","', getWhitespaceRuleName(true, "before"), itemsGrammarRuleName, ")*"
34-
]),
35-
new GbnfGrammar([
36-
"(", itemsGrammarRuleName, ")?"
37-
])
38-
]).getGrammar(grammarGenerator),
98+
new GbnfGrammar(arrayItemsGrammar).getGrammar(),
3999
getWhitespaceRuleName(false, "before"), '"]"'
40100
]).getGrammar();
41101
}

‎src/utils/gbnfJson/terminals/GbnfBoolean.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export class GbnfBoolean extends GbnfTerminal {
1010
return new GbnfOr([
1111
new GbnfGrammar('"true"'),
1212
new GbnfGrammar('"false"')
13-
]).getGrammar(grammarGenerator);
13+
], true).getGrammar(grammarGenerator);
1414
}
1515

1616
protected override getRuleName(): string {

‎src/utils/gbnfJson/terminals/GbnfCommaWhitespace.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import {GbnfTerminal} from "../GbnfTerminal.js";
2+
import {GbnfJsonScopeState} from "../utils/GbnfJsonScopeState.js";
3+
import {GbnfGrammar} from "./GbnfGrammar.js";
4+
import {GbnfWhitespace} from "./GbnfWhitespace.js";
5+
import {reservedRuleNames} from "./gbnfConsts.js";
6+
7+
8+
export class GbnfCommaWhitespace extends GbnfTerminal {
9+
public readonly scopeState: GbnfJsonScopeState;
10+
public readonly newLine: "before" | "after" | false;
11+
12+
public constructor(scopeState: GbnfJsonScopeState, {
13+
newLine = "before"
14+
}: {
15+
newLine?: "before" | "after" | false
16+
} = {}) {
17+
super();
18+
this.scopeState = scopeState;
19+
this.newLine = newLine;
20+
}
21+
22+
public getGrammar(): string {
23+
return new GbnfGrammar([
24+
'","', new GbnfWhitespace(this.scopeState, {newLine: this.newLine}).getGrammar()
25+
]).getGrammar();
26+
}
27+
28+
protected override getRuleName(): string {
29+
return reservedRuleNames.commaWhitespace({
30+
newLine: this.scopeState.settings.allowNewLines
31+
? this.newLine
32+
: false,
33+
scopeSpaces: this.scopeState.settings.scopePadSpaces,
34+
nestingScope: this.scopeState.currentNestingScope
35+
});
36+
}
37+
}

‎src/utils/gbnfJson/terminals/GbnfNumberValue.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import {GbnfTerminal} from "../GbnfTerminal.js";
2+
import {GbnfGrammarGenerator} from "../GbnfGrammarGenerator.js";
23

34

45
export class GbnfNumberValue extends GbnfTerminal {
@@ -13,7 +14,15 @@ export class GbnfNumberValue extends GbnfTerminal {
1314
return '"' + JSON.stringify(this.value) + '"';
1415
}
1516

16-
public override resolve(): string {
17-
return this.getGrammar();
17+
public override resolve(grammarGenerator: GbnfGrammarGenerator): string {
18+
const grammar = this.getGrammar();
19+
if (grammar.length <= grammarGenerator.getProposedLiteralValueRuleNameLength())
20+
return grammar;
21+
22+
return super.resolve(grammarGenerator);
23+
}
24+
25+
protected override generateRuleName(grammarGenerator: GbnfGrammarGenerator): string {
26+
return grammarGenerator.generateRuleNameForLiteralValue(this.value);
1827
}
1928
}

‎src/utils/gbnfJson/terminals/GbnfOr.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,21 @@ import {grammarNoValue} from "./gbnfConsts.js";
55

66
export class GbnfOr extends GbnfTerminal {
77
public readonly values: readonly GbnfTerminal[];
8+
public readonly useRawGrammar: boolean;
89

9-
public constructor(values: readonly GbnfTerminal[]) {
10+
public constructor(values: readonly GbnfTerminal[], useRawGrammar: boolean = false) {
1011
super();
1112
this.values = values;
13+
this.useRawGrammar = useRawGrammar;
1214
}
1315

1416
public getGrammar(grammarGenerator: GbnfGrammarGenerator): string {
1517
const mappedValues = this.values
16-
.map((v) => v.resolve(grammarGenerator))
18+
.map((v) => (
19+
this.useRawGrammar
20+
? v.getGrammar(grammarGenerator)
21+
: v.resolve(grammarGenerator)
22+
))
1723
.filter((value) => value !== "" && value !== grammarNoValue);
1824

1925
if (mappedValues.length === 0)

0 commit comments

Comments
 (0)