Skip to content

Commit 4945ec1

Browse files
mProjectsCodemProjectsCode
committed
a lot of parser work and clean up
1 parent a6abf92 commit 4945ec1

18 files changed

+3467
-12497
lines changed

‎package-lock.json‎

Lines changed: 1636 additions & 9530 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/parsers/newInputFieldParser/InputFieldParser.ts‎

Lines changed: 81 additions & 684 deletions
Large diffs are not rendered by default.

‎src/parsers/newInputFieldParser/InputFieldParsingTreeParser.ts‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import { ParsingError } from './ParsingError';
2+
import { ErrorLevel } from '../../utils/errors/MetaBindErrors';
3+
import { ParsingTree, PT_Closure, PT_Element, PT_Element_Type, PT_Literal } from './ParsingTree';
4+
import { Closure } from './ParsingUtils';
5+
import { InputFieldClosures, InputFieldToken, InputFieldTokenType } from './InputFieldTokenizer';
6+
7+
export class InputFieldParsingTreeParser {
8+
private readonly tokens: InputFieldToken[];
9+
private readonly closureStack: Closure<InputFieldTokenType>[];
10+
private readonly parsingTree: ParsingTree;
11+
private position: number;
12+
13+
constructor(str: string, tokens: InputFieldToken[]) {
14+
this.tokens = tokens;
15+
this.position = 0;
16+
this.closureStack = [];
17+
this.parsingTree = new ParsingTree(str, tokens);
18+
}
19+
20+
public parse(): ParsingTree {
21+
while (this.getCurrentToken().type !== InputFieldTokenType.EOF) {
22+
const astel = this.parseCurrentToken();
23+
this.parsingTree.children.push(astel);
24+
25+
if (this.position >= this.tokens.length) {
26+
throw new Error('index to big');
27+
}
28+
}
29+
30+
return this.parsingTree;
31+
}
32+
33+
private parseCurrentToken(): PT_Element {
34+
const token = this.getCurrentToken();
35+
36+
this.throwOnInvalidToken();
37+
38+
const ptLiteral = new PT_Literal(token, this.parsingTree.str);
39+
40+
for (const closure of InputFieldClosures) {
41+
const ptClosure = this.parseClosure(ptLiteral, closure);
42+
if (ptClosure) {
43+
return ptClosure;
44+
}
45+
}
46+
47+
// move the position to the next token
48+
this.position += 1;
49+
50+
return ptLiteral;
51+
}
52+
53+
private parseClosure(openingLiteral: PT_Literal, closure: Closure<InputFieldTokenType>): PT_Element | undefined {
54+
if (openingLiteral.token.type !== closure.openingTokenType) {
55+
return undefined;
56+
}
57+
58+
this.closureStack.push(closure);
59+
60+
let closingLiteral: PT_Literal | undefined;
61+
const children: PT_Element[] = [];
62+
63+
// skip the opening token
64+
this.position += 1;
65+
66+
while (this.getCurrentToken().type !== InputFieldTokenType.EOF) {
67+
const nestedRes = this.parseCurrentToken();
68+
69+
if (nestedRes.type === PT_Element_Type.LITERAL && (nestedRes as PT_Literal).token.type === closure.closingTokenType) {
70+
closingLiteral = nestedRes as PT_Literal;
71+
break;
72+
} else {
73+
children.push(nestedRes);
74+
}
75+
}
76+
77+
if (!closingLiteral) {
78+
// ERROR
79+
throw new ParsingError(ErrorLevel.ERROR, 'failed to parse', 'closure was not closed', {}, this.parsingTree.str, openingLiteral.token, 'PT Parser');
80+
}
81+
82+
this.closureStack.pop();
83+
84+
return new PT_Closure(this.parsingTree.str, openingLiteral, closingLiteral, children);
85+
}
86+
87+
private getCurrentToken(): InputFieldToken {
88+
return this.tokens[this.position];
89+
}
90+
91+
throwOnInvalidToken(): void {
92+
const token = this.getCurrentToken();
93+
94+
// check for closure closing tokens that do not actually close a closure
95+
const currentClosure = this.closureStack.length > 0 ? this.closureStack[this.closureStack.length - 1] : undefined;
96+
97+
for (const closure of InputFieldClosures) {
98+
// if the closure is the current token
99+
if (
100+
currentClosure &&
101+
closure.openingTokenType === currentClosure.openingTokenType &&
102+
closure.closingTokenType === currentClosure.closingTokenType
103+
) {
104+
continue;
105+
}
106+
107+
// if the current token is a closing token of a closure that is not the active closure
108+
if (token.type === closure.closingTokenType) {
109+
throw new ParsingError(
110+
ErrorLevel.ERROR,
111+
'failed to parse',
112+
'Encountered invalid token. Active closure is not the closure that this token closes.',
113+
{},
114+
this.parsingTree.str,
115+
token,
116+
'PT Parser'
117+
);
118+
}
119+
}
120+
}
121+
}

‎src/parsers/newInputFieldParser/InputFieldTokenizer.ts‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
import { AbstractToken, Closure } from './ParsingUtils';
2+
3+
export enum InputFieldTokenType {
4+
ILLEGAL = 'ILLEGAL',
5+
EOF = 'EOF',
6+
WORD = 'WORD',
7+
L_PAREN = '(',
8+
R_PAREN = ')',
9+
L_SQUARE = '[',
10+
R_SQUARE = ']',
11+
COLON = ':',
12+
HASHTAG = '#',
13+
DOT = '.',
14+
COMMA = ',',
15+
QUOTE = "'",
16+
}
17+
18+
export const InputFieldClosures: Closure<InputFieldTokenType>[] = [
19+
{
20+
openingTokenType: InputFieldTokenType.L_PAREN,
21+
closingTokenType: InputFieldTokenType.R_PAREN,
22+
},
23+
{
24+
openingTokenType: InputFieldTokenType.L_SQUARE,
25+
closingTokenType: InputFieldTokenType.R_SQUARE,
26+
},
27+
];
28+
29+
export interface InputFieldToken extends AbstractToken<InputFieldTokenType> {}
30+
31+
function createToken<TokenType extends string>(type: TokenType, literal: string, from: number, to: number): AbstractToken<TokenType> {
32+
return {
33+
type: type,
34+
literal: literal,
35+
range: {
36+
from: from,
37+
to: to,
38+
},
39+
};
40+
}
41+
42+
export class InputFieldTokenizer {
43+
private readonly input: string;
44+
private tokens: InputFieldToken[];
45+
private currentToken: InputFieldToken | undefined;
46+
private inQuotes: boolean;
47+
private position: number;
48+
private readPosition: number;
49+
private ch!: string;
50+
51+
constructor(input: string) {
52+
this.input = input;
53+
54+
this.tokens = [];
55+
this.currentToken = undefined;
56+
this.inQuotes = false;
57+
this.position = 0;
58+
this.readPosition = 0;
59+
60+
this.readChar();
61+
}
62+
63+
public getTokens(): InputFieldToken[] {
64+
while (this.currentToken?.type !== InputFieldTokenType.EOF) {
65+
this.readNextToken();
66+
}
67+
68+
this.tokens = this.tokens.filter(x => x.type !== InputFieldTokenType.QUOTE);
69+
70+
return this.tokens;
71+
}
72+
73+
private readNextToken(): void {
74+
let token: InputFieldToken | undefined;
75+
76+
if (this.inQuotes) {
77+
if (this.ch === "'") {
78+
token = this.createToken(InputFieldTokenType.QUOTE);
79+
this.inQuotes = false;
80+
} else if (this.currentToken && this.currentToken.type === InputFieldTokenType.WORD) {
81+
this.currentToken.literal += this.ch;
82+
this.currentToken.range.to = this.position;
83+
} else {
84+
token = this.createToken(InputFieldTokenType.WORD);
85+
}
86+
} else {
87+
if (this.ch === '(') {
88+
token = this.createToken(InputFieldTokenType.L_PAREN);
89+
} else if (this.ch === ')') {
90+
token = this.createToken(InputFieldTokenType.R_PAREN);
91+
} else if (this.ch === '[') {
92+
token = this.createToken(InputFieldTokenType.L_SQUARE);
93+
} else if (this.ch === ']') {
94+
token = this.createToken(InputFieldTokenType.R_SQUARE);
95+
} else if (this.ch === ':') {
96+
token = this.createToken(InputFieldTokenType.COLON);
97+
} else if (this.ch === '#') {
98+
token = this.createToken(InputFieldTokenType.HASHTAG);
99+
} else if (this.ch === '.') {
100+
token = this.createToken(InputFieldTokenType.DOT);
101+
} else if (this.ch === ',') {
102+
token = this.createToken(InputFieldTokenType.COMMA);
103+
} else if (this.ch === "'") {
104+
token = this.createToken(InputFieldTokenType.QUOTE);
105+
this.inQuotes = true;
106+
} else if (this.ch === '\0') {
107+
token = this.createToken(InputFieldTokenType.EOF, 'eof');
108+
} else {
109+
if (this.currentToken && this.currentToken.type === InputFieldTokenType.WORD) {
110+
this.currentToken.literal += this.ch;
111+
this.currentToken.range.to = this.position;
112+
} else {
113+
token = this.createToken(InputFieldTokenType.WORD);
114+
}
115+
}
116+
}
117+
118+
if (token) {
119+
this.currentToken = token;
120+
this.tokens.push(token);
121+
}
122+
123+
this.readChar();
124+
}
125+
126+
private readChar(): void {
127+
this.ch = this.peek();
128+
129+
this.position = this.readPosition;
130+
this.readPosition += 1;
131+
}
132+
133+
private peek(): string {
134+
if (this.readPosition >= this.input.length) {
135+
return '\0';
136+
} else {
137+
return this.input[this.readPosition];
138+
}
139+
}
140+
141+
private createToken(type: InputFieldTokenType, char?: string): InputFieldToken {
142+
return createToken(type, char !== undefined ? char : this.ch, this.position, this.position);
143+
}
144+
}

‎src/parsers/newInputFieldParser/ParsingError.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { ErrorLevel, ErrorType, MetaBindError } from '../../utils/errors/MetaBindErrors';
2+
import { AbstractToken } from './ParsingUtils';
3+
4+
export class ParsingError extends MetaBindError {
5+
str: string;
6+
token: AbstractToken<string> | undefined;
7+
source: string;
8+
9+
constructor(
10+
errorLevel: ErrorLevel,
11+
effect: string,
12+
cause: string | Error,
13+
context: Record<string, any>,
14+
str: string,
15+
token: AbstractToken<string> | undefined,
16+
source: string
17+
) {
18+
super(errorLevel, effect, cause, context);
19+
20+
this.str = str;
21+
this.token = token;
22+
this.source = source;
23+
24+
this.updateMessage2();
25+
}
26+
27+
public getErrorType(): ErrorType {
28+
return ErrorType.PARSING;
29+
}
30+
31+
protected updateMessage2(): void {
32+
if (this.cause instanceof Error) {
33+
this.message = `[${this.getErrorType()}] "${this.effect}" caused by error "${this.cause.message}"\n`;
34+
} else {
35+
this.message = `[${this.getErrorType()}] "${this.effect}" caused by "${this.cause}"\n`;
36+
}
37+
38+
if (this.token) {
39+
this.message += this.str + '\n';
40+
this.message += ' '.repeat(this.token.range.from) + '^'.repeat(this.token.range.to - this.token.range.from + 1) + '\n';
41+
}
42+
}
43+
}

0 commit comments

Comments
 (0)