ProLang/Lexer.py at master · Fritsch-Tech/ProLang · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
from Token import Token
from Constants import Token_type

def lex(content: list):
    tokens = []
    tokens = []
    for line_index,line in enumerate(content):
        tokens_line = []
        token = ''
        string_open = False
        for c in line:
            if c == '"' or c == "'":
                if string_open:
                    string_open = False
                    tokens_line.append(Token(Token_type.STRING,token))
                    token = ''
                else:
                    string_open = True
            elif c == '(':
                __add_previous(token,tokens_line)
                token = ''
                tokens_line.append(Token(Token_type.PAR_OPEN,None))
            elif c == ')':
                __add_previous(token,tokens_line)
                token = ''
                tokens_line.append(Token(Token_type.PAR_CLOSE,None))
            elif c == '.':
                __add_previous(token,tokens_line)
                token = ''
                tokens_line.append(Token(Token_type.PARENT,None))
            elif c == '+':
                __add_previous(token,tokens_line)
                token = ''
                tokens_line.append(Token(Token_type.ADD,None))
            elif c == '-':
                __add_previous(token,tokens_line)
                token = ''
                tokens_line.append(Token(Token_type.SUB,None))
            elif c == '*':
                __add_previous(token,tokens_line)
                token = ''
                tokens_line.append(Token(Token_type.MUL,None))
            elif c == '/':
                __add_previous(token,tokens_line)
                token = ''
                tokens_line.append(Token(Token_type.DIV,None))
            elif c == '=':
                __add_previous(token,tokens_line)
                token = ''
                if tokens_line[-1].token_type == Token_type.ASSIGN:
                    tokens_line[-1] = Token(Token_type.COMPARE,None)
                else:
                    tokens_line.append(Token(Token_type.ASSIGN,None))
            elif c == ' ':
                __add_previous(token,tokens_line)
                token = ''
            else:
                token += c

        if string_open:
            raise Exception('Open string in Line {line}'.format(line=line_index))
        if token != '':
            __add_previous(token,tokens_line)
            token = ''
        if tokens_line != []:
            tokens.append(tokens_line)
    tokens.append([Token(Token_type.EOF,None)])
    print('TOKENS:')
    for line in tokens:
        print(line)
    print('')
    print('='*100)
    print('')
    return(tokens)

def __add_previous(token: str,tokens_line: list):
    token.replace(' ','')
    if token.isdigit():
        tokens_line.append(Token(Token_type.INTEGER,int(token)))
    elif token != '':
        if token == 'ADD':
            tokens_line.append(Token(Token_type.ADD,None))
        elif token == 'SUB':
            tokens_line.append(Token(Token_type.SUB,None))
        elif token == 'MUL':
            tokens_line.append(Token(Token_type.MUL,None))
        elif token == 'DIV':
            tokens_line.append(Token(Token_type.DIV,None))
        elif token == 'IS':
            tokens_line.append(Token(Token_type.ASSIGN,None))
        elif token == 'SUB':
            tokens_line.append(Token(Token_type.SUB,None))
        else:
            tokens_line.append(Token(Token_type.COMMAND,token))
    token = ''