Skip to content

Commit 388d568

Browse files
MyNameIsTrezMyNameIsTrez
committed
Transition the parser to a better CST format
1 parent 7ca5f5d commit 388d568

File tree

6 files changed

+81
-88
lines changed

6 files changed

+81
-88
lines changed

‎Python/ini_converting/ini_cst_builder.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def get_ini_cst(input_folder_path, subfolder_path):
2020
continue
2121
elif p.is_file() and p.suffix == ".ini" and p.stem != "desktop": # Skip the desktop.ini Windows metadata file.
2222
tokens = ini_tokenizer.get_tokens(p.read_text())
23-
parsed_portion[name], _ = ini_parser.get_parsed_tokens(tokens, []) # _ is to get rid of the returned i, cause otherwise it'd return a tuple.
23+
parsed_portion[name] = ini_parser.get_parsed_tokens(tokens, [], [0])
2424
elif p.is_dir():
2525
parsed_portion[name] = get_ini_cst(input_folder_path, str(p))
2626

‎Python/ini_converting/ini_parser.py

Lines changed: 16 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
def get_parsed_tokens(tokens, parsed, i=0, depth=-1):
1+
def get_parsed_tokens(tokens, parsed, token_idx, depth=0):
22
"""
33
start -> tabs -> property -> equals -> value -> newline
44
^ v ^ v
@@ -7,48 +7,39 @@ def get_parsed_tokens(tokens, parsed, i=0, depth=-1):
77
"""
88
state = "start"
99

10-
while i < len(tokens):
11-
token = tokens[i]
10+
while token_idx[0] < len(tokens):
11+
token = tokens[token_idx[0]]
1212

13-
if depth == -1:
14-
parsed.append( { "type": "lines_tokens", "content": [] } )
15-
_, i = get_parsed_tokens(tokens, parsed[-1]["content"], i, depth + 1)
16-
elif state == "start" and token["type"] == "TABS" and is_less_deep(depth, token):
17-
return parsed, i
13+
if state == "start" and token["type"] == "TABS" and is_less_deep(depth, token):
14+
return
1815
elif state == "start" and token["type"] == "TABS" and is_deeper(depth, token):
1916
parsed.append( { "type": "lines_tokens", "content": [] } )
20-
return get_parsed_tokens(tokens, parsed[-1]["content"], i, depth + 1)
17+
get_parsed_tokens(tokens, parsed[-1]["content"], token_idx, depth + 1)
2118
elif state == "start" and token["type"] == "TABS":
22-
parsed.append( { "type": "extra", "content": token["content"] } )
23-
state = "tabs"
24-
i += 1
25-
elif state == "tabs" and token["type"] == "WORD":
26-
parsed.append( { "type": "property", "content": token["content"] } )
27-
state = "property"
28-
i += 1
29-
elif state == "start" and token["type"] == "WORD" and depth == 0:
19+
parsed.append( [ { "type": "lines_tokens", "content": [] } ] )
20+
token_idx[0] += 1
21+
get_parsed_tokens(tokens, parsed[-1], token_idx, depth)
22+
elif (state == "start" or state == "tabs") and token["type"] == "WORD":
3023
parsed.append( { "type": "property", "content": token["content"] } )
3124
state = "property"
32-
i += 1
33-
elif state == "start" and token["type"] == "WORD":
34-
return parsed, i
25+
token_idx[0] += 1
3526
elif state == "property" and token["type"] == "EQUALS":
3627
parsed.append( { "type": "extra", "content": token["content"] } )
3728
state = "equals"
38-
i += 1
29+
token_idx[0] += 1
3930
elif state == "equals" and token["type"] == "WORD":
4031
parsed.append( { "type": "value", "content": token["content"] } )
4132
state = "value"
42-
i += 1
33+
token_idx[0] += 1
4334
elif state == "value" and token["type"] == "NEWLINES":
4435
parsed.append( { "type": "extra", "content": token["content"] } )
4536
state = "start"
46-
i += 1
37+
token_idx[0] += 1
4738
else:
4839
parsed.append( { "type": "extra", "content": token["content"] } )
49-
i += 1
40+
token_idx[0] += 1
5041

51-
return parsed, i
42+
return parsed
5243

5344

5445
def is_less_deep(depth, token):

‎Python/ini_converting/ini_parser_tests.py

Lines changed: 58 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -4,51 +4,71 @@
44

55

66
def parser_tests():
7-
test("simple", [
8-
{ "type": "lines_tokens", "content": [
9-
{ "type": "property", "content": "AddEffect" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "MOPixel" }
10-
]}
11-
])
12-
test("comments", [
13-
{ "type": "lines_tokens", "content": [
14-
{ "type": "extra", "content": "\n" },
15-
{ "type": "extra", "content": "// foo"}, { "type": "extra", "content": "\n" },
16-
{ "type": "extra", "content": "/*a\nb\nc*/" }, { "type": "extra", "content": "\n" },
17-
]},
18-
])
19-
test("multiple", [
20-
{ "type": "lines_tokens", "content": [
7+
# test("simple", [
8+
# [
9+
# { "type": "property", "content": "AddEffect" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "MOPixel" }
10+
# ]
11+
# ])
12+
# test("comments", [
13+
# [
14+
# { "type": "extra", "content": "\n" },
15+
# { "type": "extra", "content": "// foo"}, { "type": "extra", "content": "\n" },
16+
# { "type": "extra", "content": "/*a\nb\nc*/" }, { "type": "extra", "content": "\n" },
17+
# ],
18+
# ])
19+
test("nested", [
20+
[
2121
{ "type": "property", "content": "Foo" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "Bar" }, { "type": "extra", "content": "\n" },
2222
{ "type": "lines_tokens", "content": [
23-
{ "type": "extra", "content": "\t" }, { "type": "property", "content": "Baz" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "Bee" }, { "type": "extra", "content": "\n" }
24-
]}
25-
]},
26-
{ "type": "lines_tokens", "content": [
27-
{ "type": "property", "content": "A" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "B" }, { "type": "extra", "content": "\n" },
28-
{ "type": "lines_tokens", "content": [
29-
{ "type": "extra", "content": "\t" }, { "type": "property", "content": "C" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "D" }, { "type": "extra", "content": "\n" }
30-
]}
31-
]}
32-
])
33-
test("complex", [
34-
{ "type": "lines_tokens", "content": [
35-
{ "type": "extra", "content": "\n" },
36-
{ "type": "extra", "content": "// foo"}, { "type": "extra", "content": "\n" },
37-
{ "type": "extra", "content": "/*a\nb\nc*/" }, { "type": "extra", "content": "\n" },
38-
{ "type": "property", "content": "AddEffect" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "MOPixel" }, { "type": "extra", "content": "//bar" }, { "type": "extra", "content": "\n" },
39-
{ "type": "lines_tokens", "content": [
40-
{ "type": "extra", "content": "\t" }, { "type": "property", "content": "PresetName" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "red_dot_tiny" }, { "type": "extra", "content": "\n" },
41-
{ "type": "lines_tokens", "content": [
42-
{ "type": "extra", "content": "\t\t" }, { "type": "property", "content": "Mass" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "0.0" }, { "type": "extra", "content": "\n" },
43-
{ "type": "extra", "content": "\t\t" }, { "type": "property", "content": "Xd" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "42" }
44-
]}
23+
[
24+
{ "type": "extra", "content": "\t" }, { "type": "property", "content": "Baz" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "Bee" }, { "type": "extra", "content": "\n" }
25+
]
4526
]}
46-
]},
27+
]
4728
])
29+
# test("multiple", [
30+
# [
31+
# { "type": "property", "content": "Foo" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "Bar" }, { "type": "extra", "content": "\n" },
32+
# { "type": "lines_tokens", "content": [
33+
# [
34+
# { "type": "extra", "content": "\t" }, { "type": "property", "content": "Baz" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "Bee" }, { "type": "extra", "content": "\n" }
35+
# ]
36+
# ]}
37+
# ],
38+
# [
39+
# { "type": "property", "content": "A" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "B" }, { "type": "extra", "content": "\n" },
40+
# { "type": "lines_tokens", "content": [
41+
# [
42+
# { "type": "extra", "content": "\t" }, { "type": "property", "content": "C" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "D" }, { "type": "extra", "content": "\n" }
43+
# ]
44+
# ]}
45+
# ]
46+
# ])
47+
# test("complex", [
48+
# [
49+
# { "type": "extra", "content": "\n" },
50+
# { "type": "extra", "content": "// foo"}, { "type": "extra", "content": "\n" },
51+
# { "type": "extra", "content": "/*a\nb\nc*/" }, { "type": "extra", "content": "\n" },
52+
# { "type": "property", "content": "AddEffect" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "MOPixel" }, { "type": "extra", "content": "//bar" }, { "type": "extra", "content": "\n" },
53+
# { "type": "lines_tokens", "content": [
54+
# [
55+
# { "type": "extra", "content": "\t" }, { "type": "property", "content": "PresetName" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "red_dot_tiny" }, { "type": "extra", "content": "\n" },
56+
# { "type": "lines_tokens", "content": [
57+
# [
58+
# { "type": "extra", "content": "\t\t" }, { "type": "property", "content": "Mass" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "0.0" }, { "type": "extra", "content": "\n" },
59+
# ],
60+
# [
61+
# { "type": "extra", "content": "\t\t" }, { "type": "property", "content": "Xd" }, { "type": "extra", "content": " " }, { "type": "extra", "content": "=" }, { "type": "extra", "content": " " }, { "type": "value", "content": "42" }
62+
# ]
63+
# ]}
64+
# ]
65+
# ]}
66+
# ]
67+
# ])
4868

4969

5070
def test(filename, expected):
5171
text = tests.read_test(filename)
5272
tokens = ini_tokenizer.get_tokens(text)
53-
ini_cst, _ = ini_parser.get_parsed_tokens(tokens, [])
73+
ini_cst = ini_parser.get_parsed_tokens(tokens, [], [0])
5474
tests.test(text, ini_cst, expected)

‎Python/ini_converting/ini_rules.py

Lines changed: 2 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,11 @@
44

55

66
"""
7-
The structure of the "parsed" variable that's passed to the apply_rules_on_ini_cst function looks like this,
8-
where the "v" character is used to point down:
9-
107
{
118
"foo.rte": {
9+
"Foo.ini": ini_cst,
1210
"FolderName": {
13-
"Bar.ini": [ < holds sections of a file
14-
section and also line_tokens
15-
v token
16-
[ v
17-
{ "type": "extra", "content": "// foo" }
18-
],
19-
20-
section and also line_tokens
21-
v token
22-
[ v
23-
{ "type": "lines_tokens", "content": [
24-
line_tokens
25-
v token
26-
[ v
27-
{ "type": "property", "content": "PresetName" },
28-
{ "type": "value", "content": "foo" }
29-
]
30-
]}
31-
]
32-
]
11+
"Bar.ini": ini_cst,
3312
}
3413
}
3514
}

‎Python/ini_converting/ini_test_files/nested.ini

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Foo = Bar
2+
Baz = Bee

‎TODO.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
* Make sure that built exes don't run the tests every time they convert something.
2-
* Rename "lines_tokens" to "children"
2+
* Rename "lines_tokens" to "children"
3+
* Check if commenting out "ini_rules.apply_rules_on_ini_cst(ini_cst)" in convert.py and some other lines results in outputted mod INI files being identical.

0 commit comments

Comments
 (0)