Skip to content

Commit 131a97c

Browse files
authored
Merge pull request #1105 from PokeAPI/veekun-scripts
Bring over scripts used when we moved away from Veekun
2 parents 3f27daa + cc439d9 commit 131a97c

25 files changed

+17356
-1
lines changed

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ format: # Format the source code
8585
black .
8686

8787
format-check: # Check the source code has been formatted
88-
black . --check
88+
black . --check --exclude 'Resources/scripts/.*'
8989

9090
pull:
9191
git checkout master
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import csv
2+
import os
3+
from read_swsh import TextFile
4+
5+
# data_path contains the countents of the `message` folder found in sword/shield's romfs:/bin/
6+
if __name__ == "__main__":
7+
path = os.path.abspath(os.path.dirname(__file__))
8+
data_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "..", "data")
9+
csv_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pokedex", "data", "csv")
10+
11+
languages = {
12+
"JPN": 1,
13+
"Korean": 3,
14+
"Trad_Chinese": 4,
15+
"French": 5,
16+
"German": 6,
17+
"Spanish": 7,
18+
"Italian": 8,
19+
"English": 9,
20+
"JPN_KANJI": 11,
21+
"Simp_Chinese": 12,
22+
}
23+
24+
header = ["ability_id", "local_language_id", "name"]
25+
entries = []
26+
27+
# conquest abilities
28+
with open(os.path.join(csv_path, "ability_names.csv"), "r", encoding="utf-8", newline="") as csv_file:
29+
reader = csv.reader(csv_file, delimiter=",")
30+
for row in reader:
31+
if row[0].isnumeric() and int(row[0]) > 10000:
32+
entries.append([int(row[0]), int(row[1]), row[2]])
33+
34+
with open(os.path.join(csv_path, "ability_names.csv"), "w", encoding="utf-8", newline="") as csv_file:
35+
writer = csv.writer(csv_file, delimiter=",", lineterminator="\n")
36+
for language_dir, language_id in languages.items():
37+
try:
38+
# Parse through the .dat and .tbl
39+
textFile = TextFile(
40+
os.path.join(data_path, language_dir, "common", "tokusei.dat"),
41+
os.path.join(data_path, language_dir, "common", "tokusei.tbl"),
42+
)
43+
dictionary = textFile.GetDict()
44+
except UserWarning as error:
45+
print(error)
46+
47+
try:
48+
if len(dictionary) == 0:
49+
raise UserWarning("Error: the files returned no data")
50+
51+
# Loop through the text file's dictionary and append the parsed data into the list
52+
for label, text in dictionary.items():
53+
id = int(label[0].split("_")[1])
54+
if id == 0:
55+
continue
56+
entries.append([id, language_id, text])
57+
58+
except UserWarning as error:
59+
print(error)
60+
61+
# Sort the list based on species id
62+
writer.writerow(header)
63+
entries.sort()
64+
writer.writerows(entries)
65+
print("Done")
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import csv
2+
import os
3+
from read_swsh import TextFile
4+
5+
# data_path contains the countents of the `message` folder found in sword/shield's romfs:/bin/
6+
if __name__ == "__main__":
7+
path = os.path.abspath(os.path.dirname(__file__))
8+
data_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "..", "data")
9+
csv_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pokedex", "data", "csv")
10+
11+
languages = {
12+
"JPN": 1,
13+
"Korean": 3,
14+
"Trad_Chinese": 4,
15+
"French": 5,
16+
"German": 6,
17+
"Spanish": 7,
18+
"Italian": 8,
19+
"English": 9,
20+
"JPN_KANJI": 11,
21+
"Simp_Chinese": 12,
22+
}
23+
24+
header = ["move_id", "local_language_id", "name"]
25+
entries = []
26+
27+
# shadow moves
28+
with open(os.path.join(csv_path, "move_names.csv"), "r", encoding="utf-8", newline="") as csv_file:
29+
reader = csv.reader(csv_file, delimiter=",")
30+
for row in reader:
31+
if row[0].isnumeric() and int(row[0]) > 10000:
32+
entries.append([int(row[0]), int(row[1]), row[2]])
33+
34+
with open(os.path.join(csv_path, "move_names.csv"), "w", encoding="utf-8", newline="") as csv_file:
35+
writer = csv.writer(csv_file, delimiter=",", lineterminator="\n")
36+
for language_dir, language_id in languages.items():
37+
try:
38+
# Parse through the .dat and .tbl
39+
textFile = TextFile(
40+
os.path.join(data_path, language_dir, "common", "wazaname.dat"),
41+
os.path.join(data_path, language_dir, "common", "wazaname.tbl"),
42+
)
43+
dictionary = textFile.GetDict()
44+
except UserWarning as error:
45+
print(error)
46+
47+
try:
48+
if len(dictionary) == 0:
49+
raise UserWarning("Error: the files returned no data")
50+
51+
# Loop through the text file's dictionary and append the parsed data into the list
52+
for label, text in dictionary.items():
53+
id = int(label[0].split("_")[1])
54+
if id == 0:
55+
continue
56+
entries.append([id, language_id, text.strip("\r")])
57+
58+
except UserWarning as error:
59+
print(error)
60+
61+
# Sort the list based on species id
62+
writer.writerow(header)
63+
entries.sort()
64+
writer.writerows(entries)
65+
print("Done")
Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
import copy
2+
import os
3+
import struct
4+
import sys
5+
6+
class TextLine():
7+
offset = None
8+
length = None
9+
10+
class TextFile():
11+
def __init__(this, path1, path2):
12+
this.__dictEntries = {}
13+
this.__magic = 0x42544841
14+
this.__KEY_BASE = 0x7C89
15+
this.__KEY_ADVANCE = 0x2983
16+
this.__KEY_VARIABLE = 0x0010
17+
this.__KEY_TERMINATOR = 0x0000
18+
19+
# Load dex labels
20+
if (os.path.splitext(path1)[1] == ".tbl"):
21+
this.__OpenTbl(path1)
22+
elif (os.path.splitext(path2)[1] == ".tbl"):
23+
this.__OpenTbl(path2)
24+
else:
25+
raise UserWarning("Error: a .tbl was not given")
26+
27+
# Load dex entries
28+
if (os.path.splitext(path1)[1] == ".dat"):
29+
this.__OpenDat(path1)
30+
elif (os.path.splitext(path2)[1] == ".dat"):
31+
this.__OpenDat(path2)
32+
else:
33+
raise UserWarning("Error: a .dat was not given")
34+
35+
# The table has 1 more entry than the dat to show when the table ends
36+
if (len(this.__labels) == len(this.__lines) + 1):
37+
for i in range(0, len(this.__lines)):
38+
this.__dictEntries[this.__labels[i]] = this.__lines[i]
39+
40+
@property
41+
def __TextSections(this):
42+
"""(2 bytes) Gets the number of text sections"""
43+
return struct.unpack_from("<H", this.__data, 0x0)[0]
44+
45+
@property
46+
def __LineCount(this):
47+
"""(2 bytes) Gets the amount of lines"""
48+
return struct.unpack_from("<H", this.__data, 0x2)[0]
49+
50+
@property
51+
def __TotalLength(this):
52+
"""(4 bytes) Gets the total character length of all text sections"""
53+
return struct.unpack_from("<I", this.__data, 0x4)[0]
54+
55+
@property
56+
def __InitialKey(this):
57+
"""(4 bytes) Gets the initial key; should be 0x00000000"""
58+
return struct.unpack_from("<I", this.__data, 0x8)[0]
59+
60+
@property
61+
def __SectionDataOffset(this):
62+
"""(4 bytes) Gets the offset where the data section begins"""
63+
return struct.unpack_from("<I", this.__data, 0xC)[0]
64+
65+
@property
66+
def __SectionLength(this):
67+
"""(4 bytes) Gets the length of characters the given section is"""
68+
offset = this.__SectionDataOffset
69+
return struct.unpack_from("<I", this.__data, offset)[0]
70+
71+
@property
72+
def __LineOffsets(this):
73+
"""Figures out the offset for each entry based on the data section offset"""
74+
result = [None] * this.__LineCount
75+
sdo = int(this.__SectionDataOffset)
76+
for i in range(0, len(result)):
77+
result[i] = TextLine()
78+
result[i].offset = struct.unpack_from("<i", this.__data, (i * 8) + sdo + 4)[0] + sdo
79+
result[i].length = struct.unpack_from("<h", this.__data, (i * 8) + sdo + 8)[0]
80+
81+
return result
82+
83+
def GetLabels(this):
84+
return this.__labels
85+
86+
def GetDict(this):
87+
return this.__dictEntries
88+
89+
def HashFNV1_64(this, word):
90+
"""Fowler-Noll-Vo hash function; 64-bit"""
91+
fnvPrime_64 = 0x100000001b3
92+
offsetBasis_64 = 0xCBF29CE484222645
93+
94+
hash = offsetBasis_64
95+
for c in word:
96+
hash = hash ^ ord(c)
97+
# Cast hash to at 64-bit value
98+
hash = (hash * fnvPrime_64) % 2**64
99+
100+
return hash
101+
102+
def __LineData(this, data):
103+
"""Loads the file into a list to later decrypt"""
104+
key = copy.copy(this.__KEY_BASE)
105+
result = [None] * this.__LineCount
106+
lines = this.__LineOffsets
107+
108+
for i in range(0, len(lines)):
109+
# Make a list twice the size of the current text line size
110+
encrypted = lines[i].length * 2
111+
# Then copy the encrypted line starting from the given offset for however long the given list is
112+
end = lines[i].offset + encrypted
113+
encrypted = this.__data[lines[i].offset:end]
114+
115+
result[i] = this.__CryptLineData(encrypted, key)
116+
# Cast key to a 16-bits (otherwise things break)
117+
key = (key + this.__KEY_ADVANCE) % 2**16
118+
119+
return result
120+
121+
def __CryptLineData(this, data, key):
122+
"""Decrypts the given line into a list of bytes"""
123+
copied = copy.copy(data)
124+
result = [None] * len(copied)
125+
126+
for i in range(0, len(copied), 2):
127+
result[i] = copied[i] ^ (key % 256)
128+
result[i + 1] = copied[i + 1] ^ ((key >> 8) % 256)
129+
# Bit-shift and OR key, then cast to 16-bits (otherwise things break)
130+
key = (key << 3 | key >> 13) % 2**16
131+
132+
return result
133+
134+
def __GetLineString(this, data):
135+
"""Turns the given list of bytes into a finished string"""
136+
if (data is None):
137+
return None
138+
139+
string = ""
140+
i = 0
141+
while (i < len(data)):
142+
# Cast 2 bytes to figure out what to do next
143+
value = struct.unpack_from("<H", data, i)[0]
144+
if (value == this.__KEY_TERMINATOR):
145+
break;
146+
i += 2
147+
148+
if (value == this.__KEY_TERMINATOR):
149+
return string
150+
elif (value == this.__KEY_VARIABLE):
151+
string += "[VAR]"
152+
elif (value == "\n"):
153+
string += "\n"
154+
elif (value == "\\"):
155+
string += "\\"
156+
elif (value == "["):
157+
string += "\["
158+
else:
159+
string += chr(value)
160+
161+
return string
162+
163+
def __MakeLabelHash(this, f):
164+
"""Returns the label name and a FNV1_64 hash"""
165+
# Next 8 bytes is the hash of the label name
166+
hash = struct.unpack("<Q", f.read(8))[0]
167+
# Next 2 bytes is the label"s name length
168+
nameLength = struct.unpack("<H", f.read(2))[0]
169+
# Read the bytes until 0x0 is found
170+
name = this.__ReadUntil(f, 0x0)
171+
172+
if (this.HashFNV1_64(name) == hash):
173+
return name, hash
174+
175+
def __OpenDat(this, path):
176+
with open(path, "rb") as file:
177+
try:
178+
this.__data = file.read()
179+
except:
180+
raise UserWarning("Error: Could not open .dat")
181+
182+
# Decrypt the text
183+
cryptedText = this.__LineData(this.__data)
184+
this.__lines = []
185+
for line in cryptedText:
186+
this.__lines.append(this.__GetLineString(bytearray(line)))
187+
188+
def __OpenTbl(this, path):
189+
with open(path, "rb") as f:
190+
try:
191+
# First four bytes is "magic"; needs to be 0x42544841
192+
testMagic = struct.unpack("<I", f.read(4))[0]
193+
if (testMagic == this.__magic):
194+
# Next four bytes is the number of entries to read
195+
count = struct.unpack("<I", f.read(4))[0]
196+
this.__labels = []
197+
# Iterate through the entries
198+
for i in range(0, count):
199+
this.__labels.append(this.__MakeLabelHash(f))
200+
201+
except struct.error:
202+
raise UserWarning("Error: Coult not open .tbl")
203+
204+
def __ReadUntil(this, f, value):
205+
"""Reads the given file until it reaches the given value"""
206+
string = ""
207+
c = f.read(1)
208+
end = bytes([value])
209+
while (c != end):
210+
# Read one byte at a time to get each character
211+
string += c.decode("utf-8")
212+
c = f.read(1)
213+
214+
return string

0 commit comments

Comments
 (0)