Skip to content

Commit 92ac850

Browse files
LogonzLogonz
authored
Merge pull request #38 from Questie/dev
Move big changes into Main
2 parents 46ac8ab + 0b845e8 commit 92ac850

File tree

93 files changed

+2175072
-793595
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+2175072
-793595
lines changed

‎.database_generator/.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
_data
2+
Questie-translations
3+
*.json
4+
*.lua-table
5+
*.txt

‎.database_generator/Dockerfile‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# FROM nickblah/lua:5.1-luarocks
2+
FROM nickblah/lua:5.1-luarocks-ubuntu
3+
4+
RUN apt-get update && apt-get install -y software-properties-common gnupg gcc
5+
# Get the latest version of Git
6+
# RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E363C90F8F1B6217 A1715D88E1DF1F24
7+
RUN add-apt-repository ppa:git-core/ppa -y
8+
RUN apt-get update && apt-get install -y git
9+
10+
# Install OpenSSL for LuaSec
11+
RUN apt-get install -y libssl-dev
12+
13+
RUN luarocks install bit32
14+
RUN luarocks install argparse
15+
RUN luarocks install luafilesystem
16+
RUN luarocks install luasocket
17+
RUN luarocks install luasec
18+
19+
RUN apt-get update && apt-get install -y wget

‎.database_generator/createStatic.lua‎

Lines changed: 356 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,356 @@
1+
local script_path = debug.getinfo(1, "S").source:sub(2)
2+
local script_dir = script_path:match("(.*/)")
3+
package.path = script_dir .. "?.lua;" .. package.path
4+
5+
-- Allow accessing private fields
6+
---@diagnostic disable: invisible
7+
require("cli.dump")
8+
local helpers = require(".db_helpers")
9+
10+
local l10n_loader = require("load_translations_l10n")
11+
12+
require(".dump")
13+
14+
require("cli.Addon_Meta")
15+
require("cli.CLI_Helpers")
16+
local lfs = require("lfs")
17+
18+
assert(Is_CLI, "This function should only be called from the CLI environment")
19+
20+
local f = string.format
21+
22+
Is_Create_Static = true
23+
24+
--- Takes the raw database files (_data/*DB.lua) and the static corrections,
25+
--- merges them, and outputs the final combined data into both .lua-table format
26+
--- (for potential debugging/diffing) and the SimpleHTML format used by the addon.
27+
---@param questiedb_version string # The WoW version identifier (e.g., "classic", "wrath", "cata"). Case-insensitive.
28+
---@param questie_version string # The Questie version identifier (e.g., "Classic", "TBC", "Wotlk"). Case-insensitive.
29+
---@param debug boolean? # Optional debug flag
30+
function DumpDatabase(questiedb_version, questie_version, debug)
31+
local lowerVersion = questiedb_version:lower()
32+
local capitalizedVersion = lowerVersion:gsub("^%l", string.upper)
33+
print(f("\n\27[36mCompiling %s database...\27[0m", capitalizedVersion))
34+
35+
-- Reset data objects, load the core addon files for the specified version, and set WoW version globals.
36+
---@type LibQuestieDB
37+
LibQuestieDBTable = AddonInitializeVersion(capitalizedVersion)
38+
39+
-- Drain all the timers
40+
print("Pre-Drain timer list")
41+
C_Timer.drainTimerList()
42+
43+
-- print("Executing event: ADDON_LOADED")
44+
-- LibQuestieDBTable.RegisteredEvents["ADDON_LOADED"](CLI_addonName or "QuestieDB")
45+
46+
-- -- Drain all the timers
47+
-- print("ADDON_LOADED timer list")
48+
-- C_Timer.drainTimerList()
49+
50+
-- if not LibQuestieDBTable.Database.Initialized then
51+
-- error("Database not initialized")
52+
-- end
53+
54+
-- Initialize tables to hold the merged data (raw data + static corrections).
55+
---@type table<ItemId, table<number, any>>
56+
local itemOverride = {}
57+
---@type table<NpcId, table<number, any>>
58+
local npcOverride = {}
59+
---@type table<ObjectId, table<number, any>>
60+
local objectOverride = {}
61+
---@type table<QuestId, table<number, any>>
62+
local questOverride = {}
63+
---@type table<ItemId|NpcId|ObjectId|QuestId, table<L10nDBKeys, table<L10nLocales, any>>>
64+
local l10nOverride = {}
65+
66+
---@type Corrections
67+
local Corrections = LibQuestieDBTable.Corrections
68+
69+
-- Run self-tests on the dump functions to ensure they produce correct output.
70+
Corrections.DumpFunctions.testDumpFunctions()
71+
72+
-- Define the entity types for which we will generate HTML files.
73+
local entityTypes = { "Item", "Npc", "Object", "Quest", }
74+
75+
-- Process Item Data: Load raw DB, load static corrections, merge corrections into raw data.
76+
do
77+
-- Load the raw ItemDB.lua file content as a string.
78+
CLI_Helpers.loadFile(f("%s/_data/%sItemDB.lua", helpers.get_script_dir(), lowerVersion))
79+
-- Execute the string to get the raw item data table.
80+
---@type table<ItemId, table<number, any>>
81+
itemOverride = loadstring(QuestieDB.itemData)() -- QuestieDB.itemData is loaded by loadFile
82+
-- Load static corrections registered within the addon environment.
83+
LibQuestieDBTable.Item.LoadOverrideData(false, true) -- includeStatic = true
84+
---@type ItemMeta
85+
local itemMeta = Corrections.ItemMeta
86+
-- Iterate through the loaded static corrections.
87+
---@param itemId ItemId
88+
---@param corrections table<string, any> @ Map of field name -> corrected value
89+
for itemId, corrections in pairs(LibQuestieDBTable.Item.override) do
90+
-- Ensure an entry exists for this ID in the main override table.
91+
if not itemOverride[itemId] then
92+
itemOverride[itemId] = {}
93+
end
94+
-- Merge each correction, converting field name to numeric index.
95+
---@param key string @ Field name (e.g., "name", "requiredLevel")
96+
---@param correction any @ The corrected value
97+
for key, correction in pairs(corrections) do
98+
---@type number
99+
local correctionIndex = itemMeta.itemKeys[key]
100+
itemOverride[itemId][correctionIndex] = correction
101+
end
102+
end
103+
end
104+
105+
-- Process NPC Data: Load raw DB, load static corrections, merge corrections into raw data.
106+
do
107+
CLI_Helpers.loadFile(f("%s/_data/%sNpcDB.lua", helpers.get_script_dir(), lowerVersion))
108+
109+
---@type table<NpcId, table<number, any>>
110+
npcOverride = loadstring(QuestieDB.npcData)()
111+
LibQuestieDBTable.Npc.LoadOverrideData(false, true)
112+
113+
---@type NpcMeta
114+
local npcMeta = Corrections.NpcMeta
115+
116+
---@param npcId NpcId
117+
---@param corrections table<string, any>
118+
for npcId, corrections in pairs(LibQuestieDBTable.Npc.override) do
119+
if not npcOverride[npcId] then
120+
npcOverride[npcId] = {}
121+
end
122+
123+
---@param key string
124+
---@param correction any
125+
for key, correction in pairs(corrections) do
126+
---@type number
127+
local correctionIndex = npcMeta.npcKeys[key]
128+
npcOverride[npcId][correctionIndex] = correction
129+
end
130+
end
131+
end
132+
133+
-- Process Object Data: Load raw DB, load static corrections, merge corrections into raw data.
134+
do
135+
CLI_Helpers.loadFile(f("%s/_data/%sObjectDB.lua", helpers.get_script_dir(), lowerVersion))
136+
137+
---@type table<ObjectId, table<number, any>>
138+
objectOverride = loadstring(QuestieDB.objectData)()
139+
LibQuestieDBTable.Object.LoadOverrideData(false, true)
140+
141+
---@type ObjectMeta
142+
local objectMeta = Corrections.ObjectMeta
143+
144+
---@param objectId ObjectId
145+
---@param corrections table<string, any>
146+
for objectId, corrections in pairs(LibQuestieDBTable.Object.override) do
147+
if not objectOverride[objectId] then
148+
objectOverride[objectId] = {}
149+
end
150+
151+
---@param key string
152+
---@param correction any
153+
for key, correction in pairs(corrections) do
154+
---@type number
155+
local correctionIndex = objectMeta.objectKeys[key]
156+
objectOverride[objectId][correctionIndex] = correction
157+
end
158+
end
159+
end
160+
161+
-- Process Quest Data: Load raw DB, load static corrections, merge corrections into raw data.
162+
do
163+
CLI_Helpers.loadFile(f("%s/_data/%sQuestDB.lua", helpers.get_script_dir(), lowerVersion))
164+
165+
---@type table<QuestId, table<number, any>>
166+
questOverride = loadstring(QuestieDB.questData)()
167+
LibQuestieDBTable.Quest.LoadOverrideData(false, true)
168+
169+
---@type QuestMeta
170+
local questMeta = Corrections.QuestMeta
171+
172+
---@param questId QuestId
173+
---@param corrections table<string, any>
174+
for questId, corrections in pairs(LibQuestieDBTable.Quest.override) do
175+
if not questOverride[questId] then
176+
questOverride[questId] = {}
177+
end
178+
179+
---@param key string
180+
---@param correction any
181+
for key, correction in pairs(corrections) do
182+
---@type number
183+
local correctionIndex = questMeta.questKeys[key]
184+
questOverride[questId][correctionIndex] = correction
185+
end
186+
end
187+
end
188+
189+
-- Process L10n Data: Load raw DB, load static corrections, merge corrections into raw data.
190+
local output
191+
do
192+
-- ? l10n dump
193+
print("Loading version: " .. questie_version)
194+
for datatype in pairs(Corrections.L10nMeta.l10nKeys) do
195+
l10n_loader.CleanFiles(questie_version, datatype)
196+
end
197+
198+
-- Create the lookup tables for the translations (This will import a singular ImportModule table)
199+
-- The "l10n" is not required here.
200+
---@see AddonInitializeVersion
201+
local l10n = QuestieLoader:ImportModule()
202+
for _, entityType in ipairs(entityTypes) do
203+
local newLookup = entityType:lower() .. "Lookup"
204+
l10n[newLookup] = {}
205+
CLI_Helpers.loadXML(helpers.get_project_dir_path() ..
206+
f("/.database_generator/Questie-translations/Localization/lookups/%s/lookup%ss/lookup%ss.clean.xml", questie_version, entityType, entityType))
207+
end
208+
209+
-- Validate that all the lookups are loaded
210+
for _, entityType in ipairs(entityTypes) do
211+
print("Validating " .. entityType .. " lookup")
212+
local lookup = l10n[entityType:lower() .. "Lookup"]
213+
if not lookup then
214+
print("Failed to load " .. entityType .. " lookup")
215+
os.exit(0)
216+
return
217+
end
218+
-- Validate that all locales are loaded
219+
for _, locale in ipairs(Corrections.L10nMeta.locales) do
220+
-- if locale ~= "enUS" then
221+
print(" Validating " .. entityType .. " lookup for locale: " .. locale)
222+
if not lookup[locale] then
223+
print("Failed to load " .. entityType .. " lookup for locale: " .. locale)
224+
os.exit(0)
225+
return
226+
end
227+
-- end
228+
end
229+
end
230+
print("All lookups and locales loaded successfully")
231+
232+
-- Create the l10n data table
233+
l10nOverride = l10n_loader.GenerateL10nTranslation(Corrections.L10nMeta.locales, entityTypes, l10n)
234+
235+
236+
-- DevTools_Dump(l10nOverride)
237+
238+
-- DevTools_Dump(l10nOverride)
239+
output = l10n_loader.DumpL10nData(Corrections.L10nMeta, entityTypes, l10nOverride)
240+
-- local l10nDumpFile = io.open(".database_generator/l10nData.lua-table", "w")
241+
-- if l10nDumpFile and output then
242+
-- l10nDumpFile:write(output)
243+
-- l10nDumpFile:close()
244+
-- -- print("Dumped l10n data to " .. f("%s/l10n/%s/l10nData.lua-table", basePath, capitalizedVersion))
245+
-- -- else
246+
-- -- print("Failed to open file for writing: " .. f("%s/l10n/%s/l10nData.lua-table", basePath, capitalizedVersion))
247+
-- end
248+
-- os.exit(0)
249+
end
250+
251+
-- Create output directories if they don't exist.
252+
-- ---@type string
253+
-- local basePath = f("%s/_data/output", helpers.get_script_dir())
254+
---@type string
255+
local basePath = f("%s../Database", helpers.get_script_dir())
256+
if not lfs.attributes(basePath, "mode") then
257+
lfs.mkdir(basePath)
258+
print("Created directory: " .. basePath)
259+
end
260+
for _, entityType in ipairs(entityTypes) do
261+
local path = f("%s/%s", basePath, entityType)
262+
if not lfs.attributes(path, "mode") then
263+
lfs.mkdir(path)
264+
print("Created directory: " .. path)
265+
end
266+
local versionPath = f("%s/%s", path, capitalizedVersion)
267+
if not lfs.attributes(versionPath, "mode") then
268+
lfs.mkdir(versionPath)
269+
print("Created directory: " .. versionPath)
270+
end
271+
end
272+
273+
-- ! We write all these files to disk for the sake of comparing changes between versions
274+
-- Write all the overrides to disk
275+
276+
-- ? Dump L10n Data
277+
print("Dumping L10n overrides")
278+
279+
local l10nDumpFile = io.open(f("%s/l10n/%s/l10nData.lua-table", basePath, capitalizedVersion), "w")
280+
if l10nDumpFile and output then
281+
l10nDumpFile:write(output)
282+
l10nDumpFile:close()
283+
print("Dumped l10n data to " .. f("%s/l10n/%s/l10nData.lua-table", basePath, capitalizedVersion))
284+
else
285+
print("Failed to open file for writing: " .. f("%s/l10n/%s/l10nData.lua-table", basePath, capitalizedVersion))
286+
end
287+
-- TODO: When we refactor the python code to lua this entire section will have changes
288+
local path = f("%s/l10n/%s/l10nData.lua-table", basePath, capitalizedVersion)
289+
print("Reading L10n data from " .. path)
290+
local l10nFile = io.open(path, "r")
291+
assert(l10nFile, "Failed to open file for reading " .. path)
292+
local l10nDataString = l10nFile:read("*a")
293+
l10nFile:close()
294+
local l10nData, errormsg = loadstring("return " .. l10nDataString)
295+
if not l10nData then
296+
print("Error loading L10n data: " .. errormsg)
297+
return
298+
end
299+
l10nData = l10nData()
300+
GenerateHtmlForEntityType(l10nData, Corrections.L10nMeta, "L10n", questiedb_version, nil, nil, debug)
301+
-- GenerateHtmlForEntityType(l10nData, Corrections.L10nMeta, "L10n", version, 75, 650, debug)
302+
303+
-- ? Dump Item Data
304+
print("Dumping item overrides")
305+
-- Generate the string representation of the merged item data.
306+
---@type string
307+
local itemDataString = helpers.dumpData(itemOverride, Corrections.ItemMeta.itemKeys, Corrections.ItemMeta.dumpFuncs,
308+
Corrections.ItemMeta.combine)
309+
-- Write the string to ItemData.lua-table.
310+
local itemFile = io.open(f("%s/Item/%s/ItemData.lua-table", basePath, capitalizedVersion), "w")
311+
assert(itemFile, "Failed to open file for writing")
312+
itemFile:write(itemDataString)
313+
itemFile:close()
314+
-- Generate the SimpleHTML files used by the addon.
315+
print("Dumping item overrides to HTML")
316+
GenerateHtmlForEntityType(itemOverride, Corrections.ItemMeta, "Item", questiedb_version, nil, nil, debug)
317+
-- GenerateHtmlForEntityType(itemOverride, Corrections.ItemMeta, "Item", version, 75, 650, debug)
318+
319+
-- ? Dump Quest Data
320+
print("Dumping quest overrides")
321+
local questDataString = helpers.dumpData(questOverride, Corrections.QuestMeta.questKeys, Corrections.QuestMeta.dumpFuncs)
322+
local questFile = io.open(f("%s/Quest/%s/QuestData.lua-table", basePath, capitalizedVersion), "w")
323+
assert(questFile, "Failed to open file for writing")
324+
questFile:write(questDataString)
325+
questFile:close()
326+
print("Dumping quest overrides to HTML")
327+
GenerateHtmlForEntityType(questOverride, Corrections.QuestMeta, "Quest", questiedb_version, nil, nil, debug)
328+
-- GenerateHtmlForEntityType(questOverride, Corrections.QuestMeta, "Quest", version, 75, 650, debug)
329+
330+
-- ? Dump Npc Data
331+
print("Dumping npc overrides")
332+
local npcDataString = helpers.dumpData(npcOverride, Corrections.NpcMeta.npcKeys, Corrections.NpcMeta.dumpFuncs,
333+
Corrections.NpcMeta.combine)
334+
local npcFile = io.open(f("%s/Npc/%s/NpcData.lua-table", basePath, capitalizedVersion), "w")
335+
assert(npcFile, "Failed to open file for writing")
336+
npcFile:write(npcDataString)
337+
npcFile:close()
338+
print("Dumping npc overrides to HTML")
339+
GenerateHtmlForEntityType(npcOverride, Corrections.NpcMeta, "Npc", questiedb_version, nil, nil, debug)
340+
-- GenerateHtmlForEntityType(npcOverride, Corrections.NpcMeta, "Npc", version, 75, 650, debug)
341+
342+
-- ? Dump Object Data
343+
print("Dumping object overrides")
344+
local objectDataString = helpers.dumpData(objectOverride, Corrections.ObjectMeta.objectKeys,
345+
Corrections.ObjectMeta.dumpFuncs)
346+
local objectFile = io.open(f("%s/Object/%s/ObjectData.lua-table", basePath, capitalizedVersion), "w")
347+
assert(objectFile, "Failed to open file for writing")
348+
objectFile:write(objectDataString)
349+
objectFile:close()
350+
print("Dumping object overrides to HTML")
351+
GenerateHtmlForEntityType(objectOverride, Corrections.ObjectMeta, "Object", questiedb_version, nil, nil, debug)
352+
-- GenerateHtmlForEntityType(objectOverride, Corrections.ObjectMeta, "Object", version, 75, 650, debug)
353+
354+
355+
print(f("\n\27[32m%s corrections dumped successfully\27[0m", capitalizedVersion))
356+
end

0 commit comments

Comments
 (0)