Recreate the getting of ItemQuestStarts #54
Description
Currently the value of these files are generated through golang, we should recreate it in either Python or lua, my suggestion is Python.
Should be simple enough tbh, just needs to be done. Check out the .wowhead code for insperation and where to put it.
A plan is to just change scrape.py argv to accept different types of runs,
if len(sys.argv) == 2:
if version == "all":
for version in allowed_expansions:
scrape(version, db_path)
stop_event.clear() # Reset the stop event for the next version
else:
scrape(version, db_path)
elif len(sys.argv) == 3:
if sys.argv[2] == "itemstarts":
from simple.getItemStarts import start_item_starts
start_item_starts(version)Example here makes is so that you run `scrape.py Classic itemstarts'
Here are some reference material from go:
package wowhead
import (
"db-extractor/extractor/database"
"errors"
"fmt"
"io"
"net/http"
"os"
"sort"
"sync"
)
func contains(s []database.ItemId, e database.ItemId) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}
func QuestieProcess() {
// This fetches and caches all creatures, the reason i do this is because this way i can download 15 at a time
var wg sync.WaitGroup
count := 0
for _, itemid := range WotlkItemIds {
wg.Add(1)
go func(itemid database.ItemId) {
defer wg.Done()
fileName := "out/wowheadCache/items/" + fmt.Sprint(itemid)
if _, err := os.Stat(fileName); errors.Is(err, os.ErrNotExist) {
GetWoWHeadItemData(itemid, false, "wotlk")
}
}(database.ItemId(itemid))
count++
if count%25 == 0 {
fmt.Println("Ids remaining: ", len(WotlkItemIds)-count)
wg.Wait()
}
}
wg.Wait()
DumpFunc := func(ids []database.ItemId, name string, version string) {
itemIdStarts := make(map[database.ItemId]Starts)
for _, itemid := range ids {
source := GetWoWHeadItemData(database.ItemId(itemid), true, version)
starts := GetStarts(source)
if starts != nil {
itemIdStarts[database.ItemId(itemid)] = starts
if len(starts) > 1 {
fmt.Println("hmm")
}
}
}
sortedIds := []int{}
for itemId := range itemIdStarts {
sortedIds = append(sortedIds, int(itemId))
}
sort.Ints(sortedIds)
writeString := "function ItemFixes.LoadItemQuestStarts()\n local itemKeys = ItemMeta.itemKeys\n\n"
writeString += " --! This file is automatically generated from wowhead data by looking at all items that starts a quest and correcting it.\n"
writeString += " return {\n"
for _, itemId := range sortedIds {
writeString += " --* Item " + fmt.Sprint(itemId) + fmt.Sprintf(" https://%s.wowhead.com/%s/item=", version, version) + fmt.Sprint(itemId) + "\n"
writeString += " --* Starts: " + itemIdStarts[database.ItemId(itemId)][0].Name + "(" + fmt.Sprint(itemIdStarts[database.ItemId(itemId)][0].ID) + ")" + fmt.Sprintf("(https://%s.wowhead.com/%s/quest=", version, version) + fmt.Sprint(itemIdStarts[database.ItemId(itemId)][0].ID) + ")\n"
writeString += fmt.Sprintf(" [%d] = {\n", itemId)
writeString += " [" + "itemKeys.startQuest" + "] = " + fmt.Sprint(itemIdStarts[database.ItemId(itemId)][0].ID) + ","
writeString += "\n },\n" //Close Item
}
writeString += " }\n"
writeString += "end"
sortedFile, _ := os.Create("out/" + "itemStart" + name + ".lua")
sortedFile.WriteString(writeString)
sortedFile.Close()
}
DumpFunc(ClassicItemIds, "Classic", "classic")
// Create a list with only items that do not exist in the ClassicItemIds list for SOD
var DeltaSodItemIds []database.ItemId
for _, id := range SodItemIds {
if !contains(ClassicItemIds, id) {
DeltaSodItemIds = append(DeltaSodItemIds, id)
}
}
DumpFunc(DeltaSodItemIds, "Sod", "classic")
// Create a list with only items that do not exist in the ClassicItemIds list
var DeltaTbcItemIds []database.ItemId
for _, id := range TbcItemIds {
if !contains(ClassicItemIds, id) {
DeltaTbcItemIds = append(DeltaTbcItemIds, id)
}
}
DumpFunc(DeltaTbcItemIds, "TBC", "tbc")
// Create a list with only items that do not exist in the TbcItemIds and ClassicItemIds list
var DeltaWotlkItemIds []database.ItemId
for _, id := range WotlkItemIds {
if !contains(TbcItemIds, id) && !contains(ClassicItemIds, id) {
DeltaWotlkItemIds = append(DeltaWotlkItemIds, id)
}
}
DumpFunc(DeltaWotlkItemIds, "Wotlk", "wotlk")
fmt.Println("Done")
}
func GetRetailWoWHeadItemData(id database.ItemId, loadFile bool) string {
source := ""
url := "https://wowhead.com/item=" + fmt.Sprint(id)
fileName := "out/wowheadCache/retailItems/" + fmt.Sprint(id)
if _, err := os.Stat(fileName); errors.Is(err, os.ErrNotExist) {
// File does not exist
fmt.Printf("Fetching ITEM %s\n", url)
resp, err := http.Get(url)
// handle the error if there is one
if err != nil {
panic(err)
}
// do this now so it won't be forgotten
defer resp.Body.Close()
// reads html as a slice of bytes
html, err := io.ReadAll(resp.Body)
if err != nil {
panic(err)
}
source = string(html)
// Save to file
cacheFile, _ := os.Create(fileName)
cacheFile.WriteString(source)
cacheFile.Close()
} else if loadFile {
fmt.Printf("Loading ITEM %s\n", fileName)
// File exists
sourceBytes, err := os.ReadFile(fileName)
if err != nil {
panic(err)
}
source = string(sourceBytes)
}
return source
}
func GetWoWHeadItemData(id database.ItemId, loadFile bool, version string) string {
source := ""
url := fmt.Sprintf("https://%s.wowhead.com/%s/item=", version, version) + fmt.Sprint(id)
fileName := "out/wowheadCache/items/" + fmt.Sprint(id)
if _, err := os.Stat(fileName); errors.Is(err, os.ErrNotExist) {
// File does not exist
fmt.Printf("Fetching ITEM %s\n", url)
resp, err := http.Get(url)
// handle the error if there is one
if err != nil {
panic(err)
}
// do this now so it won't be forgotten
defer resp.Body.Close()
// reads html as a slice of bytes
html, err := io.ReadAll(resp.Body)
if err != nil {
panic(err)
}
source = string(html)
// Save to file
cacheFile, _ := os.Create(fileName)
cacheFile.WriteString(source)
cacheFile.Close()
} else if loadFile {
// fmt.Printf("Loading ITEM %s\n", fileName)
// File exists
sourceBytes, err := os.ReadFile(fileName)
if err != nil {
panic(err)
}
source = string(sourceBytes)
}
return source
}
var itemids = []database.ItemId{ /* all ids be here, yarr, removed due to being so long */}package wowhead
import (
"db-extractor/extractor/database"
"encoding/json"
"fmt"
"regexp"
"strconv"
)
type AreaCoordinates struct {
Count int `json:"count"`
Coords [][]float64 `json:"coords"`
UIMapID int `json:"uiMapId"`
UIMapName string `json:"uiMapName"`
}
func GetVersion(source string) (Full string, Major int, Minor int, Patch int) {
regex, _ := regexp.Compile(`Added in patch ((\d*)\.(\d*)\.(\d*))\.*(\d*)`)
matches := regex.FindStringSubmatch(source)
if len(matches) >= 4 {
Major, err := strconv.Atoi(matches[2])
Minor, err2 := strconv.Atoi(matches[3])
Patch, err3 := strconv.Atoi(matches[4])
if err != nil || err2 != nil || err3 != nil {
panic("Could not parse version")
}
return matches[1], Major, Minor, Patch
}
return "", 0, 0, 0
}
type Starts []struct {
Category int `json:"category"`
Category2 int `json:"category2"`
ID int `json:"id"`
Itemchoices [][]int `json:"itemchoices"`
Level int `json:"level"`
Name string `json:"name"`
Reprewards [][]int `json:"reprewards"`
Reqlevel int `json:"reqlevel"`
Side int `json:"side"`
Wflags int `json:"wflags"`
}
func GetStarts(source string) Starts {
regex, _ := regexp.Compile(`name: WH.TERMS.starts,\n.+\n.+\n.+data:(.*),`)
// regex, _ := regexp.Compile(`name: WH.TERMS.starts,[\s\S]+?data:(.*),`)
matches := regex.FindStringSubmatch(source)
result := Starts{}
if len(matches) == 2 {
fmt.Println(matches[1])
json.Unmarshal([]byte(matches[1]), &result)
return result
}
return nil
}
// return data / sucessfully parsed
func GetGNpcs(id database.CreatureId, source string) (int, bool) {
//g_quests[10400], {"category":3483,"category2":8,"id":10400,"itemchoices":[[28040,1],[28041,1],[28042,1]],"level":63,"money":87000,"name":"Overlord","reprewards":[[946,500]],"reqlevel":58,"side":1,"type":1,"xp":15550}
regex, _ := regexp.Compile(`g_npcs\[` + fmt.Sprint(id) + `\], ({.+?})`)
matches := regex.FindStringSubmatch(source)
// result := WoWHeadQuestData{}
if len(matches) == 2 {
// fmt.Println(matches[1])
// json.Unmarshal([]byte(matches[1]), &result)
return 0, true
}
return 0, false
}