diff --git a/go.mod b/go.mod index d8bf73a..756e7ed 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -41,6 +42,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect @@ -54,5 +56,6 @@ require ( golang.org/x/sys v0.30.0 // indirect golang.org/x/text v0.22.0 // indirect google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a51ca9f..df6c260 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,7 @@ github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHG github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -54,12 +55,19 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= @@ -79,8 +87,12 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhA github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -157,6 +169,8 @@ google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwl google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index f0253cc..ac3de35 100644 --- a/main.go +++ b/main.go @@ -39,8 +39,8 @@ func main() { scrapeOrganizations := flag.Bool("organizations", false, "Alongside -scrape, signifies that SOC organizations should be scraped.") // Flag for calendar scraping scrapeCalendar := flag.Bool("calendar", false, "Alongside -scrape, signifies that calendar should be scraped.") - // Flag for astra scraping - scrapeAstra := flag.Bool("astra", false, "Alongside -scrape, signifies that Astra should be scraped.") + // Flag for astra scraping and parsing + astra := flag.Bool("astra", false, "Alongside -scrape or -parse, signifies that Astra should be scraped/parsed.") // Flag for mazevo scraping and parsing mazevo := flag.Bool("mazevo", false, "Alongside -scrape or -parse, signifies that Mazevo should be scraped/parsed.") @@ -52,6 +52,7 @@ func main() { // Flags for uploading data upload := flag.Bool("upload", false, "Puts the tool into upload mode.") replace := flag.Bool("replace", false, "Alongside -upload, specifies that uploaded data should replace existing data rather than being merged.") + events := flag.Bool("events", false, "Alongside -upload, signifies that Astra and Mazevo should be uploaded.") // Flags for logging verbose := flag.Bool("verbose", false, "Enables verbose logging, good for debugging purposes.") @@ -103,7 +104,7 @@ func main() { scrapers.ScrapeOrganizations(*outDir) case *scrapeCalendar: scrapers.ScrapeCalendar(*outDir) - case *scrapeAstra: + case *astra: scrapers.ScrapeAstra(*outDir) case *mazevo: scrapers.ScrapeMazevo(*outDir) @@ -112,13 +113,20 @@ func main() { } case *parse: switch { + case *astra: + parser.ParseAstra(*inDir, *outDir) case *mazevo: parser.ParseMazevo(*inDir, *outDir) default: parser.Parse(*inDir, *outDir, *csvDir, *skipValidation) } case *upload: - uploader.Upload(*inDir, *replace) + switch { + case *events: + uploader.UploadEvents(*inDir) + default: + uploader.Upload(*inDir, *replace) + } default: flag.PrintDefaults() return diff --git a/parser/astraParser.go b/parser/astraParser.go new file mode 100644 index 0000000..15a2c2d --- /dev/null +++ b/parser/astraParser.go @@ -0,0 +1,107 @@ +package parser + +import ( + "encoding/json" + "fmt" + "log" + "os" + "strings" + + "github.com/UTDNebula/api-tools/utils" + "github.com/UTDNebula/nebula-api/api/schema" +) + +type InputData struct { + Fields string `json:"fields"` + Data [][]interface{} `json:"data"` +} + +func ParseAstra(inDir string, outDir string) { + + astraFile, err := os.ReadFile(inDir + "/astraScraped.json") + if err != nil { + panic(err) + } + + var rawData map[string]InputData + err = json.Unmarshal(astraFile, &rawData) + if err != nil { + panic(err) + } + + var result []schema.MultiBuildingEvents[schema.AstraEvent] + + for date, data := range rawData { + fieldMap := mapFields(data.Fields) + buildingsMap := make(map[string]map[string][]schema.AstraEvent) + + for _, record := range data.Data { + building := getString(record, fieldMap["BuildingCode"]) + room := getString(record, fieldMap["RoomNumber"]) + event := schema.AstraEvent{ + ActivityName: getString(record, fieldMap["ActivityName"]), + MeetingType: getString(record, fieldMap["MeetingType"]), + StartDate: getString(record, fieldMap["StartDate"]), + EndDate: getString(record, fieldMap["EndDate"]), + CurrentState: getString(record, fieldMap["CurrentState"]), + NotAllowedUsageMask: getInt(record, fieldMap["NotAllowedUsageMask"]), + UsageColor: getString(record, fieldMap["UsageColor"]), + Capacity: getInt(record, fieldMap["Capacity"]), + } + + if building == nil || room == nil || *(building) == "" || *(room) == "" { + continue + } + + if _, exists := buildingsMap[*building]; !exists { + buildingsMap[*building] = make(map[string][]schema.AstraEvent) + } + buildingsMap[*building][*room] = append(buildingsMap[*building][*room], event) + } + + var buildings []schema.SingleBuildingEvents[schema.AstraEvent] + for buildingCode, rooms := range buildingsMap { + var roomList []schema.RoomEvents[schema.AstraEvent] + for roomNumber, events := range rooms { + roomList = append(roomList, schema.RoomEvents[schema.AstraEvent]{Room: roomNumber, Events: events}) + } + buildings = append(buildings, schema.SingleBuildingEvents[schema.AstraEvent]{Building: buildingCode, Rooms: roomList}) + } + data := schema.MultiBuildingEvents[schema.AstraEvent]{ + Date: date, + Buildings: buildings, + } + result = append(result, data) + } + + log.Print("Parsed Astra!") + + utils.WriteJSON(fmt.Sprintf("%s/astra.json", outDir), result) +} + +func mapFields(fields string) map[string]int { + fieldNames := map[string]int{} + fieldList := strings.Split(fields, ",") + for i, name := range fieldList { + fieldNames[name] = i + } + return fieldNames +} + +func getString(record []interface{}, place int) *string { + val := record[place] + if val == nil { + return nil + } + s := val.(string) + return &s +} + +func getInt(record []interface{}, place int) *float64 { + val := record[place] + if val == nil { + return nil + } + i := val.(float64) + return &i +} diff --git a/scrapers/astra.go b/scrapers/astra.go index 24bf571..d1c10ee 100644 --- a/scrapers/astra.go +++ b/scrapers/astra.go @@ -106,7 +106,7 @@ func ScrapeAstra(outDir string) { // Write event data to output file days = fmt.Sprintf("%s}", days) - fptr, err := os.Create(fmt.Sprintf("%s/reservations.json", outDir)) + fptr, err := os.Create(fmt.Sprintf("%s/astraScraped.json", outDir)) if err != nil { panic(err) } diff --git a/uploader/eventsUploader.go b/uploader/eventsUploader.go new file mode 100644 index 0000000..00babaf --- /dev/null +++ b/uploader/eventsUploader.go @@ -0,0 +1,56 @@ +/* + This file is responsible for handling uploading of parsed event data to MongoDB. +*/ + +package uploader + +import ( + "context" + "fmt" + "log" + "os" + "time" + + "github.com/UTDNebula/nebula-api/api/schema" + "github.com/joho/godotenv" +) + +// Note that this uploader assumes that the collection names match the names of these files, which they should. +// If the names of these collections ever change, the file names should be updated accordingly. + +var eventsFilesToUpload [1]string = [1]string{"astra.json"} + +func UploadEvents(inDir string) { + + //Load env vars + if err := godotenv.Load(); err != nil { + log.Panic("Error loading .env file") + } + + //Connect to mongo + client := connectDB() + + // Get 5 minute context + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + for _, path := range eventsFilesToUpload { + + // Open data file for reading + fptr, err := os.Open(fmt.Sprintf("%s/"+path, inDir)) + if err != nil { + if os.IsNotExist(err) { + log.Printf("File not found. Skipping %s", path) + continue + } + log.Panic(err) + } + + defer fptr.Close() + + switch path { + case "astra.json": + UploadData[schema.MultiBuildingEvents[schema.AstraEvent]](client, ctx, fptr, true) + } + } +}