Astra uploader

go.mod

@@ -34,13 +34,15 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+  github.com/kr/pretty v0.3.1 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
@@ -54,5 +56,6 @@ require (
 	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/text v0.22.0 // indirect
 	google.golang.org/protobuf v1.36.5 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -18,6 +18,7 @@ github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHG
 github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
 github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
 github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -54,12 +55,19 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
 github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
 github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
@@ -79,8 +87,12 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhA
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -157,6 +169,8 @@ google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwl
 google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -39,8 +39,8 @@ func main() {
 	scrapeOrganizations := flag.Bool("organizations", false, "Alongside -scrape, signifies that SOC organizations should be scraped.")
 	// Flag for calendar scraping
 	scrapeCalendar := flag.Bool("calendar", false, "Alongside -scrape, signifies that calendar should be scraped.")
-	// Flag for astra scraping
-	scrapeAstra := flag.Bool("astra", false, "Alongside -scrape, signifies that Astra should be scraped.")
+	// Flag for astra scraping and parsing
+	astra := flag.Bool("astra", false, "Alongside -scrape or -parse, signifies that Astra should be scraped/parsed.")
 	// Flag for mazevo scraping and parsing
 	mazevo := flag.Bool("mazevo", false, "Alongside -scrape or -parse, signifies that Mazevo should be scraped/parsed.")
 
@@ -52,6 +52,7 @@ func main() {
 	// Flags for uploading data
 	upload := flag.Bool("upload", false, "Puts the tool into upload mode.")
 	replace := flag.Bool("replace", false, "Alongside -upload, specifies that uploaded data should replace existing data rather than being merged.")
+	events := flag.Bool("events", false, "Alongside -upload, signifies that Astra and Mazevo should be uploaded.")
 
 	// Flags for logging
 	verbose := flag.Bool("verbose", false, "Enables verbose logging, good for debugging purposes.")
@@ -103,7 +104,7 @@ func main() {
 			scrapers.ScrapeOrganizations(*outDir)
 		case *scrapeCalendar:
 			scrapers.ScrapeCalendar(*outDir)
-		case *scrapeAstra:
+		case *astra:
 			scrapers.ScrapeAstra(*outDir)
 		case *mazevo:
 			scrapers.ScrapeMazevo(*outDir)
@@ -112,13 +113,20 @@ func main() {
 		}
 	case *parse:
 		switch {
+		case *astra:
+			parser.ParseAstra(*inDir, *outDir)
 		case *mazevo:
 			parser.ParseMazevo(*inDir, *outDir)
 		default:
 			parser.Parse(*inDir, *outDir, *csvDir, *skipValidation)
 		}
 	case *upload:
-		uploader.Upload(*inDir, *replace)
+		switch {
+		case *events:
+			uploader.UploadEvents(*inDir)
+		default:
+			uploader.Upload(*inDir, *replace)
+		}
 	default:
 		flag.PrintDefaults()
 		return

parser/astraParser.go

@@ -0,0 +1,107 @@
+package parser
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/UTDNebula/api-tools/utils"
+	"github.com/UTDNebula/nebula-api/api/schema"
+)
+
+type InputData struct {
+	Fields string          `json:"fields"`
+	Data   [][]interface{} `json:"data"`
+}
+
+func ParseAstra(inDir string, outDir string) {
+
+	astraFile, err := os.ReadFile(inDir + "/astraScraped.json")
+	if err != nil {
+		panic(err)
+	}
+
+	var rawData map[string]InputData
+	err = json.Unmarshal(astraFile, &rawData)
+	if err != nil {
+		panic(err)
+	}
+
+	var result []schema.MultiBuildingEvents[schema.AstraEvent]
+
+	for date, data := range rawData {
+		fieldMap := mapFields(data.Fields)
+		buildingsMap := make(map[string]map[string][]schema.AstraEvent)
+
+		for _, record := range data.Data {
+			building := getString(record, fieldMap["BuildingCode"])
+			room := getString(record, fieldMap["RoomNumber"])
+			event := schema.AstraEvent{
+				ActivityName:        getString(record, fieldMap["ActivityName"]),
+				MeetingType:         getString(record, fieldMap["MeetingType"]),
+				StartDate:           getString(record, fieldMap["StartDate"]),
+				EndDate:             getString(record, fieldMap["EndDate"]),
+				CurrentState:        getString(record, fieldMap["CurrentState"]),
+				NotAllowedUsageMask: getInt(record, fieldMap["NotAllowedUsageMask"]),
+				UsageColor:          getString(record, fieldMap["UsageColor"]),
+				Capacity:            getInt(record, fieldMap["Capacity"]),
+			}
+
+			if building == nil || room == nil || *(building) == "" || *(room) == "" {
+				continue
+			}
+
+			if _, exists := buildingsMap[*building]; !exists {
+				buildingsMap[*building] = make(map[string][]schema.AstraEvent)
+			}
+			buildingsMap[*building][*room] = append(buildingsMap[*building][*room], event)
+		}
+
+		var buildings []schema.SingleBuildingEvents[schema.AstraEvent]
+		for buildingCode, rooms := range buildingsMap {
+			var roomList []schema.RoomEvents[schema.AstraEvent]
+			for roomNumber, events := range rooms {
+				roomList = append(roomList, schema.RoomEvents[schema.AstraEvent]{Room: roomNumber, Events: events})
+			}
+			buildings = append(buildings, schema.SingleBuildingEvents[schema.AstraEvent]{Building: buildingCode, Rooms: roomList})
+		}
+		data := schema.MultiBuildingEvents[schema.AstraEvent]{
+			Date:      date,
+			Buildings: buildings,
+		}
+		result = append(result, data)
+	}
+
+	log.Print("Parsed Astra!")
+
+	utils.WriteJSON(fmt.Sprintf("%s/astra.json", outDir), result)
+}
+
+func mapFields(fields string) map[string]int {
+	fieldNames := map[string]int{}
+	fieldList := strings.Split(fields, ",")
+	for i, name := range fieldList {
+		fieldNames[name] = i
+	}
+	return fieldNames
+}
+
+func getString(record []interface{}, place int) *string {
+	val := record[place]
+	if val == nil {
+		return nil
+	}
+	s := val.(string)
+	return &s
+}
+
+func getInt(record []interface{}, place int) *float64 {
+	val := record[place]
+	if val == nil {
+		return nil
+	}
+	i := val.(float64)
+	return &i
+}

scrapers/astra.go

@@ -106,7 +106,7 @@ func ScrapeAstra(outDir string) {
 
 	// Write event data to output file
 	days = fmt.Sprintf("%s}", days)
-	fptr, err := os.Create(fmt.Sprintf("%s/reservations.json", outDir))
+	fptr, err := os.Create(fmt.Sprintf("%s/astraScraped.json", outDir))
 	if err != nil {
 		panic(err)
 	}

uploader/eventsUploader.go

@@ -0,0 +1,56 @@
+/*
+	This file is responsible for handling uploading of parsed event data to MongoDB.
+*/
+
+package uploader
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/UTDNebula/nebula-api/api/schema"
+	"github.com/joho/godotenv"
+)
+
+//  Note that this uploader assumes that the collection names match the names of these files, which they should.
+//  If the names of these collections ever change, the file names should be updated accordingly.
+
+var eventsFilesToUpload [1]string = [1]string{"astra.json"}
+
+func UploadEvents(inDir string) {
+
+	//Load env vars
+	if err := godotenv.Load(); err != nil {
+		log.Panic("Error loading .env file")
+	}
+
+	//Connect to mongo
+	client := connectDB()
+
+	// Get 5 minute context
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	for _, path := range eventsFilesToUpload {
+
+		// Open data file for reading
+		fptr, err := os.Open(fmt.Sprintf("%s/"+path, inDir))
+		if err != nil {
+			if os.IsNotExist(err) {
+				log.Printf("File not found. Skipping %s", path)
+				continue
+			}
+			log.Panic(err)
+		}
+
+		defer fptr.Close()
+
+		switch path {
+		case "astra.json":
+			UploadData[schema.MultiBuildingEvents[schema.AstraEvent]](client, ctx, fptr, true)
+		}
+	}
+}