Merge pull request #102 from UTDNebula/comet-cal-upload-and-automate

Comet calendar upload and automate

Author: mikehquan19

main.go

@@ -39,7 +39,7 @@ func main() {
 	// Flag for soc scraping
 	scrapeOrganizations := flag.Bool("organizations", false, "Alongside -scrape, signifies that SOC organizations should be scraped.")
 	// Flag for calendar scraping and parsing
-	calendar := flag.Bool("calendar", false, "Alongside -scrape or -parse, signifies that calendar should be scraped.")
+	cometCalendar := flag.Bool("cometCalendar", false, "Alongside -scrape or -parse, signifies that the Comet Calendar should be scraped/parsed.")
 	// Flag for astra scraping and parsing
 	astra := flag.Bool("astra", false, "Alongside -scrape or -parse, signifies that Astra should be scraped/parsed.")
 	// Flag for mazevo scraping and parsing
@@ -56,7 +56,7 @@ func main() {
 	upload := flag.Bool("upload", false, "Puts the tool into upload mode.")
 	replace := flag.Bool("replace", false, "Alongside -upload, specifies that uploaded data should replace existing data rather than being merged.")
 	staticOnly := flag.Bool("static", false, "Alongside -upload, specifies that we should only build and upload the static aggregations.")
-	events := flag.Bool("events", false, "Alongside -upload, signifies that Astra and Mazevo should be uploaded.")
+	events := flag.Bool("events", false, "Alongside -upload, signifies that Astra, Mazevo, and the Comet Calendar should be uploaded.")
 
 	// Flags for logging
 	verbose := flag.Bool("verbose", false, "Enables verbose logging, good for debugging purposes.")
@@ -106,8 +106,8 @@ func main() {
 			scrapers.ScrapeCoursebook(*term, *startPrefix, *outDir, *resume)
 		case *scrapeOrganizations:
 			scrapers.ScrapeOrganizations(*outDir)
-		case *calendar:
-			scrapers.ScrapeCalendar(*outDir)
+		case *cometCalendar:
+			scrapers.ScrapeCometCalendar(*outDir)
 		case *astra:
 			scrapers.ScrapeAstra(*outDir)
 		case *mazevo:
@@ -119,8 +119,8 @@ func main() {
 		}
 	case *parse:
 		switch {
-		case *calendar:
-			parser.ParseCalendar(*inDir, *outDir)
+		case *cometCalendar:
+			parser.ParseCometCalendar(*inDir, *outDir)
 		case *astra:
 			parser.ParseAstra(*inDir, *outDir)
 		case *mazevo:

parser/calendarParser.go parser/cometCalendarParser.goparser/calendarParser.go renamed to parser/cometCalendarParser.go

@@ -131,13 +131,13 @@ var validAbbreviations []string = []string{
 	"RCW",
 }
 
-func ParseCalendar(inDir string, outDir string) {
-	
-	calendarFile, err := os.ReadFile(inDir + "/eventScraped.json")
+func ParseCometCalendar(inDir string, outDir string) {
+
+	calendarFile, err := os.ReadFile(inDir + "/cometCalendarScraped.json")
 	if err != nil {
 		panic(err)
 	}
-	
+
 	var allEvents []schema.Event
 
 	err = json.Unmarshal(calendarFile, &allEvents)
@@ -147,7 +147,7 @@ func ParseCalendar(inDir string, outDir string) {
 
 	multiBuildingMap := make(map[string]map[string]map[string][]schema.Event)
 
-	for _, event := range(allEvents) {
+	for _, event := range allEvents {
 
 		// Get date
 		dateTime := event.StartTime
@@ -166,7 +166,7 @@ func ParseCalendar(inDir string, outDir string) {
 
 		// buildingRegexp might capture something that isn't a valid building abbreviation (e.g., UTD)
 		isValidBuilding := slices.Contains(validAbbreviations, building)
-		
+
 		// If location doesn't have building abbreviation or buildingRegexp captured an invalid abbreviation,
 		// check for the full building name
 		lowercaseLocation := strings.ToLower(*location)
@@ -178,13 +178,13 @@ func ParseCalendar(inDir string, outDir string) {
 				}
 			}
 		}
-		
+
 		// If location doesn't have room number, check to see if location included a room
 		if room == "" && isValidBuilding {
 			locationParts := strings.SplitN(*location, ",", 2)
 			if len(locationParts) == 2 {
 				room = locationParts[1]
-			} 
+			}
 		}
 
 		// If building is still empty string, then location was initally an empty string
@@ -218,13 +218,13 @@ func ParseCalendar(inDir string, outDir string) {
 			var roomEvents []schema.RoomEvents[schema.Event]
 			for room, events := range rooms {
 				roomEvents = append(roomEvents, schema.RoomEvents[schema.Event]{
-					Room:   room,
+					Room:   strings.TrimSpace(room),
 					Events: events,
 				})
 			}
 
 			singleBuildings = append(singleBuildings, schema.SingleBuildingEvents[schema.Event]{
-				Building: building,
+				Building: strings.TrimSpace(building),
 				Rooms:    roomEvents,
 			})
 		}
@@ -234,8 +234,8 @@ func ParseCalendar(inDir string, outDir string) {
 			Buildings: singleBuildings,
 		})
 	}
-	
-	log.Print("Parsed Calendar!")
 
-	utils.WriteJSON(fmt.Sprintf("%s/events.json", outDir), result)
-}
+	log.Print("Parsed Comet Calendar!")
+
+	utils.WriteJSON(fmt.Sprintf("%s/cometCalendar.json", outDir), result)
+}

runners/daily.sh

@@ -7,4 +7,6 @@
 ./api-tools -headless -verbose -parse -mazevo
 ./api-tools -headless -verbose -scrape -astra
 ./api-tools -headless -verbose -parse -astra
+./api-tools -headless -verbose -scrape -cometCalendar
+./api-tools -headless -verbose -parse -cometCalendar
 ./api-tools -headless -verbose -upload -events

scrapers/calendar.go scrapers/cometCalendar.goscrapers/calendar.go renamed to scrapers/cometCalendar.go

@@ -31,8 +31,8 @@ type APICalendarResponse struct {
 	Date   map[string]string `json:"date"`
 }
 
-// ScrapeCalendar retrieves calendar events through the API and writes normalized JSON output.
-func ScrapeCalendar(outDir string) {
+// ScrapeCometCalendar retrieves calendar events through the API and writes normalized JSON output.
+func ScrapeCometCalendar(outDir string) {
 	err := os.MkdirAll(outDir, 0777)
 	if err != nil {
 		panic(err)
@@ -134,7 +134,7 @@ func ScrapeCalendar(outDir string) {
 		log.Printf("Parsed the events of page %d successfully!\n\n", page+1)
 	}
 
-	if err := utils.WriteJSON(fmt.Sprintf("%s/eventScraped.json", outDir), events); err != nil {
+	if err := utils.WriteJSON(fmt.Sprintf("%s/cometCalendarScraped.json", outDir), events); err != nil {
 		panic(err)
 	}
 	log.Printf("Finished parsing %d events successfully!\n\n", len(events))

uploader/eventsUploader.go

@@ -18,7 +18,7 @@ import (
 //  Note that this uploader assumes that the collection names match the names of these files, which they should.
 //  If the names of these collections ever change, the file names should be updated accordingly.
 
-var eventsFilesToUpload [2]string = [2]string{"astra.json", "mazevo.json"}
+var eventsFilesToUpload [3]string = [3]string{"astra.json", "mazevo.json", "cometCalendar.json"}
 
 // UploadEvents loads event JSON files and replaces the corresponding MongoDB collections.
 func UploadEvents(inDir string) {
@@ -54,6 +54,8 @@ func UploadEvents(inDir string) {
 			UploadData[schema.MultiBuildingEvents[schema.AstraEvent]](client, ctx, fptr, true)
 		case "mazevo.json":
 			UploadData[schema.MultiBuildingEvents[schema.MazevoEvent]](client, ctx, fptr, true)
+		case "cometCalendar.json":
+			UploadData[schema.MultiBuildingEvents[schema.Event]](client, ctx, fptr, true)
 		}
 	}
 }