diff --git a/main.go b/main.go index 627eadf..fd8369c 100644 --- a/main.go +++ b/main.go @@ -39,7 +39,7 @@ func main() { // Flag for soc scraping scrapeOrganizations := flag.Bool("organizations", false, "Alongside -scrape, signifies that SOC organizations should be scraped.") // Flag for calendar scraping and parsing - calendar := flag.Bool("calendar", false, "Alongside -scrape or -parse, signifies that calendar should be scraped.") + cometCalendar := flag.Bool("cometCalendar", false, "Alongside -scrape or -parse, signifies that the Comet Calendar should be scraped/parsed.") // Flag for astra scraping and parsing astra := flag.Bool("astra", false, "Alongside -scrape or -parse, signifies that Astra should be scraped/parsed.") // Flag for mazevo scraping and parsing @@ -56,7 +56,7 @@ func main() { upload := flag.Bool("upload", false, "Puts the tool into upload mode.") replace := flag.Bool("replace", false, "Alongside -upload, specifies that uploaded data should replace existing data rather than being merged.") staticOnly := flag.Bool("static", false, "Alongside -upload, specifies that we should only build and upload the static aggregations.") - events := flag.Bool("events", false, "Alongside -upload, signifies that Astra and Mazevo should be uploaded.") + events := flag.Bool("events", false, "Alongside -upload, signifies that Astra, Mazevo, and the Comet Calendar should be uploaded.") // Flags for logging verbose := flag.Bool("verbose", false, "Enables verbose logging, good for debugging purposes.") @@ -106,8 +106,8 @@ func main() { scrapers.ScrapeCoursebook(*term, *startPrefix, *outDir, *resume) case *scrapeOrganizations: scrapers.ScrapeOrganizations(*outDir) - case *calendar: - scrapers.ScrapeCalendar(*outDir) + case *cometCalendar: + scrapers.ScrapeCometCalendar(*outDir) case *astra: scrapers.ScrapeAstra(*outDir) case *mazevo: @@ -119,8 +119,8 @@ func main() { } case *parse: switch { - case *calendar: - parser.ParseCalendar(*inDir, *outDir) + case *cometCalendar: + parser.ParseCometCalendar(*inDir, *outDir) case *astra: parser.ParseAstra(*inDir, *outDir) case *mazevo: diff --git a/parser/calendarParser.go b/parser/cometCalendarParser.go similarity index 94% rename from parser/calendarParser.go rename to parser/cometCalendarParser.go index 8f1beb9..a2f8e6e 100644 --- a/parser/calendarParser.go +++ b/parser/cometCalendarParser.go @@ -131,13 +131,13 @@ var validAbbreviations []string = []string{ "RCW", } -func ParseCalendar(inDir string, outDir string) { - - calendarFile, err := os.ReadFile(inDir + "/eventScraped.json") +func ParseCometCalendar(inDir string, outDir string) { + + calendarFile, err := os.ReadFile(inDir + "/cometCalendarScraped.json") if err != nil { panic(err) } - + var allEvents []schema.Event err = json.Unmarshal(calendarFile, &allEvents) @@ -147,7 +147,7 @@ func ParseCalendar(inDir string, outDir string) { multiBuildingMap := make(map[string]map[string]map[string][]schema.Event) - for _, event := range(allEvents) { + for _, event := range allEvents { // Get date dateTime := event.StartTime @@ -166,7 +166,7 @@ func ParseCalendar(inDir string, outDir string) { // buildingRegexp might capture something that isn't a valid building abbreviation (e.g., UTD) isValidBuilding := slices.Contains(validAbbreviations, building) - + // If location doesn't have building abbreviation or buildingRegexp captured an invalid abbreviation, // check for the full building name lowercaseLocation := strings.ToLower(*location) @@ -178,13 +178,13 @@ func ParseCalendar(inDir string, outDir string) { } } } - + // If location doesn't have room number, check to see if location included a room if room == "" && isValidBuilding { locationParts := strings.SplitN(*location, ",", 2) if len(locationParts) == 2 { room = locationParts[1] - } + } } // If building is still empty string, then location was initally an empty string @@ -218,13 +218,13 @@ func ParseCalendar(inDir string, outDir string) { var roomEvents []schema.RoomEvents[schema.Event] for room, events := range rooms { roomEvents = append(roomEvents, schema.RoomEvents[schema.Event]{ - Room: room, + Room: strings.TrimSpace(room), Events: events, }) } singleBuildings = append(singleBuildings, schema.SingleBuildingEvents[schema.Event]{ - Building: building, + Building: strings.TrimSpace(building), Rooms: roomEvents, }) } @@ -234,8 +234,8 @@ func ParseCalendar(inDir string, outDir string) { Buildings: singleBuildings, }) } - - log.Print("Parsed Calendar!") - utils.WriteJSON(fmt.Sprintf("%s/events.json", outDir), result) -} \ No newline at end of file + log.Print("Parsed Comet Calendar!") + + utils.WriteJSON(fmt.Sprintf("%s/cometCalendar.json", outDir), result) +} diff --git a/runners/daily.sh b/runners/daily.sh index 0a4c30a..f80f7dd 100644 --- a/runners/daily.sh +++ b/runners/daily.sh @@ -7,4 +7,6 @@ ./api-tools -headless -verbose -parse -mazevo ./api-tools -headless -verbose -scrape -astra ./api-tools -headless -verbose -parse -astra +./api-tools -headless -verbose -scrape -cometCalendar +./api-tools -headless -verbose -parse -cometCalendar ./api-tools -headless -verbose -upload -events diff --git a/scrapers/calendar.go b/scrapers/cometCalendar.go similarity index 96% rename from scrapers/calendar.go rename to scrapers/cometCalendar.go index 6fc493a..2149aa0 100644 --- a/scrapers/calendar.go +++ b/scrapers/cometCalendar.go @@ -31,8 +31,8 @@ type APICalendarResponse struct { Date map[string]string `json:"date"` } -// ScrapeCalendar retrieves calendar events through the API and writes normalized JSON output. -func ScrapeCalendar(outDir string) { +// ScrapeCometCalendar retrieves calendar events through the API and writes normalized JSON output. +func ScrapeCometCalendar(outDir string) { err := os.MkdirAll(outDir, 0777) if err != nil { panic(err) @@ -134,7 +134,7 @@ func ScrapeCalendar(outDir string) { log.Printf("Parsed the events of page %d successfully!\n\n", page+1) } - if err := utils.WriteJSON(fmt.Sprintf("%s/eventScraped.json", outDir), events); err != nil { + if err := utils.WriteJSON(fmt.Sprintf("%s/cometCalendarScraped.json", outDir), events); err != nil { panic(err) } log.Printf("Finished parsing %d events successfully!\n\n", len(events)) diff --git a/uploader/eventsUploader.go b/uploader/eventsUploader.go index 70d43df..dbfea1b 100644 --- a/uploader/eventsUploader.go +++ b/uploader/eventsUploader.go @@ -18,7 +18,7 @@ import ( // Note that this uploader assumes that the collection names match the names of these files, which they should. // If the names of these collections ever change, the file names should be updated accordingly. -var eventsFilesToUpload [2]string = [2]string{"astra.json", "mazevo.json"} +var eventsFilesToUpload [3]string = [3]string{"astra.json", "mazevo.json", "cometCalendar.json"} // UploadEvents loads event JSON files and replaces the corresponding MongoDB collections. func UploadEvents(inDir string) { @@ -54,6 +54,8 @@ func UploadEvents(inDir string) { UploadData[schema.MultiBuildingEvents[schema.AstraEvent]](client, ctx, fptr, true) case "mazevo.json": UploadData[schema.MultiBuildingEvents[schema.MazevoEvent]](client, ctx, fptr, true) + case "cometCalendar.json": + UploadData[schema.MultiBuildingEvents[schema.Event]](client, ctx, fptr, true) } } }