Dynamical retrieval working

Author: mikehquan19

parser/cometCalendarParser.go

@@ -12,13 +12,13 @@ import (
 	"regexp"
 	"slices"
 	"strings"
+	"time"
 
+	"github.com/UTDNebula/api-tools/scrapers"
 	"github.com/UTDNebula/api-tools/utils"
 	"github.com/UTDNebula/nebula-api/api/schema"
 )
 
-// TODO: FIND A WAY TO DYNAMICALLY RETRIEVE THIS
-
 // Some events have only the building name, not the abbreviation
 // Maps building names to their abbreviations
 var DefaultBuildings = map[string]string{
@@ -152,7 +152,7 @@ func ParseCometCalendar(inDir string, outDir string) {
 	}
 
 	multiBuildingMap := make(map[string]map[string]map[string][]schema.Event)
-	buildingAbbreviations, validAbbreviations := getAbbreviations(inDir)
+	buildingAbbreviations, validAbbreviations := getLocationAbbreviations(inDir)
 
 	for _, event := range allEvents {
 
@@ -248,32 +248,45 @@ func ParseCometCalendar(inDir string, outDir string) {
 }
 
 // getAbbreviations dynamically retrieves the all of the locations abbreviations
-func getAbbreviations(inDir string) (map[string]string, []string) {
+func getLocationAbbreviations(inDir string) (map[string]string, []string) {
 	// Get the locations from the map scraper
+	var mapFile []byte
+
 	mapFile, err := os.ReadFile(inDir + "/mapLocations.json")
 	if err != nil {
-		// Fall back if we haven't scraped the locations yet
-		return DefaultBuildings, DefaultValid
+		if os.IsNotExist(err) {
+			// Scrape the data if the it doesn't exist yet and then get the map file
+			scrapers.ScrapeMapLocations(inDir)
+			time.Sleep(2 * time.Second)
+			ParseMapLocations(inDir, inDir)
+
+			// If fail to get the locations again, not because unscraped
+			mapFile, _ = os.ReadFile(inDir + "/mapLocations.json")
+		} else {
+			panic(err)
+		}
 	}
-	var locations []map[string]any
+	var locations []schema.MapBuilding
 	if err = json.Unmarshal(mapFile, &locations); err != nil {
 		panic(err)
 	}
 
 	// Process the abbreviations
-	buildingsAbbrs := make(map[string]string, 0)
-	validAbbrs := make([]string, 0)
+	buildingsAbbreviations := make(map[string]string, 0)
+	validAbbreviations := make([]string, 0)
 
 	for _, location := range locations {
-		name := *utils.ConvertFromInterface[string](location["name"])
-		acronym := *utils.ConvertFromInterface[string](location["acronym"])
-
-		// Trim the tailing acronym in the name
-		trimmedName := strings.Split(name, " (")[0]
-		buildingsAbbrs[trimmedName] = acronym
+		// Trim the following acronym in the name
+		trimmedName := strings.Split(*location.Name, " (")[0]
+		// Fallback on the locations that have no acronyms
+		acronym := ""
+		if location.Acronym != nil {
+			acronym = *location.Acronym
+		}
 
-		validAbbrs = append(validAbbrs, acronym)
+		buildingsAbbreviations[trimmedName] = acronym
+		validAbbreviations = append(validAbbreviations, acronym)
 	}
 
-	return buildingsAbbrs, validAbbrs
+	return buildingsAbbreviations, validAbbreviations
 }

scrapers/cometCalendar.go

@@ -19,7 +19,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
-const BASE_CAL_URL string = "https://calendar.utdallas.edu/api/2/events"
+const CAL_URL string = "https://calendar.utdallas.edu/api/2/events"
 
 // RawEvent mirrors the nested event payload returned by the calendar API.
 type RawEvent struct {
@@ -44,7 +44,8 @@ func ScrapeCometCalendar(outDir string) {
 
 	// Get the total number of pages
 	log.Printf("Getting the number of pages...")
-	if err := callAPIAndUnmarshal(&client, 0, &calendarData); err != nil {
+
+	if err := callAndUnmarshal(&client, 0, &calendarData); err != nil {
 		panic(err)
 	}
 	numPages := calendarData.Page["total"]
@@ -53,7 +54,7 @@ func ScrapeCometCalendar(outDir string) {
 	var calendarEvents []schema.Event
 	for page := range numPages {
 		log.Printf("Scraping events of page %d...", page+1)
-		if err := callAPIAndUnmarshal(&client, page+1, &calendarData); err != nil {
+		if err := callAndUnmarshal(&client, page+1, &calendarData); err != nil {
 			panic(err)
 		}
 		for _, rawEvent := range calendarData.Events {
@@ -81,20 +82,22 @@ func ScrapeCometCalendar(outDir string) {
 				ContactPhoneNumber: contactInfo[2],
 			})
 		}
+
 		log.Printf("Scraped events of page %d successfully!\n", page+1)
 	}
 
 	writePath := fmt.Sprintf("%s/cometCalendarScraped.json", outDir)
 	if err := utils.WriteJSON(writePath, calendarEvents); err != nil {
 		panic(err)
 	}
+
 	log.Printf("Finished scraping %d events successfully!\n\n", len(calendarEvents))
 }
 
 // scrapeAndUnmarshal fetches a calendar page and decodes it into data.
-func callAPIAndUnmarshal(client *http.Client, page int, data *APICalendarResponse) error {
+func callAndUnmarshal(client *http.Client, page int, data *APICalendarResponse) error {
 	// Call API to get the byte data
-	calendarUrl := fmt.Sprintf("%s?days=365&pp=100&page=%d", BASE_CAL_URL, page)
+	calendarUrl := fmt.Sprintf("%s?days=365&pp=100&page=%d", CAL_URL, page)
 	request, err := http.NewRequest("GET", calendarUrl, nil)
 	if err != nil {
 		return err
@@ -162,7 +165,7 @@ func getEventLocation(event RawEvent) string {
 // getFilters parses the types, topics, and target audiences
 func getFilters(event RawEvent) ([]string, []string, []string) {
 	types := []string{}
-	targetAudiences := []string{}
+	audiences := []string{}
 	topics := []string{}
 
 	filters := convert[map[string]any](event.Event["filters"])
@@ -174,15 +177,15 @@ func getFilters(event RawEvent) ([]string, []string, []string) {
 
 	rawAudiences := convert[[]any](filters["event_target_audience"])
 	for _, audience := range rawAudiences {
-		targetAudiences = append(targetAudiences, convert[string](convert[map[string]any](audience)["name"]))
+		audiences = append(audiences, convert[string](convert[map[string]any](audience)["name"]))
 	}
 
 	rawTopics := convert[[]any](filters["event_topic"])
 	for _, topic := range rawTopics {
 		topics = append(topics, convert[string](convert[map[string]any](topic)["name"]))
 	}
 
-	return types, targetAudiences, topics
+	return types, audiences, topics
 }
 
 // getDepartmentsAndTags parses the departments, and tags