Merge branch 'develop' into issue-40

Author: mikehquan19

parser/cometCalendarParser.go

@@ -1,3 +1,7 @@
+/*
+	This file contains the code for the comet calendar events parser.
+*/
+
 package parser
 
 import (
@@ -8,14 +12,16 @@ import (
 	"regexp"
 	"slices"
 	"strings"
+	"time"
 
+	"github.com/UTDNebula/api-tools/scrapers"
 	"github.com/UTDNebula/api-tools/utils"
 	"github.com/UTDNebula/nebula-api/api/schema"
 )
 
 // Some events have only the building name, not the abbreviation
 // Maps building names to their abbreviations
-var buildingAbbreviations = map[string]string{
+var DefaultBuildings = map[string]string{
 	"Activity Center":                              "AB",
 	"Activity Center Bookstore":                    "ACB",
 	"Administration":                               "AD",
@@ -74,7 +80,7 @@ var buildingAbbreviations = map[string]string{
 }
 
 // Valid building abreviations for checking
-var validAbbreviations []string = []string{
+var DefaultValid []string = []string{
 	"AB",
 	"ACB",
 	"AD",
@@ -146,6 +152,11 @@ func ParseCometCalendar(inDir string, outDir string) {
 	}
 
 	multiBuildingMap := make(map[string]map[string]map[string][]schema.Event)
+	// Some events have only the building name, not the abbreviation
+	buildingAbbreviations, validAbbreviations, err := getLocationAbbreviations(inDir)
+	if err != nil {
+		panic(err)
+	}
 
 	for _, event := range allEvents {
 
@@ -181,7 +192,7 @@ func ParseCometCalendar(inDir string, outDir string) {
 
 		// If location doesn't have room number, check to see if location included a room
 		if room == "" && isValidBuilding {
-			locationParts := strings.SplitN(*location, ",", 2)
+			locationParts := strings.SplitN(*location, ", ", 2)
 			if len(locationParts) == 2 {
 				room = locationParts[1]
 			}
@@ -239,3 +250,52 @@ func ParseCometCalendar(inDir string, outDir string) {
 
 	utils.WriteJSON(fmt.Sprintf("%s/cometCalendar.json", outDir), result)
 }
+
+// getAbbreviations dynamically retrieves the all of the locations abbreviations
+func getLocationAbbreviations(inDir string) (map[string]string, []string, error) {
+	// Get the locations from the map scraper
+	var mapFile []byte
+
+	mapFile, err := os.ReadFile(inDir + "/mapLocations.json")
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Scrape the data if the it doesn't exist yet and then get the map file
+			scrapers.ScrapeMapLocations(inDir)
+			time.Sleep(2 * time.Second)
+			ParseMapLocations(inDir, inDir)
+			time.Sleep(2 * time.Second)
+
+			// If fail to get the locations again, it's not because location is unscraped
+			mapFile, err = os.ReadFile(inDir + "/mapLocations.json")
+			if err != nil {
+				return nil, nil, err
+			}
+		} else {
+			return nil, nil, err
+		}
+	}
+
+	var locations []schema.MapBuilding
+	if err = json.Unmarshal(mapFile, &locations); err != nil {
+		return nil, nil, err
+	}
+
+	// Process the abbreviations
+	buildingsAbbreviations := make(map[string]string, 0) // Maps building names to their abbreviations
+	validAbbreviations := make([]string, 0)              // Valid building abreviations for checking
+
+	for _, location := range locations {
+		// Trim the following acronym in the name
+		trimmedName := strings.Split(*location.Name, " (")[0]
+		// Fallback on the locations that have no acronyms
+		abbreviation := ""
+		if location.Acronym != nil {
+			abbreviation = *location.Acronym
+		}
+
+		buildingsAbbreviations[trimmedName] = abbreviation
+		validAbbreviations = append(validAbbreviations, abbreviation)
+	}
+
+	return buildingsAbbreviations, validAbbreviations, nil
+}

parser/gradeLoader.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 )
 
+var grades = []string{"A+", "A", "A-", "B+", "B", "B-", "C+", "C", "C-", "D+", "D", "D-", "F", "W", "P", "CR", "NC", "I"}
+
 func loadGrades(csvDir string) map[string]map[string][]int {
 
 	// MAP[SEMESTER] -> MAP[SUBJECT + NUMBER + SECTION] -> GRADE DISTRIBUTION
@@ -73,68 +75,48 @@ func csvToMap(csvFile *os.File, logFile *os.File) map[string][]int {
 	if err != nil {
 		log.Panicf("Error parsing %s: %s", csvFile.Name(), err.Error())
 	}
-	// look for the subject column and w column
-	subjectCol := -1
-	catalogNumberCol := -1
-	sectionCol := -1
-	wCol := -1
-	aPlusCol := -1
-
-	headerRow := records[0]
-
-	for j := 0; j < len(headerRow); j++ {
-		switch {
-		case headerRow[j] == "Subject":
-			subjectCol = j
-		case headerRow[j] == "Catalog Number" || headerRow[j] == "Catalog Nbr":
-			catalogNumberCol = j
-		case headerRow[j] == "Section":
-			sectionCol = j
-		case headerRow[j] == "W" || headerRow[j] == "Total W" || headerRow[j] == "W Total":
-			wCol = j
-		case headerRow[j] == "A+":
-			aPlusCol = j
-		}
-		if wCol == -1 || subjectCol == -1 || catalogNumberCol == -1 || sectionCol == -1 || aPlusCol == -1 {
-			continue
-		} else {
-			break
+
+	indexMap := make(map[string]int)
+
+	for j, col := range records[0] {
+		switch col {
+		case "Catalog Number", "Catalog Nbr":
+			indexMap["Catalog Number"] = j
+		case "W", "Total W", "W Total":
+			indexMap["W"] = j
+		default:
+			indexMap[col] = j
 		}
 	}
 
-	if wCol == -1 {
-		logFile.WriteString("could not find W column")
-		//log.Panicf("could not find W column")
-	}
-	if sectionCol == -1 {
-		logFile.WriteString("could not find Section column")
-		log.Panicf("could not find Section column")
-	}
-	if subjectCol == -1 {
-		logFile.WriteString("could not find Subject column")
-		log.Panicf("could not find Subject column")
-	}
-	if catalogNumberCol == -1 {
-		logFile.WriteString("could not find catalog # column")
-		log.Panicf("could not find catalog # column")
+	// required columns
+	for _, name := range []string{"Section", "Subject", "Catalog Number", "A+"} {
+		if _, ok := indexMap[name]; !ok {
+			fmt.Fprintf(logFile, "could not find %s column", name)
+			log.Panicf("could not find %s column", name)
+		}
 	}
-	if aPlusCol == -1 {
-		logFile.WriteString("could not find A+ column")
-		log.Panicf("could not find A+ column")
+
+	// optional columns
+	for _, name := range []string{"W", "P", "CR", "NC", "I"} {
+		if _, ok := indexMap[name]; !ok {
+			logFile.WriteString(fmt.Sprintf("could not find %s column\n", name))
+		}
 	}
 
+	sectionCol := indexMap["Section"]
+	subjectCol := indexMap["Subject"]
+	catalogNumberCol := indexMap["Catalog Number"]
+
 	distroMap := make(map[string][]int)
 
-	for _, record := range records {
+	for _, record := range records[1:] {
 		// convert grade distribution from string to int
-		intSlice := [14]int{}
-
-		for j := 0; j < 13; j++ {
-			intSlice[j], _ = strconv.Atoi(record[aPlusCol+j])
-		}
-		// add w number to the grade_distribution slice
-		if wCol != -1 {
-			intSlice[13], _ = strconv.Atoi(record[wCol])
+		intSlice := make([]int, len(grades))
+		for i, col := range grades {
+			if pos, ok := indexMap[col]; ok {
+				intSlice[i], _ = strconv.Atoi(record[pos])
+			}
 		}
 
 		// add new grade distribution to map, keyed by SUBJECT + NUMBER + SECTION

parser/professorParser.go

@@ -10,7 +10,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
-func parseProfessors(sectionId primitive.ObjectID, rowInfo map[string]*goquery.Selection, classInfo map[string]string) []primitive.ObjectID {
+func parseProfessors(sectionId primitive.ObjectID, rowInfo map[string]*goquery.Selection) []primitive.ObjectID {
 	professorText := utils.TrimWhitespace(rowInfo["Instructor(s):"].Text())
 	professorMatches := personRegexp.FindAllStringSubmatch(professorText, -1)
 	var profRefs []primitive.ObjectID = make([]primitive.ObjectID, 0, len(professorMatches))

parser/sectionParser.go

@@ -52,7 +52,7 @@ func parseSection(rowInfo map[string]*goquery.Selection, classInfo map[string]st
 		Section_number:        sectionNumber,
 		Course_reference:      courseRef.Id,
 		Academic_session:      session,
-		Professors:            parseProfessors(id, rowInfo, classInfo),
+		Professors:            parseProfessors(id, rowInfo),
 		Teaching_assistants:   getTeachingAssistants(rowInfo),
 		Internal_class_number: classNum,
 		Instruction_mode:      getInstructionMode(classInfo),

parser/validator_test.go

@@ -381,7 +381,8 @@ func testCourseReferenceFail(fail string, courseIx int, sectionIx int, t *testin
 	var sectionID, originalID primitive.ObjectID // used to store IDs of modified sections
 
 	// Build the failed section map based on fail type
-	if fail == "missing" {
+	switch fail {
+	case "missing":
 		// Misses a section
 		for i, section := range testSections {
 			if sectionIx != i {
@@ -390,7 +391,7 @@ func testCourseReferenceFail(fail string, courseIx int, sectionIx int, t *testin
 				sectionID = section.Id // Nonexistent ID referenced by course
 			}
 		}
-	} else if fail == "modified" {
+	case "modified":
 		// One section doesn't reference to correct courses
 		for i, section := range testSections {
 			sectionMap[section.Id] = section

scrapers/adacemicCalendars.go

@@ -47,7 +47,7 @@ func ScrapeAcademicCalendars(outDir string) {
 
 	// Extract data from links
 	// Current
-	academicCalendars := []AcademicCalendar{AcademicCalendar{"", "", "current"}}
+	academicCalendars := []AcademicCalendar{{"", "", "current"}}
 	chromedp.Run(chromedpCtx, chromedp.TextContent("h2.wp-block-heading", &academicCalendars[0].Title, chromedp.ByQuery))
 	var currentNode []*cdp.Node
 	chromedp.Run(chromedpCtx, chromedp.Nodes("a.wp-block-button__link", &currentNode, chromedp.ByQuery))