Merge pull request #88 from UTDNebula/academic-calendar-scraper

Academic calendar scraper, parser, and uploader

Author: TyHil

.env.template

@@ -5,6 +5,15 @@ LOGIN_ASTRA_USERNAME=
 LOGIN_ASTRA_PASSWORD=
 #Login to https://east.mymazevo.com/main-home then go to https://east.mymazevo.com/api/tenantsettings/GetApiKey
 MAZEVO_API_KEY=
+#Academic Calendars
+GOOGLE_GENAI_USE_VERTEXAI=
+GOOGLE_CLOUD_PROJECT=
+GOOGLE_APPLICATION_CREDENTIALS=
+GEMINI_SERVICE_ACCOUNT=
+NEBULA_API_URL=
+NEBULA_API_STORAGE_BUCKET=
+NEBULA_API_KEY=
+NEBULA_API_STORAGE_KEY=
 
 # Uploader
 MONGODB_URI=

.github/workflows/deploy.yml

@@ -55,6 +55,20 @@ jobs:
         job: 'daily-prod'
         image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner'
         region: us-central1
+
+    - name: Upload to weekly dev job
+      uses: google-github-actions/deploy-cloudrun@v2
+      with:
+        job: 'weekly-dev'
+        image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner'
+        region: us-central1
+
+    - name: Upload to weekly prod job
+      uses: google-github-actions/deploy-cloudrun@v2
+      with:
+        job: 'weekly-prod'
+        image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner'
+        region: us-central1
 
     - name: Upload to monthly dev job
       uses: google-github-actions/deploy-cloudrun@v2

.gitignore

@@ -30,6 +30,7 @@ yarn-error.log*
 *.env.development.local
 *.env.test.local
 *.env
+*service-account.json
 
 # vercel
 .vercel

go.mod

@@ -9,9 +9,11 @@ require (
 	github.com/chromedp/chromedp v0.12.1
 	github.com/google/go-cmp v0.7.0
 	github.com/joho/godotenv v1.5.1
+	github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80
 	github.com/valyala/fastjson v1.6.4
 	go.mongodb.org/mongo-driver v1.17.3
 	golang.org/x/net v0.43.0
+	google.golang.org/genai v1.32.0
 )
 
 require (
@@ -54,6 +56,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.5 // indirect
 	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.8 // indirect

go.sum

@@ -109,6 +109,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.5 h1:VgzTY2jogw3xt39CusE
 github.com/googleapis/enterprise-certificate-proxy v0.3.5/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
 github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q=
 github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -243,6 +245,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.224.0 h1:Ir4UPtDsNiwIOHdExr3fAj4xZ42QjK7uQte3lORLJwU=
 google.golang.org/api v0.224.0/go.mod h1:3V39my2xAGkodXy0vEqcEtkqgw2GtrFL5WuBZlCTCOQ=
+google.golang.org/genai v1.32.0 h1:kku/m3kWOncjnw8EIa2sgmrPLhaxFHaP+uqOq5ZckvI=
+google.golang.org/genai v1.32.0/go.mod h1:7pAilaICJlQBonjKKJNhftDFv3SREhZcTe9F6nRcjbg=
 google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE=
 google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE=
 google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950=

main.go

@@ -46,6 +46,8 @@ func main() {
 	mazevo := flag.Bool("mazevo", false, "Alongside -scrape or -parse, signifies that Mazevo should be scraped/parsed.")
 	// Flag for map scraping, parsing, and uploading
 	mapFlag := flag.Bool("map", false, "Alongside -scrape, -parse, or -upload, signifies that the UTD map should be scraped/parsed/uploaded.")
+	// Flag for academic calendar scraping
+	academicCalendars := flag.Bool("academicCalendars", false, "Alongside -scrape, -parse, or -upload, signifies that the academic calendars should be scraped/parsed/uploaded.")
 
 	// Flags for parsing
 	parse := flag.Bool("parse", false, "Puts the tool into parsing mode.")
@@ -114,6 +116,8 @@ func main() {
 			scrapers.ScrapeMazevo(*outDir)
 		case *mapFlag:
 			scrapers.ScrapeMapLocations(*outDir)
+		case *academicCalendars:
+			scrapers.ScrapeAcademicCalendars(*outDir)
 		default:
 			log.Panic("You must specify which type of scraping you would like to perform with one of the scraping flags!")
 		}
@@ -127,6 +131,8 @@ func main() {
 			parser.ParseMazevo(*inDir, *outDir)
 		case *mapFlag:
 			parser.ParseMapLocations(*inDir, *outDir)
+		case *academicCalendars:
+			parser.ParseAcademicCalendars(*inDir, *outDir)
 		default:
 			parser.Parse(*inDir, *outDir, *csvDir, *skipValidation)
 		}
@@ -136,6 +142,8 @@ func main() {
 			uploader.UploadEvents(*inDir)
 		case *mapFlag:
 			uploader.UploadMapLocations(*inDir)
+		case *academicCalendars:
+			uploader.UploadAcademicCalendars(*inDir)
 		default:
 			uploader.Upload(*inDir, *replace, *staticOnly)
 		}