Merge from develop again

Author: mikehquan19

.env.template

@@ -5,6 +5,15 @@ LOGIN_ASTRA_USERNAME=
 LOGIN_ASTRA_PASSWORD=
 #Login to https://east.mymazevo.com/main-home then go to https://east.mymazevo.com/api/tenantsettings/GetApiKey
 MAZEVO_API_KEY=
+#Academic Calendars
+GOOGLE_GENAI_USE_VERTEXAI=
+GOOGLE_CLOUD_PROJECT=
+GOOGLE_APPLICATION_CREDENTIALS=
+GEMINI_SERVICE_ACCOUNT=
+NEBULA_API_URL=
+NEBULA_API_STORAGE_BUCKET=
+NEBULA_API_KEY=
+NEBULA_API_STORAGE_KEY=
 
 # Uploader
 MONGODB_URI=

.github/workflows/deploy.yml

@@ -55,6 +55,20 @@ jobs:
         job: 'daily-prod'
         image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner'
         region: us-central1
+
+    - name: Upload to weekly dev job
+      uses: google-github-actions/deploy-cloudrun@v2
+      with:
+        job: 'weekly-dev'
+        image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner'
+        region: us-central1
+
+    - name: Upload to weekly prod job
+      uses: google-github-actions/deploy-cloudrun@v2
+      with:
+        job: 'weekly-prod'
+        image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner'
+        region: us-central1
 
     - name: Upload to monthly dev job
       uses: google-github-actions/deploy-cloudrun@v2

.gitignore

@@ -30,6 +30,7 @@ yarn-error.log*
 *.env.development.local
 *.env.test.local
 *.env
+*service-account.json
 
 # vercel
 .vercel

Dockerfile

@@ -23,6 +23,9 @@ RUN apt-get update && apt-get install -y chromium
 ENV CHROMIUM_BIN=/usr/bin/chromium
 ENV GOOGLE_CHROME_BIN=/usr/bin/chromium
 
+# Install poppler-utils for pdftotext for academic calendar parsing
+RUN apt-get install -y poppler-utils
+
 # Copy build file from builder
 COPY --from=builder /app/api-tools /app/api-tools
 
@@ -32,5 +35,5 @@ RUN chmod +x /app/runners/setup.sh
 ENTRYPOINT ["/app/runners/setup.sh"]
 
 # Optional .env copy for development
-FROM base AS dev
+FROM base AS local
 COPY .env /app/.env

README.md

@@ -98,8 +98,8 @@ Docker is used for automated running on Google Cloud Platform. More info [here](
 
 To build the container for local testing first make sure all scripts in the `runners` folder have LF line endings then run:
 ```
-docker build --target dev -t my-runner:local .
-docker run --rm -e ENVIRONMENT=dev -e RUNNER_SCRIPT_NAME=daily.sh my-runner:local
+docker build --target local -t my-runner:local .
+docker run --rm -e ENVIRONMENT=local -e RUNNER_SCRIPT_NAME=daily.sh my-runner:local
 ```
 
 ## Questions?

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/valyala/fastjson v1.6.4
 	go.mongodb.org/mongo-driver v1.17.3
 	golang.org/x/net v0.43.0
+	google.golang.org/genai v1.32.0
 )
 
 require (
@@ -54,6 +55,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.5 // indirect
 	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.8 // indirect

go.sum

@@ -109,6 +109,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.5 h1:VgzTY2jogw3xt39CusE
 github.com/googleapis/enterprise-certificate-proxy v0.3.5/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
 github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q=
 github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -243,6 +245,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.224.0 h1:Ir4UPtDsNiwIOHdExr3fAj4xZ42QjK7uQte3lORLJwU=
 google.golang.org/api v0.224.0/go.mod h1:3V39my2xAGkodXy0vEqcEtkqgw2GtrFL5WuBZlCTCOQ=
+google.golang.org/genai v1.32.0 h1:kku/m3kWOncjnw8EIa2sgmrPLhaxFHaP+uqOq5ZckvI=
+google.golang.org/genai v1.32.0/go.mod h1:7pAilaICJlQBonjKKJNhftDFv3SREhZcTe9F6nRcjbg=
 google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE=
 google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE=
 google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950=

main.go

@@ -39,13 +39,15 @@ func main() {
 	// Flag for soc scraping
 	scrapeOrganizations := flag.Bool("organizations", false, "Alongside -scrape, signifies that SOC organizations should be scraped.")
 	// Flag for calendar scraping and parsing
-	calendar := flag.Bool("calendar", false, "Alongside -scrape or -parse, signifies that calendar should be scraped.")
+	cometCalendar := flag.Bool("cometCalendar", false, "Alongside -scrape or -parse, signifies that the Comet Calendar should be scraped/parsed.")
 	// Flag for astra scraping and parsing
 	astra := flag.Bool("astra", false, "Alongside -scrape or -parse, signifies that Astra should be scraped/parsed.")
 	// Flag for mazevo scraping and parsing
 	mazevo := flag.Bool("mazevo", false, "Alongside -scrape or -parse, signifies that Mazevo should be scraped/parsed.")
 	// Flag for map scraping, parsing, and uploading
 	mapFlag := flag.Bool("map", false, "Alongside -scrape, -parse, or -upload, signifies that the UTD map should be scraped/parsed/uploaded.")
+	// Flag for academic calendar scraping
+	academicCalendars := flag.Bool("academicCalendars", false, "Alongside -scrape, -parse, or -upload, signifies that the academic calendars should be scraped/parsed/uploaded.")
 
 	// Flags for parsing
 	parse := flag.Bool("parse", false, "Puts the tool into parsing mode.")
@@ -56,7 +58,7 @@ func main() {
 	upload := flag.Bool("upload", false, "Puts the tool into upload mode.")
 	replace := flag.Bool("replace", false, "Alongside -upload, specifies that uploaded data should replace existing data rather than being merged.")
 	staticOnly := flag.Bool("static", false, "Alongside -upload, specifies that we should only build and upload the static aggregations.")
-	events := flag.Bool("events", false, "Alongside -upload, signifies that Astra and Mazevo should be uploaded.")
+	events := flag.Bool("events", false, "Alongside -upload, signifies that Astra, Mazevo, and the Comet Calendar should be uploaded.")
 
 	// Flags for logging
 	verbose := flag.Bool("verbose", false, "Enables verbose logging, good for debugging purposes.")
@@ -106,27 +108,31 @@ func main() {
 			scrapers.ScrapeCoursebook(*term, *startPrefix, *outDir, *resume)
 		case *scrapeOrganizations:
 			scrapers.ScrapeOrganizations(*outDir)
-		case *calendar:
-			scrapers.ScrapeCalendar(*outDir)
+		case *cometCalendar:
+			scrapers.ScrapeCometCalendar(*outDir)
 		case *astra:
 			scrapers.ScrapeAstra(*outDir)
 		case *mazevo:
 			scrapers.ScrapeMazevo(*outDir)
 		case *mapFlag:
 			scrapers.ScrapeMapLocations(*outDir)
+		case *academicCalendars:
+			scrapers.ScrapeAcademicCalendars(*outDir)
 		default:
 			log.Panic("You must specify which type of scraping you would like to perform with one of the scraping flags!")
 		}
 	case *parse:
 		switch {
-		case *calendar:
-			parser.ParseCalendar(*inDir, *outDir)
+		case *cometCalendar:
+			parser.ParseCometCalendar(*inDir, *outDir)
 		case *astra:
 			parser.ParseAstra(*inDir, *outDir)
 		case *mazevo:
 			parser.ParseMazevo(*inDir, *outDir)
 		case *mapFlag:
 			parser.ParseMapLocations(*inDir, *outDir)
+		case *academicCalendars:
+			parser.ParseAcademicCalendars(*inDir, *outDir)
 		default:
 			parser.Parse(*inDir, *outDir, *csvDir, *skipValidation)
 		}
@@ -136,6 +142,8 @@ func main() {
 			uploader.UploadEvents(*inDir)
 		case *mapFlag:
 			uploader.UploadMapLocations(*inDir)
+		case *academicCalendars:
+			uploader.UploadAcademicCalendars(*inDir)
 		default:
 			uploader.Upload(*inDir, *replace, *staticOnly)
 		}