Run api-tools daily on google cloud (#54)

* Docker stuff

* GitHub Action

* Quotes

* is the secret there?

* deployment env

* Revert "is the secret there?"

This reverts commit 086d3f3.

* use federated thing

* permissions

* provider

* github/actions-oidc-debugger

* Revert "github/actions-oidc-debugger"

This reverts commit 12cfd92.

* Use gcloud CLI

* build-push-action

* docker/login-action@

* id auth

* token_format: access_token

* google-github-actions/deploy-cloudrun

* service -> job

* bump versions

* region

* scrape mazevo too

* only on develop

* Revert "only on develop"

This reverts commit 65a85db.

* big region change

* remove exec

* Reapply "only on develop"

This reverts commit eab3852.

* mazevo indentation

* print when done in mazevo and astra scrapers for logs

* detect empty env values

* Use a service account to auth the env

* Revert "Reapply "only on develop""

This reverts commit aa4309d.

* Reapply "Reapply "only on develop""

This reverts commit 6ca207e.

---------

Co-authored-by: Josh <[email protected]>

Author: TyHil

.github/workflows/deploy.yml

@@ -0,0 +1,55 @@
+# This workflow deploys the repository to run daily on google cloud run
+
+name: Deploy
+
+on:
+  push:
+    branches: develop
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    environment: deployment
+
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+    - id: auth
+      name: Google Cloud Auth
+      uses: google-github-actions/auth@v2
+      with:
+        token_format: access_token
+        workload_identity_provider: projects/762526944259/locations/global/workloadIdentityPools/pool/providers/github-deploy
+        service_account: '[email protected]'
+
+    - name: Set up Google Cloud
+      uses: google-github-actions/setup-gcloud@v2
+
+    - name: Login to Artifact Registry
+      uses: docker/login-action@v3
+      with:
+        registry: us-central1-docker.pkg.dev
+        username: oauth2accesstoken
+        password: ${{ steps.auth.outputs.access_token }}
+
+    - name: Build Docker image
+      uses: docker/build-push-action@v6
+      with:
+        push: true 
+        tags: us-central1-docker.pkg.dev/api-tools-451421/runners/daily-update-events
+
+    - name: Upload to dev job
+      uses: google-github-actions/deploy-cloudrun@v2
+      with:
+        job: 'daily-update-events-dev'
+        image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily-update-events'
+        region: us-central1
+
+    - name: Upload to prod job
+      uses: google-github-actions/deploy-cloudrun@v2
+      with:
+        job: 'daily-update-events-prod'
+        image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily-update-events'
+        region: us-central1

.github/workflows/go.yml

@@ -5,9 +5,9 @@ name: Go
 
 on:
   push:
-    branches: [ "develop" ]
+    branches: develop
   pull_request:
-    branches: [ "develop" ]
+    branches: develop
   workflow_dispatch:
 
 jobs:
@@ -17,10 +17,10 @@ jobs:
     steps:
 
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: '1.23'
 

Dockerfile

@@ -0,0 +1,31 @@
+FROM golang:1.23 AS builder
+
+WORKDIR /app
+COPY . .
+
+# Run setup and checks
+RUN make setup
+RUN make check
+RUN make build
+
+# Use a lightweight final image
+FROM debian:12-slim
+WORKDIR /app
+
+# Install gcloud CLI
+RUN apt-get update && apt-get install -y wget gnupg apt-transport-https lsb-release ca-certificates
+RUN wget -qO - https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor > /usr/share/keyrings/cloud.google.gpg
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN apt-get update && apt-get install -y google-cloud-sdk
+
+# Install chromium
+RUN apt-get update && apt-get install -y chromium
+ENV CHROMIUM_BIN /usr/bin/chromium
+ENV GOOGLE_CHROME_BIN /usr/bin/chromium # Also set this for compatibility
+
+# Copy build file from builder
+COPY --from=builder /app/api-tools /app/api-tools
+COPY deploy.sh /app/deploy.sh
+
+RUN chmod +x /app/deploy.sh
+ENTRYPOINT ["/app/deploy.sh"]

Makefile

@@ -9,8 +9,8 @@ check:
 	go mod tidy
 	go vet ./... 
 	staticcheck ./...
-	gofmt -w ./..
-	goimports -w ./..
+	gofmt -w .
+	goimports -w .
 
 build: ./main.go
 	go build -o $(EXEC_NAME) ./main.go

deploy.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# service account
+gcloud secrets versions access latest --secret="$SERVICE_ACCOUNT_SECRET_NAME" > service_account.json
+gcloud auth activate-service-account --key-file=service_account.json
+rm service_account.json
+
+# .env
+gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env
+
+# Scrape, parse, and upload
+./api-tools -scrape -mazevo -verbose
+#./api-tools -parse -mazevo -verbose
+./api-tools -scrape -astra -verbose
+#./api-tools -parse -astra -verbose
+#./api-tools -upload -events -verbose

scrapers/astra.go

@@ -102,6 +102,8 @@ func ScrapeAstra(outDir string) {
 		date = date.Add(time.Hour * 24)
 	}
 
+	log.Printf("Scraped Astra up to %s!", date.Format("2006-01-02"))
+
 	// Write event data to output file
 	days = fmt.Sprintf("%s}", days)
 	fptr, err := os.Create(fmt.Sprintf("%s/reservations.json", outDir))

scrapers/mazevo.go

@@ -43,38 +43,37 @@ func ScrapeMazevo(outDir string) {
 	endDate := date.Add(time.Hour * 24 * 365).Format(time.RFC3339)
 
 	// Request events
-	stringBody := ""
-	{
-		url := "https://east.mymazevo.com/api/PublicCalendar/GetCalendarEvents"
-		requestBodyMap := map[string]string{
-			"apiKey": apikey,
-			"end":    endDate,
-			"start":  startDate,
-		}
-		requestBodyBytes, _ := json.Marshal(requestBodyMap)
-		requestBody := bytes.NewBuffer(requestBodyBytes)
-		req, err := http.NewRequest("POST", url, requestBody)
-		if err != nil {
-			panic(err)
-		}
-		req.Header = http.Header{
-			"Content-type": {"application/json"},
-			"Accept":       {"application/json"},
-		}
-		res, err := cli.Do(req)
-		if err != nil {
-			panic(err)
-		}
-		if res.StatusCode != 200 {
-			log.Panicf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status)
-		}
-		body, err := io.ReadAll(res.Body)
-		if err != nil {
-			panic(err)
-		}
-		res.Body.Close()
-		stringBody = string(body)
+	url := "https://east.mymazevo.com/api/PublicCalendar/GetCalendarEvents"
+	requestBodyMap := map[string]string{
+		"apiKey": apikey,
+		"end":    endDate,
+		"start":  startDate,
 	}
+	requestBodyBytes, _ := json.Marshal(requestBodyMap)
+	requestBody := bytes.NewBuffer(requestBodyBytes)
+	req, err := http.NewRequest("POST", url, requestBody)
+	if err != nil {
+		panic(err)
+	}
+	req.Header = http.Header{
+		"Content-type": {"application/json"},
+		"Accept":       {"application/json"},
+	}
+	res, err := cli.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	if res.StatusCode != 200 {
+		log.Panicf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status)
+	}
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		panic(err)
+	}
+	res.Body.Close()
+	stringBody := string(body)
+
+	log.Printf("Scraped Mazevo up to %s!", endDate)
 
 	// Write event data to output file
 	fptr, err := os.Create(fmt.Sprintf("%s/mazevoReservations.json", outDir))

utils/methods.go

@@ -28,7 +28,7 @@ var Headless = true
 // Finds .env value and produces proper error if not found
 func GetEnv(name string) (string, error) {
 	value, exists := os.LookupEnv(name)
-	if !exists {
+	if !exists || value == "" {
 		return "", errors.New(name + " is missing from .env!")
 	}
 	return value, nil
@@ -152,7 +152,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
 			for _, cookie := range cookies {
 				cookieStr = fmt.Sprintf("%s%s=%s; ", cookieStr, cookie.Name, cookie.Value)
 				if cookie.Name == "UTXDallas.ASPXFORMSAUTH" {
-					VPrintf("Got new token: PTGSESSID = %s", cookie.Value)
+					VPrintf("Got new token: UTXDallas.ASPXFORMSAUTH = %s", cookie.Value)
 					gotToken = true
 				}
 			}