diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..833503d --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,55 @@ +# This workflow deploys the repository to run daily on google cloud run + +name: Deploy + +on: + push: + branches: develop + +jobs: + main: + runs-on: ubuntu-latest + environment: deployment + + permissions: + contents: 'read' + id-token: 'write' + + steps: + - id: auth + name: Google Cloud Auth + uses: google-github-actions/auth@v2 + with: + token_format: access_token + workload_identity_provider: projects/762526944259/locations/global/workloadIdentityPools/pool/providers/github-deploy + service_account: 'github-deploy@api-tools-451421.iam.gserviceaccount.com' + + - name: Set up Google Cloud + uses: google-github-actions/setup-gcloud@v2 + + - name: Login to Artifact Registry + uses: docker/login-action@v3 + with: + registry: us-central1-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Build Docker image + uses: docker/build-push-action@v6 + with: + push: true + tags: us-central1-docker.pkg.dev/api-tools-451421/runners/daily-update-events + + - name: Upload to dev job + uses: google-github-actions/deploy-cloudrun@v2 + with: + job: 'daily-update-events-dev' + image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily-update-events' + region: us-central1 + + - name: Upload to prod job + uses: google-github-actions/deploy-cloudrun@v2 + with: + job: 'daily-update-events-prod' + image: 'us-central1-docker.pkg.dev/api-tools-451421/runners/daily-update-events' + region: us-central1 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index bf898c8..39ef559 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -5,9 +5,9 @@ name: Go on: push: - branches: [ "develop" ] + branches: develop pull_request: - branches: [ "develop" ] + branches: develop workflow_dispatch: jobs: @@ -17,10 +17,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '1.23' diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a4bdc4c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM golang:1.23 AS builder + +WORKDIR /app +COPY . . + +# Run setup and checks +RUN make setup +RUN make check +RUN make build + +# Use a lightweight final image +FROM debian:12-slim +WORKDIR /app + +# Install gcloud CLI +RUN apt-get update && apt-get install -y wget gnupg apt-transport-https lsb-release ca-certificates +RUN wget -qO - https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor > /usr/share/keyrings/cloud.google.gpg +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +RUN apt-get update && apt-get install -y google-cloud-sdk + +# Install chromium +RUN apt-get update && apt-get install -y chromium +ENV CHROMIUM_BIN /usr/bin/chromium +ENV GOOGLE_CHROME_BIN /usr/bin/chromium # Also set this for compatibility + +# Copy build file from builder +COPY --from=builder /app/api-tools /app/api-tools +COPY deploy.sh /app/deploy.sh + +RUN chmod +x /app/deploy.sh +ENTRYPOINT ["/app/deploy.sh"] diff --git a/Makefile b/Makefile index 4b66cd2..9903b9f 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,8 @@ check: go mod tidy go vet ./... staticcheck ./... - gofmt -w ./.. - goimports -w ./.. + gofmt -w . + goimports -w . build: ./main.go go build -o $(EXEC_NAME) ./main.go diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..75aad83 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# service account +gcloud secrets versions access latest --secret="$SERVICE_ACCOUNT_SECRET_NAME" > service_account.json +gcloud auth activate-service-account --key-file=service_account.json +rm service_account.json + +# .env +gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env + +# Scrape, parse, and upload +./api-tools -scrape -mazevo -verbose +#./api-tools -parse -mazevo -verbose +./api-tools -scrape -astra -verbose +#./api-tools -parse -astra -verbose +#./api-tools -upload -events -verbose diff --git a/scrapers/astra.go b/scrapers/astra.go index 4a8626e..24bf571 100644 --- a/scrapers/astra.go +++ b/scrapers/astra.go @@ -102,6 +102,8 @@ func ScrapeAstra(outDir string) { date = date.Add(time.Hour * 24) } + log.Printf("Scraped Astra up to %s!", date.Format("2006-01-02")) + // Write event data to output file days = fmt.Sprintf("%s}", days) fptr, err := os.Create(fmt.Sprintf("%s/reservations.json", outDir)) diff --git a/scrapers/mazevo.go b/scrapers/mazevo.go index 187a55a..553caee 100644 --- a/scrapers/mazevo.go +++ b/scrapers/mazevo.go @@ -43,38 +43,37 @@ func ScrapeMazevo(outDir string) { endDate := date.Add(time.Hour * 24 * 365).Format(time.RFC3339) // Request events - stringBody := "" - { - url := "https://east.mymazevo.com/api/PublicCalendar/GetCalendarEvents" - requestBodyMap := map[string]string{ - "apiKey": apikey, - "end": endDate, - "start": startDate, - } - requestBodyBytes, _ := json.Marshal(requestBodyMap) - requestBody := bytes.NewBuffer(requestBodyBytes) - req, err := http.NewRequest("POST", url, requestBody) - if err != nil { - panic(err) - } - req.Header = http.Header{ - "Content-type": {"application/json"}, - "Accept": {"application/json"}, - } - res, err := cli.Do(req) - if err != nil { - panic(err) - } - if res.StatusCode != 200 { - log.Panicf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status) - } - body, err := io.ReadAll(res.Body) - if err != nil { - panic(err) - } - res.Body.Close() - stringBody = string(body) + url := "https://east.mymazevo.com/api/PublicCalendar/GetCalendarEvents" + requestBodyMap := map[string]string{ + "apiKey": apikey, + "end": endDate, + "start": startDate, } + requestBodyBytes, _ := json.Marshal(requestBodyMap) + requestBody := bytes.NewBuffer(requestBodyBytes) + req, err := http.NewRequest("POST", url, requestBody) + if err != nil { + panic(err) + } + req.Header = http.Header{ + "Content-type": {"application/json"}, + "Accept": {"application/json"}, + } + res, err := cli.Do(req) + if err != nil { + panic(err) + } + if res.StatusCode != 200 { + log.Panicf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status) + } + body, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + res.Body.Close() + stringBody := string(body) + + log.Printf("Scraped Mazevo up to %s!", endDate) // Write event data to output file fptr, err := os.Create(fmt.Sprintf("%s/mazevoReservations.json", outDir)) diff --git a/utils/methods.go b/utils/methods.go index 5bf5e78..9c97324 100644 --- a/utils/methods.go +++ b/utils/methods.go @@ -28,7 +28,7 @@ var Headless = true // Finds .env value and produces proper error if not found func GetEnv(name string) (string, error) { value, exists := os.LookupEnv(name) - if !exists { + if !exists || value == "" { return "", errors.New(name + " is missing from .env!") } return value, nil @@ -152,7 +152,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string { for _, cookie := range cookies { cookieStr = fmt.Sprintf("%s%s=%s; ", cookieStr, cookie.Name, cookie.Value) if cookie.Name == "UTXDallas.ASPXFORMSAUTH" { - VPrintf("Got new token: PTGSESSID = %s", cookie.Value) + VPrintf("Got new token: UTXDallas.ASPXFORMSAUTH = %s", cookie.Value) gotToken = true } }