Merge pull request #75 from UTDNebula/fix-daily-runner

Add headless flag and documentation

Author: mikehquan19

.github/workflows/deploy.yml

@@ -40,6 +40,7 @@ jobs:
       with:
         push: true 
         tags: us-central1-docker.pkg.dev/api-tools-451421/runners/runner
+        target: base
 
     - name: Upload to daily dev job
       uses: google-github-actions/deploy-cloudrun@v2

Dockerfile

@@ -9,7 +9,7 @@ RUN make check
 RUN make build
 
 # Use a lightweight final image
-FROM debian:12-slim
+FROM debian:12-slim AS base
 WORKDIR /app
 
 # Install gcloud CLI
@@ -20,12 +20,17 @@ RUN apt-get update && apt-get install -y google-cloud-sdk
 
 # Install chromium
 RUN apt-get update && apt-get install -y chromium
-ENV CHROMIUM_BIN /usr/bin/chromium
-ENV GOOGLE_CHROME_BIN /usr/bin/chromium # Also set this for compatibility
+ENV CHROMIUM_BIN=/usr/bin/chromium
+ENV GOOGLE_CHROME_BIN=/usr/bin/chromium
 
 # Copy build file from builder
 COPY --from=builder /app/api-tools /app/api-tools
-COPY runners /app/runners
 
+# Copy setup and runner scripts for entrypoint
+COPY runners /app/runners
 RUN chmod +x /app/runners/setup.sh
 ENTRYPOINT ["/app/runners/setup.sh"]
+
+# Optional .env copy for development
+FROM base AS dev
+COPY .env /app/.env

README.md

@@ -35,3 +35,13 @@ To build the project, simply clone the repository and then either:
   - Run `build.bat` on Windows systems (unless you want to deal with getting `make` to work on Windows :P)
 
 The build process will output an executable file named `api-tools`; this executable is the CLI and can be ran in your terminal!
+
+#### Docker
+
+Docker is used for automated running on Google Cloud Platform. More info [here](https://nebula-labs.atlassian.net/wiki/x/AYBjFw).
+
+To build the container for local testing first make sure all scripts in the `runners` folder have LF line endings then run:
+```
+docker build --target dev -t my-runner:local .
+docker run --rm -e ENVIRONMENT=dev -e RUNNER_SCRIPT_NAME=daily.sh my-runner:local
+```

cloudbuild.yaml

@@ -0,0 +1,5 @@
+steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['build', '--target=base', '-t', 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner', '.']
+images:
+  - 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner'

runners/daily.sh

@@ -3,8 +3,8 @@
 # for daily tasks to run
 
 # scrape, parse, and upload events
-./api-tools -scrape -mazevo -verbose
-./api-tools -parse -mazevo -verbose
-./api-tools -scrape -astra -verbose
-./api-tools -parse -astra -verbose
-./api-tools -upload -events -verbose
+./api-tools -headless -verbose -scrape -mazevo
+./api-tools -headless -verbose -parse -mazevo
+./api-tools -headless -verbose -scrape -astra
+./api-tools -headless -verbose -parse -astra
+./api-tools -headless -verbose -upload -events

runners/setup.sh

@@ -1,12 +1,16 @@
 #!/bin/sh
 
-# auth with service account
-gcloud secrets versions access latest --secret="$SERVICE_ACCOUNT_SECRET_NAME" > service_account.json
-gcloud auth activate-service-account --key-file=service_account.json
-rm service_account.json
+if [ "$ENVIRONMENT" = "prod" ]; then
+  # auth with service account
+  gcloud secrets versions access latest --secret="$SERVICE_ACCOUNT_SECRET_NAME" > service_account.json
+  gcloud auth activate-service-account --key-file=service_account.json
+  rm service_account.json
 
-# use service account to access environment variables from GCP secrets, create .env
-gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env
+  # use service account to access environment variables from GCP secrets, create .env
+  gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env
+else
+  echo "ENVIRONMENT is set to '$ENVIRONMENT'. Skipping env setup."
+fi
 
 # run commands from the file path specified in the GCP run job's variable
 sh "/app/runners/$RUNNER_SCRIPT_NAME"