diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6530729..06b087b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -40,6 +40,7 @@ jobs: with: push: true tags: us-central1-docker.pkg.dev/api-tools-451421/runners/runner + target: base - name: Upload to daily dev job uses: google-github-actions/deploy-cloudrun@v2 diff --git a/Dockerfile b/Dockerfile index 7c907c9..ec0abf8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN make check RUN make build # Use a lightweight final image -FROM debian:12-slim +FROM debian:12-slim AS base WORKDIR /app # Install gcloud CLI @@ -20,12 +20,17 @@ RUN apt-get update && apt-get install -y google-cloud-sdk # Install chromium RUN apt-get update && apt-get install -y chromium -ENV CHROMIUM_BIN /usr/bin/chromium -ENV GOOGLE_CHROME_BIN /usr/bin/chromium # Also set this for compatibility +ENV CHROMIUM_BIN=/usr/bin/chromium +ENV GOOGLE_CHROME_BIN=/usr/bin/chromium # Copy build file from builder COPY --from=builder /app/api-tools /app/api-tools -COPY runners /app/runners +# Copy setup and runner scripts for entrypoint +COPY runners /app/runners RUN chmod +x /app/runners/setup.sh ENTRYPOINT ["/app/runners/setup.sh"] + +# Optional .env copy for development +FROM base AS dev +COPY .env /app/.env diff --git a/README.md b/README.md index 45e67b2..39ad786 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,13 @@ To build the project, simply clone the repository and then either: - Run `build.bat` on Windows systems (unless you want to deal with getting `make` to work on Windows :P) The build process will output an executable file named `api-tools`; this executable is the CLI and can be ran in your terminal! + +#### Docker + +Docker is used for automated running on Google Cloud Platform. More info [here](https://nebula-labs.atlassian.net/wiki/x/AYBjFw). + +To build the container for local testing first make sure all scripts in the `runners` folder have LF line endings then run: +``` +docker build --target dev -t my-runner:local . +docker run --rm -e ENVIRONMENT=dev -e RUNNER_SCRIPT_NAME=daily.sh my-runner:local +``` diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 0000000..3bc2d7d --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,5 @@ +steps: + - name: 'gcr.io/cloud-builders/docker' + args: ['build', '--target=base', '-t', 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner', '.'] +images: + - 'us-central1-docker.pkg.dev/api-tools-451421/runners/runner' diff --git a/runners/daily.sh b/runners/daily.sh index 764ee13..0a4c30a 100644 --- a/runners/daily.sh +++ b/runners/daily.sh @@ -3,8 +3,8 @@ # for daily tasks to run # scrape, parse, and upload events -./api-tools -scrape -mazevo -verbose -./api-tools -parse -mazevo -verbose -./api-tools -scrape -astra -verbose -./api-tools -parse -astra -verbose -./api-tools -upload -events -verbose +./api-tools -headless -verbose -scrape -mazevo +./api-tools -headless -verbose -parse -mazevo +./api-tools -headless -verbose -scrape -astra +./api-tools -headless -verbose -parse -astra +./api-tools -headless -verbose -upload -events diff --git a/runners/setup.sh b/runners/setup.sh index d239dca..8e83122 100644 --- a/runners/setup.sh +++ b/runners/setup.sh @@ -1,12 +1,16 @@ #!/bin/sh -# auth with service account -gcloud secrets versions access latest --secret="$SERVICE_ACCOUNT_SECRET_NAME" > service_account.json -gcloud auth activate-service-account --key-file=service_account.json -rm service_account.json +if [ "$ENVIRONMENT" = "prod" ]; then + # auth with service account + gcloud secrets versions access latest --secret="$SERVICE_ACCOUNT_SECRET_NAME" > service_account.json + gcloud auth activate-service-account --key-file=service_account.json + rm service_account.json -# use service account to access environment variables from GCP secrets, create .env -gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env + # use service account to access environment variables from GCP secrets, create .env + gcloud secrets versions access latest --secret="$ENV_SECRET_NAME" > .env +else + echo "ENVIRONMENT is set to '$ENVIRONMENT'. Skipping env setup." +fi # run commands from the file path specified in the GCP run job's variable sh "/app/runners/$RUNNER_SCRIPT_NAME"