Merge pull request #6162 from mozilla/local-dev-docker-compose

feat(local): add docker compose for easier local setup

Author: kschelonka

.docker/postgres/init-db.sql

@@ -0,0 +1,5 @@
+CREATE USER blurts WITH ENCRYPTED PASSWORD 'blurts';
+CREATE DATABASE blurts WITH OWNER 'blurts';
+CREATE DATABASE "test-blurts" WITH OWNER 'blurts';
+
+ALTER DEFAULT privileges IN SCHEMA public GRANT ALL ON tables to blurts;

.docker/pubsub/setup_pubsub.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Ensure the emulator host is set
+PUBSUB_HOST=$1
+PUBSUB_PORT=$2
+PROJECT_ID=$3
+
+PUBSUB_EMULATOR_HOST="${PUBSUB_HOST}:${PUBSUB_PORT}"
+
+# Start PubSub emulator in the background
+gcloud beta emulators pubsub start --host-port=0.0.0.0:${PUBSUB_PORT} --project=${PROJECT_ID} &
+echo "Waiting for Pub/Sub emulator to be ready..."
+until curl -s "http://${PUBSUB_EMULATOR_HOST}/v1/projects/${PROJECT_ID}/schemas" > /dev/null; do
+    echo "Pub/Sub emulator not ready yet..."
+    sleep 2
+done
+echo "Pub/Sub emulator is ready!"
+echo "Initializing Pub/Sub emulator..."
+
+create_topic_and_subscription  () {
+    if [ $# -ne 2 ]; then 
+        echo "create_topic_and_subscription takes 2 arguments"
+        exit 1
+    fi
+    if [ -z "$1" ]; then
+        echo "create_topic_and_subscription requires non-empty topic"
+        exit 1
+    fi
+    if [ -z "$2" ]; then 
+        echo "create_topic_and_subscription requires non-empty subscription"
+        exit 1
+    fi
+
+    topic=$1
+    subscription=$2
+    echo "Creating topic '$topic' with subscription '$subscription'..."
+
+    # Create the topic using REST API instead of gcloud
+    curl -s -X PUT "${PUBSUB_EMULATOR_HOST}/v1/projects/${PROJECT_ID}/topics/${topic}"
+
+    # Create the subscription using REST API instead of gcloud
+    curl -s -X PUT "${PUBSUB_EMULATOR_HOST}/v1/projects/${PROJECT_ID}/subscriptions/${subscription}" \
+        -H "Content-Type: application/json" \
+        -d "{\"topic\": \"projects/${PROJECT_ID}/topics/${topic}\"}"
+}
+
+create_topic_and_subscription hibp-breaches hibp-cron
+
+echo "Pub/Sub initialization completed."
+touch /tmp/startup.done
+
+# Keep emulator running
+wait

.env.local.example

@@ -94,4 +94,6 @@ DATA_BROKER_REMOVAL_ESTIMATES_DATA=[]
 GCP_PUBSUB_PROJECT_ID=your-project-name
 GCP_PUBSUB_TOPIC_NAME=hibp-breaches
 GCP_PUBSUB_SUBSCRIPTION_NAME=hibp-cron
-PUBSUB_EMULATOR_HOST=localhost:8085
+PUBSUB_HOST=localhost
+PUBSUB_PORT=8085
+PUBSUB_EMULATOR_HOST="${PUBSUB_HOST}:${PUBSUB_PORT}"

README.md

@@ -47,7 +47,9 @@ We track commits that are largely style/formatting via `.git-blame-ignore-revs`.
 
 ### Database
 
-To create the database tables ...
+To create the database tables you have two options: manually, or using docker-compose
+
+#### Manual setup
 
 1. Create the `blurts` database:
 
@@ -69,6 +71,28 @@ To create the database tables ...
    npm run db:migrate
    ```
 
+#### Via docker-compose
+
+This will automatically provision the databases 'blurts' and 'test-blurts', and a user 'blurts' with password 'blurts'. The connection string in your .env.local file should be: "postgres://blurts:blurts@localhost:5432/blurts"
+
+1. Ensure that you have an up-to-date .env.local file.
+
+   ```$sh
+   cp .env.local.example .env.local
+   ```
+
+2. Start docker containers (this will stand up all services defined, including pubsub)
+
+   ```sh
+   docker compose --env-file .env.local up -d
+   ```
+
+3. To tear down (this will delete stored data):
+
+   ```sh
+   docker compose --env-file .env.local down
+   ```
+
 ### Install
 
 1. Clone and change to the directory:
@@ -140,24 +164,44 @@ To create the database tables ...
 
 ### PubSub
 
-Monitor uses GCP PubSub for processing incoming breach data, this can be tested locally using an emulator: https://cloud.google.com/pubsub/docs/emulator
+Monitor uses GCP PubSub for processing incoming breach data, this can be tested locally using an emulator: <https://cloud.google.com/pubsub/docs/emulator>
+
+You can run the emulator manually or via docker-compose.
 
-#### Run the GCP PubSub emulator:
+#### Manual Setup
+
+##### Run the GCP PubSub emulator
 
 ```sh
 gcloud beta emulators pubsub start --project=your-project-name
 ```
 
 (Set `your-project-name` as the value for `GCP_PUBSUB_PROJECT_ID` in your `.env.local`.)
 
-### In a different shell, set the environment to point at the emulator and run Monitor in dev mode:
+#### Docker Compose Setup
+
+This will automatically provision a pubsub topic named 'hibp-breaches' with a subscription named 'hibp-cron'.
+
+1. Ensure that you have an up-to-date .env.local file.
+
+   ```$sh
+   cp .env.local.example .env.local
+   ```
+
+2. Start docker containers (this will stand up all services defined, including postgres)
+
+   ```sh
+   docker compose --env-file .env.local up -d
+   ```
+
+### In a different shell, set the environment to point at the emulator and run Monitor in dev mode
 
 ```sh
 $(gcloud beta emulators pubsub env-init)
 npm run dev
 ```
 
-### Incoming WebHook requests from HIBP will be of the form:
+### Incoming WebHook requests from HIBP will be of the form
 
 ```sh
 curl -d '{ "breachName": "000webhost", "hashPrefix": "test", "hashSuffixes": ["test"] }' \
@@ -168,7 +212,7 @@ curl -d '{ "breachName": "000webhost", "hashPrefix": "test", "hashSuffixes": ["t
 This emulates HIBP notifying our API that a new breach was found. Our API will
 then add it to the (emulated) pubsub queue.
 
-### This pubsub queue will be consumed by this cron job, which is responsible for looking up and emailing impacted users:
+### This pubsub queue will be consumed by this cron job, which is responsible for looking up and emailing impacted users
 
 ```sh
 NODE_ENV="development" npm run dev:cron:breach-alerts
@@ -181,7 +225,7 @@ Monitor generates multiple emails that get sent to subscribers. To preview or te
 ### Mozilla accounts ("FxA", formerly known as Firefox accounts)
 
 The repo comes with a development FxA oauth app pre-configured in `.env`, which
-should work fine running the app on http://localhost:6060. You'll need to get
+should work fine running the app on <http://localhost:6060>. You'll need to get
 the `OAUTH_CLIENT_SECRET` value from a team member or someone in #fxmonitor-engineering.
 
 ## Testing
@@ -216,7 +260,7 @@ To test this part of Monitor:
 
 k6 is used for load testing.
 
-See https://grafana.com/docs/k6/latest/get-started/running-k6/ for more information.
+See <https://grafana.com/docs/k6/latest/get-started/running-k6/> for more information.
 
 ##### HIBP breach alerts
 

docker-compose.yml

@@ -0,0 +1,46 @@
+services:
+  postgres:
+    image: postgres:latest
+    ports:
+      - "5432:5432"
+    volumes:
+      - "./.docker/postgres:/docker-entrypoint-initdb.d:delegated,z"
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_USER: postgres
+      PGUSER: postgres
+      POSTGRES_DB: postgres
+    command: ["-c", "log_statement=all"]
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 1s
+      timeout: 5s
+      retries: 10
+      start_period: 80s
+    networks:
+      - shared
+
+  pubsub:
+    image: google/cloud-sdk:529.0.0
+    platform: linux/amd64
+    entrypoint: ["/bin/bash", "-c"]
+    command:
+      - |
+        /pubsub-init/setup_pubsub.sh ${PUBSUB_HOST} ${PUBSUB_PORT} ${GCP_PUBSUB_PROJECT_ID}
+    ports:
+      - "${PUBSUB_PORT}:${PUBSUB_PORT}"
+    volumes:
+      - "./.docker/pubsub:/pubsub-init:delegated,z"
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/startup.done"]
+      interval: 15s
+      timeout: 10s
+      retries: 20
+      start_period: 30s
+      start_interval: 10s
+    networks:
+      - shared
+
+networks:
+  shared:
+    name: shared