Easier developer workflow

.github/workflows/ci.yaml

@@ -229,6 +229,8 @@ jobs:
           docker image ls -a
 
       - name: Run transcript tests
+        env:
+          SHARE_PROJECT_ROOT: ${{ github.workspace }}
         # If it takes longer than this, it's probably stalled out.
         timeout-minutes: 10
         run: |
@@ -239,12 +241,11 @@ jobs:
           curl -L https://github.com/unisonweb/unison/releases/download/release%2F0.5.44/ucm-linux-x64.tar.gz | tar -xz -C ucm
           export PATH=$PWD/ucm:$PATH
 
-          # Clean up old postgres data if it exists.
-          docker volume rm docker_postgresVolume 2>/dev/null || true
-
           # Start share and it's dependencies in the background
-          docker compose -f docker/docker-compose.yml up --wait
+          docker compose -f docker/docker-compose.base.yml -f docker/docker-compose.share.yml up --wait
 
+          # Configure it as a transcript database
+          ./transcripts/configure_transcript_database.zsh
 
           # Run the transcript tests
           zsh ./transcripts/run-transcripts.zsh

Makefile

@@ -1,5 +1,7 @@
-.PHONY: all clean install docker_server_build docker_push serve auth_example transcripts fixtures transcripts
+.PHONY: all clean install docker_server_build docker_push serve auth_example transcripts fixtures transcripts reset_fixtures
 
+SHARE_PROJECT_ROOT := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+export SHARE_PROJECT_ROOT
 UNAME := $(shell uname)
 STACK_FLAGS := "--fast"
 dist_dir := $(shell stack path | awk '/^dist-dir/{print $$2}')
@@ -36,39 +38,66 @@ $(installed_share): $(exe) $(target_dir)
 auth_example:
 	stack build --fast test-auth-app
 
-docker_server_build: $(installed_share)
-	docker build $(docker_platform_flag) -f docker/Dockerfile --build-arg share_commit=$(share_commit) -t share docker
-
-docker_server_release: $(installed_share)
-	docker build $(docker_platform_flag) -f docker/Dockerfile -t $(docker_registry)/share:$(DRONE_BUILD_NUMBER) docker
-
-docker_push: $(docker_server_release)
-	docker push $(docker_registry)/share:$(DRONE_BUILD_NUMBER)
-
-docker_staging_release: $(installed_share)
-	docker build $(docker_platform_flag) -f docker/Dockerfile -t $(docker_registry)/share-staging:$(DRONE_BUILD_NUMBER) docker
-
-docker_staging_push: $(docker_server_release)
-	docker push $(docker_registry)/share-staging:$(DRONE_BUILD_NUMBER)
-
+# Build Share and run it alongside its dependencies via docker-compose
 serve: $(installed_share)
-	trap 'docker compose -f docker/docker-compose.yml down' EXIT INT TERM
-	docker compose -f docker/docker-compose.yml up postgres redis vault &
-	while  ! (  pg_isready --host localhost -U postgres -p 5432 && redis-cli -p 6379 ping && VAULT_ADDR=http://localhost:8200 vault status)  do \
-		echo "Waiting for postgres and redis..."; \
+	@docker compose -f docker/docker-compose.base.yml down || true
+	@trap 'docker compose -f docker/docker-compose.base.yml down' EXIT INT TERM
+	@echo "Booting up docker dependencies..."
+	docker compose -f docker/docker-compose.base.yml -f docker/docker-compose.fixtures.yml up  --remove-orphans --detach
+	@echo "Booting up docker dependencies...";
+	@while  ! (  pg_isready --host localhost -U postgres -p 5432 >/dev/null 2>&1 && redis-cli -p 6379 ping  >/dev/null 2>&1 && VAULT_ADDR=http://localhost:8200 vault status >/dev/null 2>&1 )  do \
 	  sleep 1; \
 	done;
-	echo "Running Share at http://localhost:5424"
-
-	if [ ${OPEN_BROWSER} = "true" ] ; then \
-	  (sleep 1 && $(OPEN) "http://localhost:5424/local/user/test/login" || true) & \
-	fi
-	(. ./local.env && $(exe) 2>&1)
-
-fixtures:
-	echo "Resetting local database to fixture data"
-	PGPASSWORD="sekrit" psql -U postgres -p 5432 -h localhost -f "transcripts/sql/clean.sql"
-	PGPASSWORD="sekrit" psql -U postgres -p 5432 -h localhost -f "transcripts/sql/inserts.sql"
+	@echo "Starting up Share at http://localhost:5424";
+	@if curl -f -s http://localhost:1234 >/dev/null 2>&1 ; then \
+	  (sleep 1 && $(OPEN) "http://localhost:5424/local/user/test/login" && $(OPEN) "http://localhost:1234" || true) & \
+	fi;
+	@(. ./local.env && $(exe) 2>&1)
+
+# Loads the local testing share with a bunch of realistic code.
+reset_fixtures:
+	# Prompt for confirmation
+	@echo "This will delete all data in your local share database. Are you sure? (y/N) "
+	@read -r confirmation && [ "$$confirmation" = "y" ] || [ "$$confirmation" = "Y" ]
+	@docker compose -f docker/docker-compose.base.yml down || true
+	@trap 'docker compose -f docker/docker-compose.base.yml down' EXIT INT TERM
+	# Remove the existing postgres volume to reset the database
+	@echo "Removing any existing postgres volume"
+	docker volume rm docker_postgresVolume || true
+	@echo "Booting up docker dependencies..."
+	docker compose -f docker/docker-compose.base.yml -f docker/docker-compose.fixtures.yml up --remove-orphans --detach
+	@echo "Initializing fixture data";
+	@while  ! (  pg_isready --host localhost -U postgres -p 5432 >/dev/null 2>&1 && redis-cli -p 6379 ping  >/dev/null 2>&1 && VAULT_ADDR=http://localhost:8200 vault status >/dev/null 2>&1 )  do \
+	  sleep 1; \
+	done;
+	@echo "Booting up share";
+	@( . ./local.env \
+		$(exe) 2>&1 & \
+		SERVER_PID=$$!; \
+	trap "kill $$SERVER_PID 2>/dev/null || true" EXIT INT TERM; \
+	echo "Loading fixtures"; \
+	./transcripts/fixtures/run.zsh; \
+	kill $$SERVER_PID 2>/dev/null || true; \
+	) 
+	@echo "Done!";
 
 transcripts:
-	./transcripts/run-transcripts.zsh
+	@echo "Taking down any existing docker dependencies"
+	@docker compose -f docker/docker-compose.base.yml down || true
+	@trap 'docker compose -f docker/docker-compose.base.yml down' EXIT INT TERM
+	@echo "Booting up transcript docker dependencies..."
+	docker compose -f docker/docker-compose.base.yml up --remove-orphans --detach
+	@while  ! (  pg_isready --host localhost -U postgres -p 5432 >/dev/null 2>&1 && redis-cli -p 6379 ping  >/dev/null 2>&1 && VAULT_ADDR=http://localhost:8200 vault status >/dev/null 2>&1 )  do \
+	  sleep 1; \
+	done;
+	./transcripts/configure_transcript_database.zsh
+	@echo "Booting up share";
+	( . ./local.env ; \
+		$(exe) & \
+		SERVER_PID=$$!; \
+		trap "kill $$SERVER_PID 2>/dev/null || true" EXIT INT TERM; \
+		echo "Running transcripts"; \
+		./transcripts/run-transcripts.zsh $(pattern); \
+		kill $$SERVER_PID 2>/dev/null || true; \
+	) 
+	@echo "Transcripts complete!";

README.md

@@ -61,10 +61,14 @@ A [flake.nix](flake.nix) file is provided in this repo. It currently doesn't use
 
 ## Running Locally
 
-Start the server and its dependencies with `make serve`.
-You may wish to run `make fixtures` to fill some in some data for local testing.
+The first time you run locally, start with `make reset_fixtures`, then on subsequent runs just use `make serve`.
 
-See `./docker/docker-compose.yml` to see how the postgres and redis services are configured.
+Data changes in Postgres using `make serve` are persistent locally.
+You can reset the database to a known state with `make reset_fixtures`.
+
+`make transcripts` will take down the database and use a temporary one for running the transcripts.
+
+See the `Makefile` and `./docker/docker-compose.base.yml` to learn more.
 
 ### Debugging and observability
 

docker/docker-compose.base.yml

@@ -0,0 +1,83 @@
+# docker compose config for local development.
+services:
+  postgres:
+    image: postgres:15.4
+    container_name: postgres
+    restart: always
+    healthcheck:
+      # Ensure the database is up, and the tables are initialized
+      test: ["CMD", "psql", "-U", "postgres", "-c", "SELECT from users;"]
+      interval: 3s
+      timeout: 10s
+      retries: 5
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_PASSWORD: sekrit
+    volumes:
+      - ../sql:/docker-entrypoint-initdb.d
+      - ./postgresql.conf:/etc/postgresql/postgresql.conf
+    command: postgres -c config_file=/etc/postgresql/postgresql.conf # -c log_statement=all
+
+  redis:
+    image: redis:6.2.6
+    container_name: redis
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 3
+    ports:
+     - "6379:6379"
+
+  vault:
+    image: 'hashicorp/vault:1.19'
+    container_name: vault
+    healthcheck:
+      test: ["CMD", "vault", "status"]
+      interval: 3s
+      timeout: 10s
+      retries: 3
+    ports:
+      - "8200:8200"
+    environment:
+      VAULT_DEV_ROOT_TOKEN_ID: "sekrit"
+      VAULT_KV_V1_MOUNT_PATH: "secret"
+      VAULT_ADDR: "http://127.0.0.1:8200"
+    cap_add:
+      - IPC_LOCK
+    # # Use kv version 1
+    # command: server -dev
+
+  http-echo:
+    image: 'mendhak/http-https-echo:36'
+    container_name: http-echo
+    environment:
+      HTTP_PORT: 9999
+      ECHO_BACK_TO_CLIENT: "false"
+    ports:
+      - "9999:9999"
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
+    ports:
+      - 1888:1888 # pprof extension
+      - 8888:8888 # Prometheus metrics exposed by the Collector
+      - 8889:8889 # Prometheus exporter metrics
+      - 13133:13133 # health_check extension
+      - 4317:4317 # OTLP gRPC receiver
+      - 4318:4318 # OTLP http receiver
+      - 55679:55679 # zpages extension
+
+    depends_on:
+      - jaeger
+
+  jaeger:
+      image: cr.jaegertracing.io/jaegertracing/jaeger:2.8.0
+      ports:
+        - 16686:16686  # Jaeger UI
+
+      environment:
+        - COLLECTOR_OTLP_ENABLED=true

docker/docker-compose.fixtures.yml

@@ -0,0 +1,9 @@
+# Sets postgres container to use a persistent volume for its data
+services:
+  postgres:
+    volumes:
+      # Persist the data
+      - postgresVolume:/var/lib/postgresql/data
+
+volumes:
+  postgresVolume:

docker/docker-compose.share.yml

@@ -0,0 +1,55 @@
+services:
+  # Define the share service for use in CI
+  share:
+    image: share-api
+    container_name: share-api
+    depends_on:
+          redis:
+            condition: service_healthy
+          postgres:
+            condition: service_healthy
+          vault:
+            condition: service_healthy
+          http-echo:
+            condition: service_started
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5424/health"]
+      interval: 3s
+      timeout: 10s
+      retries: 3
+    ports:
+      - "5424:5424"
+
+    environment:
+      # Placeholder values for development
+      - SHARE_DEPLOYMENT=local
+      - SHARE_API_ORIGIN=http://localhost:5424
+      - SHARE_SERVER_PORT=5424
+      - SHARE_REDIS=redis://redis:6379
+      - SHARE_POSTGRES=postgresql://postgres:sekrit@postgres:5432
+      - SHARE_POSTGRES_CONN_TTL=30
+      - SHARE_POSTGRES_CONN_MAX=10
+      - SHARE_HMAC_KEY=hmac-key-test-key-test-key-test-
+      - SHARE_EDDSA_KEY=eddsa-key-test-key-test-key-test
+      - SHARE_SHARE_UI_ORIGIN=http://localhost:1234
+      - SHARE_CLOUD_UI_ORIGIN=http://localhost:5678
+      - SHARE_HOMEPAGE_ORIGIN=http://localhost:1111
+      - SHARE_CLOUD_HOMEPAGE_ORIGIN=http://localhost:2222
+      - SHARE_CLOUD_API_ORIGIN=http://localhost:3333
+      - SHARE_CLOUD_API_JWKS_ENDPOINT=http://localhost:3333/.well-known/jwks.json
+      - SHARE_LOG_LEVEL=DEBUG
+      - SHARE_COMMIT=dev
+      - SHARE_MAX_PARALLELISM_PER_DOWNLOAD_REQUEST=1
+      - SHARE_MAX_PARALLELISM_PER_UPLOAD_REQUEST=5
+      - VAULT_HOST=http://vault:8200/v1
+      - VAULT_TOKEN=sekrit
+      - USER_SECRETS_VAULT_MOUNT=secret # A default mount in dev vault
+      - SHARE_GITHUB_CLIENTID=invalid
+      - SHARE_GITHUB_CLIENT_SECRET=invalid
+      - OTEL_SERVICE_NAME=share-api
+      - OTEL_SERVICE_VERSION=local
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+      - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+      - OTEL_RESOURCE_ATTRIBUTES=service.name=share-api,service.version=local,deployment.environment=local
+      - OTEL_TRACES_SAMPLER=traceidratio
+      - OTEL_TRACES_SAMPLER_ARG=1.0