Skip to content

Commit f50e678

Browse files
ChrisPennerChrisPenner
authored
Merge pull request #139 from unisoncomputing/cp/dev-workflow
Easier developer workflow
2 parents e3d6971 + f0eff65 commit f50e678

18 files changed

+554
-393
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,8 @@ jobs:
229229
docker image ls -a
230230
231231
- name: Run transcript tests
232+
env:
233+
SHARE_PROJECT_ROOT: ${{ github.workspace }}
232234
# If it takes longer than this, it's probably stalled out.
233235
timeout-minutes: 10
234236
run: |
@@ -239,12 +241,11 @@ jobs:
239241
curl -L https://github.com/unisonweb/unison/releases/download/release%2F0.5.44/ucm-linux-x64.tar.gz | tar -xz -C ucm
240242
export PATH=$PWD/ucm:$PATH
241243
242-
# Clean up old postgres data if it exists.
243-
docker volume rm docker_postgresVolume 2>/dev/null || true
244-
245244
# Start share and it's dependencies in the background
246-
docker compose -f docker/docker-compose.yml up --wait
245+
docker compose -f docker/docker-compose.base.yml -f docker/docker-compose.share.yml up --wait
247246
247+
# Configure it as a transcript database
248+
./transcripts/configure_transcript_database.zsh
248249
249250
# Run the transcript tests
250251
zsh ./transcripts/run-transcripts.zsh

‎Makefile‎

Lines changed: 61 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
.PHONY: all clean install docker_server_build docker_push serve auth_example transcripts fixtures transcripts
1+
.PHONY: all clean install docker_server_build docker_push serve auth_example transcripts fixtures transcripts reset_fixtures
22

3+
SHARE_PROJECT_ROOT := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
4+
export SHARE_PROJECT_ROOT
35
UNAME := $(shell uname)
46
STACK_FLAGS := "--fast"
57
dist_dir := $(shell stack path | awk '/^dist-dir/{print $$2}')
@@ -36,39 +38,66 @@ $(installed_share): $(exe) $(target_dir)
3638
auth_example:
3739
stack build --fast test-auth-app
3840

39-
docker_server_build: $(installed_share)
40-
docker build $(docker_platform_flag) -f docker/Dockerfile --build-arg share_commit=$(share_commit) -t share docker
41-
42-
docker_server_release: $(installed_share)
43-
docker build $(docker_platform_flag) -f docker/Dockerfile -t $(docker_registry)/share:$(DRONE_BUILD_NUMBER) docker
44-
45-
docker_push: $(docker_server_release)
46-
docker push $(docker_registry)/share:$(DRONE_BUILD_NUMBER)
47-
48-
docker_staging_release: $(installed_share)
49-
docker build $(docker_platform_flag) -f docker/Dockerfile -t $(docker_registry)/share-staging:$(DRONE_BUILD_NUMBER) docker
50-
51-
docker_staging_push: $(docker_server_release)
52-
docker push $(docker_registry)/share-staging:$(DRONE_BUILD_NUMBER)
53-
41+
# Build Share and run it alongside its dependencies via docker-compose
5442
serve: $(installed_share)
55-
trap 'docker compose -f docker/docker-compose.yml down' EXIT INT TERM
56-
docker compose -f docker/docker-compose.yml up postgres redis vault &
57-
while ! ( pg_isready --host localhost -U postgres -p 5432 && redis-cli -p 6379 ping && VAULT_ADDR=http://localhost:8200 vault status) do \
58-
echo "Waiting for postgres and redis..."; \
43+
@docker compose -f docker/docker-compose.base.yml down || true
44+
@trap 'docker compose -f docker/docker-compose.base.yml down' EXIT INT TERM
45+
@echo "Booting up docker dependencies..."
46+
docker compose -f docker/docker-compose.base.yml -f docker/docker-compose.fixtures.yml up --remove-orphans --detach
47+
@echo "Booting up docker dependencies...";
48+
@while ! ( pg_isready --host localhost -U postgres -p 5432 >/dev/null 2>&1 && redis-cli -p 6379 ping >/dev/null 2>&1 && VAULT_ADDR=http://localhost:8200 vault status >/dev/null 2>&1 ) do \
5949
sleep 1; \
6050
done;
61-
echo "Running Share at http://localhost:5424"
62-
63-
if [ ${OPEN_BROWSER} = "true" ] ; then \
64-
(sleep 1 && $(OPEN) "http://localhost:5424/local/user/test/login" || true) & \
65-
fi
66-
(. ./local.env && $(exe) 2>&1)
67-
68-
fixtures:
69-
echo "Resetting local database to fixture data"
70-
PGPASSWORD="sekrit" psql -U postgres -p 5432 -h localhost -f "transcripts/sql/clean.sql"
71-
PGPASSWORD="sekrit" psql -U postgres -p 5432 -h localhost -f "transcripts/sql/inserts.sql"
51+
@echo "Starting up Share at http://localhost:5424";
52+
@if curl -f -s http://localhost:1234 >/dev/null 2>&1 ; then \
53+
(sleep 1 && $(OPEN) "http://localhost:5424/local/user/test/login" && $(OPEN) "http://localhost:1234" || true) & \
54+
fi;
55+
@(. ./local.env && $(exe) 2>&1)
56+
57+
# Loads the local testing share with a bunch of realistic code.
58+
reset_fixtures:
59+
# Prompt for confirmation
60+
@echo "This will delete all data in your local share database. Are you sure? (y/N) "
61+
@read -r confirmation && [ "$$confirmation" = "y" ] || [ "$$confirmation" = "Y" ]
62+
@docker compose -f docker/docker-compose.base.yml down || true
63+
@trap 'docker compose -f docker/docker-compose.base.yml down' EXIT INT TERM
64+
# Remove the existing postgres volume to reset the database
65+
@echo "Removing any existing postgres volume"
66+
docker volume rm docker_postgresVolume || true
67+
@echo "Booting up docker dependencies..."
68+
docker compose -f docker/docker-compose.base.yml -f docker/docker-compose.fixtures.yml up --remove-orphans --detach
69+
@echo "Initializing fixture data";
70+
@while ! ( pg_isready --host localhost -U postgres -p 5432 >/dev/null 2>&1 && redis-cli -p 6379 ping >/dev/null 2>&1 && VAULT_ADDR=http://localhost:8200 vault status >/dev/null 2>&1 ) do \
71+
sleep 1; \
72+
done;
73+
@echo "Booting up share";
74+
@( . ./local.env \
75+
$(exe) 2>&1 & \
76+
SERVER_PID=$$!; \
77+
trap "kill $$SERVER_PID 2>/dev/null || true" EXIT INT TERM; \
78+
echo "Loading fixtures"; \
79+
./transcripts/fixtures/run.zsh; \
80+
kill $$SERVER_PID 2>/dev/null || true; \
81+
)
82+
@echo "Done!";
7283

7384
transcripts:
74-
./transcripts/run-transcripts.zsh
85+
@echo "Taking down any existing docker dependencies"
86+
@docker compose -f docker/docker-compose.base.yml down || true
87+
@trap 'docker compose -f docker/docker-compose.base.yml down' EXIT INT TERM
88+
@echo "Booting up transcript docker dependencies..."
89+
docker compose -f docker/docker-compose.base.yml up --remove-orphans --detach
90+
@while ! ( pg_isready --host localhost -U postgres -p 5432 >/dev/null 2>&1 && redis-cli -p 6379 ping >/dev/null 2>&1 && VAULT_ADDR=http://localhost:8200 vault status >/dev/null 2>&1 ) do \
91+
sleep 1; \
92+
done;
93+
./transcripts/configure_transcript_database.zsh
94+
@echo "Booting up share";
95+
( . ./local.env ; \
96+
$(exe) & \
97+
SERVER_PID=$$!; \
98+
trap "kill $$SERVER_PID 2>/dev/null || true" EXIT INT TERM; \
99+
echo "Running transcripts"; \
100+
./transcripts/run-transcripts.zsh $(pattern); \
101+
kill $$SERVER_PID 2>/dev/null || true; \
102+
)
103+
@echo "Transcripts complete!";

‎README.md‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,14 @@ A [flake.nix](flake.nix) file is provided in this repo. It currently doesn't use
6161

6262
## Running Locally
6363

64-
Start the server and its dependencies with `make serve`.
65-
You may wish to run `make fixtures` to fill some in some data for local testing.
64+
The first time you run locally, start with `make reset_fixtures`, then on subsequent runs just use `make serve`.
6665

67-
See `./docker/docker-compose.yml` to see how the postgres and redis services are configured.
66+
Data changes in Postgres using `make serve` are persistent locally.
67+
You can reset the database to a known state with `make reset_fixtures`.
68+
69+
`make transcripts` will take down the database and use a temporary one for running the transcripts.
70+
71+
See the `Makefile` and `./docker/docker-compose.base.yml` to learn more.
6872

6973
### Debugging and observability
7074

‎docker/docker-compose.base.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# docker compose config for local development.
2+
services:
3+
postgres:
4+
image: postgres:15.4
5+
container_name: postgres
6+
restart: always
7+
healthcheck:
8+
# Ensure the database is up, and the tables are initialized
9+
test: ["CMD", "psql", "-U", "postgres", "-c", "SELECT from users;"]
10+
interval: 3s
11+
timeout: 10s
12+
retries: 5
13+
ports:
14+
- "5432:5432"
15+
environment:
16+
POSTGRES_PASSWORD: sekrit
17+
volumes:
18+
- ../sql:/docker-entrypoint-initdb.d
19+
- ./postgresql.conf:/etc/postgresql/postgresql.conf
20+
command: postgres -c config_file=/etc/postgresql/postgresql.conf # -c log_statement=all
21+
22+
redis:
23+
image: redis:6.2.6
24+
container_name: redis
25+
healthcheck:
26+
test: ["CMD", "redis-cli", "ping"]
27+
interval: 3s
28+
timeout: 10s
29+
retries: 3
30+
ports:
31+
- "6379:6379"
32+
33+
vault:
34+
image: 'hashicorp/vault:1.19'
35+
container_name: vault
36+
healthcheck:
37+
test: ["CMD", "vault", "status"]
38+
interval: 3s
39+
timeout: 10s
40+
retries: 3
41+
ports:
42+
- "8200:8200"
43+
environment:
44+
VAULT_DEV_ROOT_TOKEN_ID: "sekrit"
45+
VAULT_KV_V1_MOUNT_PATH: "secret"
46+
VAULT_ADDR: "http://127.0.0.1:8200"
47+
cap_add:
48+
- IPC_LOCK
49+
# # Use kv version 1
50+
# command: server -dev
51+
52+
http-echo:
53+
image: 'mendhak/http-https-echo:36'
54+
container_name: http-echo
55+
environment:
56+
HTTP_PORT: 9999
57+
ECHO_BACK_TO_CLIENT: "false"
58+
ports:
59+
- "9999:9999"
60+
61+
otel-collector:
62+
image: otel/opentelemetry-collector-contrib
63+
volumes:
64+
- ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
65+
ports:
66+
- 1888:1888 # pprof extension
67+
- 8888:8888 # Prometheus metrics exposed by the Collector
68+
- 8889:8889 # Prometheus exporter metrics
69+
- 13133:13133 # health_check extension
70+
- 4317:4317 # OTLP gRPC receiver
71+
- 4318:4318 # OTLP http receiver
72+
- 55679:55679 # zpages extension
73+
74+
depends_on:
75+
- jaeger
76+
77+
jaeger:
78+
image: cr.jaegertracing.io/jaegertracing/jaeger:2.8.0
79+
ports:
80+
- 16686:16686 # Jaeger UI
81+
82+
environment:
83+
- COLLECTOR_OTLP_ENABLED=true

‎docker/docker-compose.fixtures.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Sets postgres container to use a persistent volume for its data
2+
services:
3+
postgres:
4+
volumes:
5+
# Persist the data
6+
- postgresVolume:/var/lib/postgresql/data
7+
8+
volumes:
9+
postgresVolume:

‎docker/docker-compose.share.yml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
services:
2+
# Define the share service for use in CI
3+
share:
4+
image: share-api
5+
container_name: share-api
6+
depends_on:
7+
redis:
8+
condition: service_healthy
9+
postgres:
10+
condition: service_healthy
11+
vault:
12+
condition: service_healthy
13+
http-echo:
14+
condition: service_started
15+
healthcheck:
16+
test: ["CMD", "curl", "-f", "http://localhost:5424/health"]
17+
interval: 3s
18+
timeout: 10s
19+
retries: 3
20+
ports:
21+
- "5424:5424"
22+
23+
environment:
24+
# Placeholder values for development
25+
- SHARE_DEPLOYMENT=local
26+
- SHARE_API_ORIGIN=http://localhost:5424
27+
- SHARE_SERVER_PORT=5424
28+
- SHARE_REDIS=redis://redis:6379
29+
- SHARE_POSTGRES=postgresql://postgres:sekrit@postgres:5432
30+
- SHARE_POSTGRES_CONN_TTL=30
31+
- SHARE_POSTGRES_CONN_MAX=10
32+
- SHARE_HMAC_KEY=hmac-key-test-key-test-key-test-
33+
- SHARE_EDDSA_KEY=eddsa-key-test-key-test-key-test
34+
- SHARE_SHARE_UI_ORIGIN=http://localhost:1234
35+
- SHARE_CLOUD_UI_ORIGIN=http://localhost:5678
36+
- SHARE_HOMEPAGE_ORIGIN=http://localhost:1111
37+
- SHARE_CLOUD_HOMEPAGE_ORIGIN=http://localhost:2222
38+
- SHARE_CLOUD_API_ORIGIN=http://localhost:3333
39+
- SHARE_CLOUD_API_JWKS_ENDPOINT=http://localhost:3333/.well-known/jwks.json
40+
- SHARE_LOG_LEVEL=DEBUG
41+
- SHARE_COMMIT=dev
42+
- SHARE_MAX_PARALLELISM_PER_DOWNLOAD_REQUEST=1
43+
- SHARE_MAX_PARALLELISM_PER_UPLOAD_REQUEST=5
44+
- VAULT_HOST=http://vault:8200/v1
45+
- VAULT_TOKEN=sekrit
46+
- USER_SECRETS_VAULT_MOUNT=secret # A default mount in dev vault
47+
- SHARE_GITHUB_CLIENTID=invalid
48+
- SHARE_GITHUB_CLIENT_SECRET=invalid
49+
- OTEL_SERVICE_NAME=share-api
50+
- OTEL_SERVICE_VERSION=local
51+
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
52+
- OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
53+
- OTEL_RESOURCE_ATTRIBUTES=service.name=share-api,service.version=local,deployment.environment=local
54+
- OTEL_TRACES_SAMPLER=traceidratio
55+
- OTEL_TRACES_SAMPLER_ARG=1.0

0 commit comments

Comments
 (0)