diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eb157ff6..c2ebd010 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -229,6 +229,8 @@ jobs: docker image ls -a - name: Run transcript tests + env: + SHARE_PROJECT_ROOT: ${{ github.workspace }} # If it takes longer than this, it's probably stalled out. timeout-minutes: 10 run: | @@ -239,12 +241,11 @@ jobs: curl -L https://github.com/unisonweb/unison/releases/download/release%2F0.5.44/ucm-linux-x64.tar.gz | tar -xz -C ucm export PATH=$PWD/ucm:$PATH - # Clean up old postgres data if it exists. - docker volume rm docker_postgresVolume 2>/dev/null || true - # Start share and it's dependencies in the background - docker compose -f docker/docker-compose.yml up --wait + docker compose -f docker/docker-compose.base.yml -f docker/docker-compose.share.yml up --wait + # Configure it as a transcript database + ./transcripts/configure_transcript_database.zsh # Run the transcript tests zsh ./transcripts/run-transcripts.zsh diff --git a/Makefile b/Makefile index e381b22c..4d2256b2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ -.PHONY: all clean install docker_server_build docker_push serve auth_example transcripts fixtures transcripts +.PHONY: all clean install docker_server_build docker_push serve auth_example transcripts fixtures transcripts reset_fixtures +SHARE_PROJECT_ROOT := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +export SHARE_PROJECT_ROOT UNAME := $(shell uname) STACK_FLAGS := "--fast" dist_dir := $(shell stack path | awk '/^dist-dir/{print $$2}') @@ -36,39 +38,66 @@ $(installed_share): $(exe) $(target_dir) auth_example: stack build --fast test-auth-app -docker_server_build: $(installed_share) - docker build $(docker_platform_flag) -f docker/Dockerfile --build-arg share_commit=$(share_commit) -t share docker - -docker_server_release: $(installed_share) - docker build $(docker_platform_flag) -f docker/Dockerfile -t $(docker_registry)/share:$(DRONE_BUILD_NUMBER) docker - -docker_push: $(docker_server_release) - docker push $(docker_registry)/share:$(DRONE_BUILD_NUMBER) - -docker_staging_release: $(installed_share) - docker build $(docker_platform_flag) -f docker/Dockerfile -t $(docker_registry)/share-staging:$(DRONE_BUILD_NUMBER) docker - -docker_staging_push: $(docker_server_release) - docker push $(docker_registry)/share-staging:$(DRONE_BUILD_NUMBER) - +# Build Share and run it alongside its dependencies via docker-compose serve: $(installed_share) - trap 'docker compose -f docker/docker-compose.yml down' EXIT INT TERM - docker compose -f docker/docker-compose.yml up postgres redis vault & - while ! ( pg_isready --host localhost -U postgres -p 5432 && redis-cli -p 6379 ping && VAULT_ADDR=http://localhost:8200 vault status) do \ - echo "Waiting for postgres and redis..."; \ + @docker compose -f docker/docker-compose.base.yml down || true + @trap 'docker compose -f docker/docker-compose.base.yml down' EXIT INT TERM + @echo "Booting up docker dependencies..." + docker compose -f docker/docker-compose.base.yml -f docker/docker-compose.fixtures.yml up --remove-orphans --detach + @echo "Booting up docker dependencies..."; + @while ! ( pg_isready --host localhost -U postgres -p 5432 >/dev/null 2>&1 && redis-cli -p 6379 ping >/dev/null 2>&1 && VAULT_ADDR=http://localhost:8200 vault status >/dev/null 2>&1 ) do \ sleep 1; \ done; - echo "Running Share at http://localhost:5424" - - if [ ${OPEN_BROWSER} = "true" ] ; then \ - (sleep 1 && $(OPEN) "http://localhost:5424/local/user/test/login" || true) & \ - fi - (. ./local.env && $(exe) 2>&1) - -fixtures: - echo "Resetting local database to fixture data" - PGPASSWORD="sekrit" psql -U postgres -p 5432 -h localhost -f "transcripts/sql/clean.sql" - PGPASSWORD="sekrit" psql -U postgres -p 5432 -h localhost -f "transcripts/sql/inserts.sql" + @echo "Starting up Share at http://localhost:5424"; + @if curl -f -s http://localhost:1234 >/dev/null 2>&1 ; then \ + (sleep 1 && $(OPEN) "http://localhost:5424/local/user/test/login" && $(OPEN) "http://localhost:1234" || true) & \ + fi; + @(. ./local.env && $(exe) 2>&1) + +# Loads the local testing share with a bunch of realistic code. +reset_fixtures: + # Prompt for confirmation + @echo "This will delete all data in your local share database. Are you sure? (y/N) " + @read -r confirmation && [ "$$confirmation" = "y" ] || [ "$$confirmation" = "Y" ] + @docker compose -f docker/docker-compose.base.yml down || true + @trap 'docker compose -f docker/docker-compose.base.yml down' EXIT INT TERM + # Remove the existing postgres volume to reset the database + @echo "Removing any existing postgres volume" + docker volume rm docker_postgresVolume || true + @echo "Booting up docker dependencies..." + docker compose -f docker/docker-compose.base.yml -f docker/docker-compose.fixtures.yml up --remove-orphans --detach + @echo "Initializing fixture data"; + @while ! ( pg_isready --host localhost -U postgres -p 5432 >/dev/null 2>&1 && redis-cli -p 6379 ping >/dev/null 2>&1 && VAULT_ADDR=http://localhost:8200 vault status >/dev/null 2>&1 ) do \ + sleep 1; \ + done; + @echo "Booting up share"; + @( . ./local.env \ + $(exe) 2>&1 & \ + SERVER_PID=$$!; \ + trap "kill $$SERVER_PID 2>/dev/null || true" EXIT INT TERM; \ + echo "Loading fixtures"; \ + ./transcripts/fixtures/run.zsh; \ + kill $$SERVER_PID 2>/dev/null || true; \ + ) + @echo "Done!"; transcripts: - ./transcripts/run-transcripts.zsh + @echo "Taking down any existing docker dependencies" + @docker compose -f docker/docker-compose.base.yml down || true + @trap 'docker compose -f docker/docker-compose.base.yml down' EXIT INT TERM + @echo "Booting up transcript docker dependencies..." + docker compose -f docker/docker-compose.base.yml up --remove-orphans --detach + @while ! ( pg_isready --host localhost -U postgres -p 5432 >/dev/null 2>&1 && redis-cli -p 6379 ping >/dev/null 2>&1 && VAULT_ADDR=http://localhost:8200 vault status >/dev/null 2>&1 ) do \ + sleep 1; \ + done; + ./transcripts/configure_transcript_database.zsh + @echo "Booting up share"; + ( . ./local.env ; \ + $(exe) & \ + SERVER_PID=$$!; \ + trap "kill $$SERVER_PID 2>/dev/null || true" EXIT INT TERM; \ + echo "Running transcripts"; \ + ./transcripts/run-transcripts.zsh $(pattern); \ + kill $$SERVER_PID 2>/dev/null || true; \ + ) + @echo "Transcripts complete!"; diff --git a/README.md b/README.md index 0b3e1b38..7e1b1a44 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,14 @@ A [flake.nix](flake.nix) file is provided in this repo. It currently doesn't use ## Running Locally -Start the server and its dependencies with `make serve`. -You may wish to run `make fixtures` to fill some in some data for local testing. +The first time you run locally, start with `make reset_fixtures`, then on subsequent runs just use `make serve`. -See `./docker/docker-compose.yml` to see how the postgres and redis services are configured. +Data changes in Postgres using `make serve` are persistent locally. +You can reset the database to a known state with `make reset_fixtures`. + +`make transcripts` will take down the database and use a temporary one for running the transcripts. + +See the `Makefile` and `./docker/docker-compose.base.yml` to learn more. ### Debugging and observability diff --git a/docker/docker-compose.base.yml b/docker/docker-compose.base.yml new file mode 100644 index 00000000..eeb0c292 --- /dev/null +++ b/docker/docker-compose.base.yml @@ -0,0 +1,83 @@ +# docker compose config for local development. +services: + postgres: + image: postgres:15.4 + container_name: postgres + restart: always + healthcheck: + # Ensure the database is up, and the tables are initialized + test: ["CMD", "psql", "-U", "postgres", "-c", "SELECT from users;"] + interval: 3s + timeout: 10s + retries: 5 + ports: + - "5432:5432" + environment: + POSTGRES_PASSWORD: sekrit + volumes: + - ../sql:/docker-entrypoint-initdb.d + - ./postgresql.conf:/etc/postgresql/postgresql.conf + command: postgres -c config_file=/etc/postgresql/postgresql.conf # -c log_statement=all + + redis: + image: redis:6.2.6 + container_name: redis + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 10s + retries: 3 + ports: + - "6379:6379" + + vault: + image: 'hashicorp/vault:1.19' + container_name: vault + healthcheck: + test: ["CMD", "vault", "status"] + interval: 3s + timeout: 10s + retries: 3 + ports: + - "8200:8200" + environment: + VAULT_DEV_ROOT_TOKEN_ID: "sekrit" + VAULT_KV_V1_MOUNT_PATH: "secret" + VAULT_ADDR: "http://127.0.0.1:8200" + cap_add: + - IPC_LOCK + # # Use kv version 1 + # command: server -dev + + http-echo: + image: 'mendhak/http-https-echo:36' + container_name: http-echo + environment: + HTTP_PORT: 9999 + ECHO_BACK_TO_CLIENT: "false" + ports: + - "9999:9999" + + otel-collector: + image: otel/opentelemetry-collector-contrib + volumes: + - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - 1888:1888 # pprof extension + - 8888:8888 # Prometheus metrics exposed by the Collector + - 8889:8889 # Prometheus exporter metrics + - 13133:13133 # health_check extension + - 4317:4317 # OTLP gRPC receiver + - 4318:4318 # OTLP http receiver + - 55679:55679 # zpages extension + + depends_on: + - jaeger + + jaeger: + image: cr.jaegertracing.io/jaegertracing/jaeger:2.8.0 + ports: + - 16686:16686 # Jaeger UI + + environment: + - COLLECTOR_OTLP_ENABLED=true diff --git a/docker/docker-compose.fixtures.yml b/docker/docker-compose.fixtures.yml new file mode 100644 index 00000000..0d8db71b --- /dev/null +++ b/docker/docker-compose.fixtures.yml @@ -0,0 +1,9 @@ +# Sets postgres container to use a persistent volume for its data +services: + postgres: + volumes: + # Persist the data + - postgresVolume:/var/lib/postgresql/data + +volumes: + postgresVolume: diff --git a/docker/docker-compose.share.yml b/docker/docker-compose.share.yml new file mode 100644 index 00000000..d0ce1049 --- /dev/null +++ b/docker/docker-compose.share.yml @@ -0,0 +1,55 @@ +services: + # Define the share service for use in CI + share: + image: share-api + container_name: share-api + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + vault: + condition: service_healthy + http-echo: + condition: service_started + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5424/health"] + interval: 3s + timeout: 10s + retries: 3 + ports: + - "5424:5424" + + environment: + # Placeholder values for development + - SHARE_DEPLOYMENT=local + - SHARE_API_ORIGIN=http://localhost:5424 + - SHARE_SERVER_PORT=5424 + - SHARE_REDIS=redis://redis:6379 + - SHARE_POSTGRES=postgresql://postgres:sekrit@postgres:5432 + - SHARE_POSTGRES_CONN_TTL=30 + - SHARE_POSTGRES_CONN_MAX=10 + - SHARE_HMAC_KEY=hmac-key-test-key-test-key-test- + - SHARE_EDDSA_KEY=eddsa-key-test-key-test-key-test + - SHARE_SHARE_UI_ORIGIN=http://localhost:1234 + - SHARE_CLOUD_UI_ORIGIN=http://localhost:5678 + - SHARE_HOMEPAGE_ORIGIN=http://localhost:1111 + - SHARE_CLOUD_HOMEPAGE_ORIGIN=http://localhost:2222 + - SHARE_CLOUD_API_ORIGIN=http://localhost:3333 + - SHARE_CLOUD_API_JWKS_ENDPOINT=http://localhost:3333/.well-known/jwks.json + - SHARE_LOG_LEVEL=DEBUG + - SHARE_COMMIT=dev + - SHARE_MAX_PARALLELISM_PER_DOWNLOAD_REQUEST=1 + - SHARE_MAX_PARALLELISM_PER_UPLOAD_REQUEST=5 + - VAULT_HOST=http://vault:8200/v1 + - VAULT_TOKEN=sekrit + - USER_SECRETS_VAULT_MOUNT=secret # A default mount in dev vault + - SHARE_GITHUB_CLIENTID=invalid + - SHARE_GITHUB_CLIENT_SECRET=invalid + - OTEL_SERVICE_NAME=share-api + - OTEL_SERVICE_VERSION=local + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - OTEL_RESOURCE_ATTRIBUTES=service.name=share-api,service.version=local,deployment.environment=local + - OTEL_TRACES_SAMPLER=traceidratio + - OTEL_TRACES_SAMPLER_ARG=1.0 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 426e5869..00000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,141 +0,0 @@ -services: - postgres: - image: postgres:15.4 - container_name: postgres - restart: always - healthcheck: - # Ensure the database is up, and the tables are initialized - test: ["CMD", "psql", "-U", "postgres", "-c", "SELECT from users;"] - interval: 3s - timeout: 10s - retries: 5 - ports: - - "5432:5432" - environment: - POSTGRES_PASSWORD: sekrit - volumes: - - ../sql:/docker-entrypoint-initdb.d - # Optionally persist the data between container invocations - # - postgresVolume:/var/lib/postgresql/data - - ./postgresql.conf:/etc/postgresql/postgresql.conf - command: postgres -c config_file=/etc/postgresql/postgresql.conf # -c log_statement=all - - - redis: - image: redis:6.2.6 - container_name: redis - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 3s - timeout: 10s - retries: 3 - ports: - - "6379:6379" - - vault: - image: 'hashicorp/vault:1.19' - container_name: vault - healthcheck: - test: ["CMD", "vault", "status"] - interval: 3s - timeout: 10s - retries: 3 - ports: - - "8200:8200" - environment: - VAULT_DEV_ROOT_TOKEN_ID: "sekrit" - VAULT_KV_V1_MOUNT_PATH: "secret" - VAULT_ADDR: "http://127.0.0.1:8200" - cap_add: - - IPC_LOCK - # # Use kv version 1 - # command: server -dev - - share: - image: share-api - container_name: share-api - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - vault: - condition: service_healthy - http-echo: - condition: service_started - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:5424/health"] - interval: 3s - timeout: 10s - retries: 3 - ports: - - "5424:5424" - - environment: - # Placeholder values for development - - SHARE_DEPLOYMENT=local - - SHARE_API_ORIGIN=http://localhost:5424 - - SHARE_SERVER_PORT=5424 - - SHARE_REDIS=redis://redis:6379 - - SHARE_POSTGRES=postgresql://postgres:sekrit@postgres:5432 - - SHARE_POSTGRES_CONN_TTL=30 - - SHARE_POSTGRES_CONN_MAX=10 - - SHARE_HMAC_KEY=hmac-key-test-key-test-key-test- - - SHARE_EDDSA_KEY=eddsa-key-test-key-test-key-test - - SHARE_SHARE_UI_ORIGIN=http://localhost:1234 - - SHARE_CLOUD_UI_ORIGIN=http://localhost:5678 - - SHARE_HOMEPAGE_ORIGIN=http://localhost:1111 - - SHARE_CLOUD_HOMEPAGE_ORIGIN=http://localhost:2222 - - SHARE_CLOUD_API_ORIGIN=http://localhost:3333 - - SHARE_CLOUD_API_JWKS_ENDPOINT=http://localhost:3333/.well-known/jwks.json - - SHARE_LOG_LEVEL=DEBUG - - SHARE_COMMIT=dev - - SHARE_MAX_PARALLELISM_PER_DOWNLOAD_REQUEST=1 - - SHARE_MAX_PARALLELISM_PER_UPLOAD_REQUEST=5 - - VAULT_HOST=http://vault:8200/v1 - - VAULT_TOKEN=sekrit - - USER_SECRETS_VAULT_MOUNT=secret # A default mount in dev vault - - SHARE_GITHUB_CLIENTID=invalid - - SHARE_GITHUB_CLIENT_SECRET=invalid - - OTEL_SERVICE_NAME=share-api - - OTEL_SERVICE_VERSION=local - - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 - - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf - - OTEL_RESOURCE_ATTRIBUTES=service.name=share-api,service.version=local,deployment.environment=local - - OTEL_TRACES_SAMPLER=traceidratio - - OTEL_TRACES_SAMPLER_ARG=1.0 - - http-echo: - image: 'mendhak/http-https-echo:36' - container_name: http-echo - environment: - HTTP_PORT: 9999 - ECHO_BACK_TO_CLIENT: "false" - ports: - - "9999:9999" - - otel-collector: - image: otel/opentelemetry-collector-contrib - volumes: - - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml - ports: - - 1888:1888 # pprof extension - - 8888:8888 # Prometheus metrics exposed by the Collector - - 8889:8889 # Prometheus exporter metrics - - 13133:13133 # health_check extension - - 4317:4317 # OTLP gRPC receiver - - 4318:4318 # OTLP http receiver - - 55679:55679 # zpages extension - - depends_on: - - jaeger - - jaeger: - image: cr.jaegertracing.io/jaegertracing/jaeger:2.8.0 - ports: - - 16686:16686 # Jaeger UI - - environment: - - COLLECTOR_OTLP_ENABLED=true -# volumes: -# postgresVolume: diff --git a/transcripts/configure_transcript_database.zsh b/transcripts/configure_transcript_database.zsh new file mode 100755 index 00000000..045a5f9b --- /dev/null +++ b/transcripts/configure_transcript_database.zsh @@ -0,0 +1,6 @@ +#!/bin/zsh + +set -e +set -u + +PGPASSWORD="sekrit" psql -q -U postgres -p 5432 -h localhost -t -A -f "${SHARE_PROJECT_ROOT}/transcripts/sql/configure_transcript_database.sql" > /dev/null diff --git a/transcripts/fixtures/load-from-share.sh b/transcripts/fixtures/load-from-share.sh new file mode 100755 index 00000000..a8124b2c --- /dev/null +++ b/transcripts/fixtures/load-from-share.sh @@ -0,0 +1,59 @@ +#!/bin/zsh + +# Note: this doesn't work as intended yet. + +set -e +set -u + +echo "This is not yet implemented" +exit 1 + +. "${SHARE_PROJECT_ROOT}/transcripts/transcript_functions.sh" + +cache_dir="$HOME/.cache/share-api" + +if [ ! -d "$cache_dir" ]; then + mkdir -p "$cache_dir" +fi + +typeset -A projects +projects=( + base '@unison/base' +) + +for project_name project_ref in "${(@kv)projects}"; do + echo "Downloading sync file for $project_ref" + output_file="$(mktemp)" + curl -X GET --location "https://api.unison-lang.org/ucm/v1/projects/project?name=${project_ref}" \ + --header 'Content-Type: application/json' \ + >"$output_file" + + echo "Response for project $project_ref:" + cat "$output_file" + + latest_release="$(jq -r '.payload."latest-release"' <"$output_file")" + projectId="$(jq -r '.payload."project-id"' <"$output_file")" + branch_ref="releases/${latest_release}" + project_branch_ref="${project_ref}/${branch_ref}" + + curl -X GET --location "https://api.unison-lang.org/ucm/v1/projects/project-branch?projectId=${projectId}&branchName=releases/${latest_release}" \ + --header 'Content-Type: application/json' \ + >"$output_file" + branch_head="$(jq -r '.payload."branch-head"' <"$output_file")" + + echo "Response for project branch $project_branch_ref:" + cat "$output_file" + + sync_file="$cache_dir/${project_branch_ref}" + + if [ -f "$sync_file" ]; then + echo "Sync file for $project_branch_ref already exists, skipping download." + continue + else + mkdir -p "$(dirname "$sync_file")" + echo "Downloading sync file for $project_branch_ref into $sync_file" + curl -X POST --location 'https://api.unison-lang.org/ucm/v2/sync/entities/download' \ + --header 'Content-Type: application/json' \ + --data-raw "{\"branchRef\": \"${project_branch_ref}\", \"causalHash\": \"${branch_head}\", \"knownHashes\":[]}" >"$cache_dir/${project_branch_ref}" + fi +done diff --git a/transcripts/fixtures/run.zsh b/transcripts/fixtures/run.zsh new file mode 100755 index 00000000..e6b331e4 --- /dev/null +++ b/transcripts/fixtures/run.zsh @@ -0,0 +1,8 @@ +#!/bin/zsh + +set -e + +source "${SHARE_PROJECT_ROOT}/transcripts/transcript_functions.sh" +# Set up database so helper scripts know it's a local db +pg_file "${SHARE_PROJECT_ROOT}/transcripts/sql/configure_local_database.sql" +pg_init_fixtures diff --git a/transcripts/run-transcripts.zsh b/transcripts/run-transcripts.zsh index c571d210..d86fa625 100755 --- a/transcripts/run-transcripts.zsh +++ b/transcripts/run-transcripts.zsh @@ -1,6 +1,7 @@ #!/usr/bin/env zsh set -e +set -u unset UNISON_SYNC_VERSION @@ -35,7 +36,7 @@ transcripts=( for transcript dir in "${(@kv)transcripts}"; do # If the first argument is missing, run all transcripts, otherwise run only transcripts which match a prefix of the argument - if [ -z "$1" ] || [[ "$transcript" == "$1"* ]]; then + if [ -z "${1:-}" ] || [[ "$transcript" == "$1"* ]]; then pg_reset_fixtures echo "Running transcript $transcript" (cd "$dir" && {rm -f ./*.json(N) || true} && ./run.zsh); diff --git a/transcripts/share-apis/releases/run.zsh b/transcripts/share-apis/releases/run.zsh index 040ea756..d707169a 100755 --- a/transcripts/share-apis/releases/run.zsh +++ b/transcripts/share-apis/releases/run.zsh @@ -1,6 +1,7 @@ #!/usr/bin/env zsh set -e +set -u source "../../transcript_helpers.sh" diff --git a/transcripts/sql/clean.sql b/transcripts/sql/clean.sql index 27e051fe..3b49958f 100644 --- a/transcripts/sql/clean.sql +++ b/transcripts/sql/clean.sql @@ -1,3 +1,15 @@ +\set ON_ERROR_STOP true + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_name = 'is_transcript_database' + ) THEN + RAISE EXCEPTION 'Refusing to clean non-transcript database.'; + END IF; +END $$; + -- Resets most relevant tables, useful to run between tests. -- Doesn't clean codebase tables since that just slows things down, but does clean out codebase ownership. SET client_min_messages TO WARNING; diff --git a/transcripts/sql/configure_local_database.sql b/transcripts/sql/configure_local_database.sql new file mode 100644 index 00000000..a1aa4992 --- /dev/null +++ b/transcripts/sql/configure_local_database.sql @@ -0,0 +1 @@ +CREATE TABLE IF NOT EXISTS is_local_database(); diff --git a/transcripts/sql/configure_transcript_database.sql b/transcripts/sql/configure_transcript_database.sql new file mode 100644 index 00000000..a20dda35 --- /dev/null +++ b/transcripts/sql/configure_transcript_database.sql @@ -0,0 +1,2 @@ +CREATE TABLE IF NOT EXISTS is_local_database(); +CREATE TABLE IF NOT EXISTS is_transcript_database(); diff --git a/transcripts/sql/inserts.sql b/transcripts/sql/inserts.sql index aade5e6a..478ee820 100644 --- a/transcripts/sql/inserts.sql +++ b/transcripts/sql/inserts.sql @@ -1,3 +1,15 @@ +\set ON_ERROR_STOP true + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_name = 'is_local_database' + ) THEN + RAISE EXCEPTION 'Refusing to insert fixtures on non-local database.'; + END IF; +END $$; + SET client_min_messages TO WARNING; -- Useful for local performance testing. CREATE EXTENSION IF NOT EXISTS pg_stat_statements; diff --git a/transcripts/transcript_functions.sh b/transcripts/transcript_functions.sh new file mode 100644 index 00000000..291352f7 --- /dev/null +++ b/transcripts/transcript_functions.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash + +# Sets up a bunch of utility functions and environment variables, +# but doesn't run anything. +set -e +set -u + +transcripts_dir="${SHARE_PROJECT_ROOT}/transcripts" +ucm_xdg_data_dir=$(mktemp -d) +mkdir -p "${ucm_xdg_data_dir}/unisonlanguage" +ucm_credentials_file="${ucm_xdg_data_dir}/unisonlanguage/credentials.json" + +# Executable to use when running unison transcripts +export UCM_PATH="${1:-"$(which ucm)"}" +export empty_causal_hash='sg60bvjo91fsoo7pkh9gejbn0qgc95vra87ap6l5d35ri0lkaudl7bs12d71sf3fh6p23teemuor7mk1i9n567m50ibakcghjec5ajg' +export echo_server_port=9999 +export echo_server="http://localhost:${echo_server_port}" +# if CI=true, change the echo server url to a url from the docker network. +if [ "${CI:-false}" = "true" ]; then + export echo_server="http://http-echo:${echo_server_port}" +fi + +# UCM to use within transcripts +transcript_ucm() { + XDG_DATA_HOME="${ucm_xdg_data_dir}" UNISON_SHARE_HOST="http://localhost:5424" "${UCM_PATH}" "$@" +} + +cookie_jar_dir=$(mktemp -d) +export cookie_jar_dir + +cookie_jar_for_user_id () { + if [ -z "$1" ]; then + echo "cookie_jar_for_user_id requires a user id" >&2 + exit 1 + fi + echo "${cookie_jar_dir}/$1" +} + +# SQL stuff + +# Run sql against the local pg +pg_sql () { + PGPASSWORD="sekrit" psql -q -U postgres -p 5432 -h localhost -t -A -c "$1" +} + +pg_file () { + PGPASSWORD="sekrit" psql -q -U postgres -p 5432 -h localhost -t -A -f "$1" +} + +pg_init_fixtures() { + # Initialize the database with the schema and inserts + pg_file "${transcripts_dir}/sql/inserts.sql" > /dev/null +} + + +# Reset all the fixtures to the state in `inserts.sql` +pg_reset_fixtures () { + pg_file "${transcripts_dir}/sql/clean.sql" > /dev/null + pg_init_fixtures +} + +user_id_from_handle () { + if [ -z "$1" ]; then + echo "user_id_from_handle requires a handle" >&2 + exit 1 + fi + handle="$1" + pg_sql "SELECT 'U-' || id FROM users WHERE handle = '${handle}';" +} + +project_id_from_handle_and_slug () { + if [ -z "$1" ]; then + echo "project_id_from_handle_and_slug requires a handle" >&2 + exit 1 + fi + if [ -z "$2" ]; then + echo "project_id_from_handle_and_slug requires a slug" >&2 + exit 1 + fi + handle="$1" + slug="$2" + pg_sql "SELECT 'P-' || p.id FROM projects p JOIN users u ON p.owner_user_id = u.id WHERE u.handle = '${handle}' AND p.slug = '${slug}';" +} + +# Creates a user and returns the user id +create_user () { + handle="$1" + uid=$(pg_sql "INSERT INTO users (handle, primary_email, email_verified) VALUES ('${handle}', '${handle}@example.com', true) RETURNING 'U-' || id;") + # Log the user in, storing credentials in their cookie jar. + curl -s --cookie-jar "$(cookie_jar_for_user_id "$uid")" "http://localhost:5424/local/user/${handle}/login" > /dev/null + echo "$uid" +} + +clean_for_transcript() { + # Replace all ISO8601 in stdin with the string "" + sed -E 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z?//g' | \ + # Replace all uuids in stdin with the string "" + sed -E 's/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}//g' | \ + # Replace all cursors in stdin with the string "" + sed -E 's/Cursor": ?"[^"]+"/Cursor": ""/g' | \ + # Replace all JWTs in stdin with the string "" + sed -E 's/eyJ([[:alnum:]_.-]+)//g' | \ + # In docker we network things together with host names, but locally we just use localhost; so + # this normalizes everything in transcripts. + sed -E 's/(localhost|http-echo):/:/g' +} + +fetch() { + testname="$3" + result_file="$(mktemp)" + status_code_file="$(mktemp)" + api_path="$4" + echo "${testname}" "${api_path}" + fetch_data "$@" 2> "${status_code_file}" | clean_for_transcript > "${result_file}" + # Try embedding the json response as-is, but if it's not valid json (e.g. it's an error message instead), embed it as a string. + jq --sort-keys -n --slurpfile status "${status_code_file}" --slurpfile body "${result_file}" '{"status": $status, "body": ($body | .[0])}' > "./$testname.json" 2> /dev/null || { + jq --sort-keys -n --slurpfile status "${status_code_file}" --rawfile body "${result_file}" '{"status": $status, "body": $body}' > "./$testname.json" + } + jq_exit=$? + if [ $jq_exit -ne 0 ]; then + echo "Failed to parse JSON response for test ${testname} with jq exit code ${jq_exit}" >&2 + echo "Response body: $(cat "${result_file}")" >&2 + exit 1 + fi +} + +# fetch which returns the result, +# stderr gets '{"status_code:xxx"}' +# stdout gets the body +fetch_data() { + if [ "$#" -lt 4 ]; then + echo "fetch requires at least 4 arguments: user_id, method, testname, api_path, [data]" >&2 + exit 1 + fi + if [ -z "${1:-}" ]; then + echo "fetch requires a user id" >&2 + exit 1 + fi + cookie_jar="$(cookie_jar_for_user_id "$1")" + method="$2" + testname="$3" + api_path="$4" + data="${5:-}" + url="http://localhost:5424${api_path}" + result_file="$(mktemp)" + status_code_file="$(mktemp)" + + case $method in + GET) + curl --request "GET" -L -s --cookie "$cookie_jar" -H "Accept: application/json" -w '%{stderr} {"status_code":%{http_code}}' "$url" + ;; + *) + curl --request "$method" -L -s --cookie "$cookie_jar" -H "Accept: application/json" -H "Content-Type: application/json" --data-raw "$data" -w '%{stderr} {"status_code":%{http_code}}' "$url" + ;; + esac +} + +fetch_data_jq() { + if [ "$#" -lt 5 ]; then + echo "fetch requires at least 5 arguments: user_id, method, testname, api_path, jq_pattern, [data]" >&2 + exit 1 + fi + if [ -z "$1" ]; then + echo "fetch requires a user id" >&2 + exit 1 + fi + cookie_jar="$1" + method="$2" + testname="$3" + api_path="$4" + jq_pattern="$5" + data="${6:-}" + fetch_data "$cookie_jar" "$method" "$testname" "$api_path" "$data" 2> /dev/null | \ + jq --sort-keys -r "$jq_pattern" +} + +# Credentials setup + +login_user_for_ucm() { + if [ -z "${1:-}" ]; then + echo "login_user_for_ucm requires a user handle" >&2 + exit 1 + fi + user_handle="$1" + user_id=$(user_id_from_handle "$user_handle") + access_token=$(curl -L -s "http://localhost:5424/local/user/${user_handle}/access-token" ) + now=$(date -u "+%F") + + # Save the credentials to a file so that UCM can find them +cat << EOF > "${ucm_credentials_file}" +{ + "active_profile": "default", + "credentials": { + "default": { + "localhost:5424": { + "discovery_uri": "http://localhost:5424/.well-known/openid-configuration", + "fetch_time": "${now}T00:00:00.000000Z", + "tokens": { + "access_token": "${access_token}", + "expires_in": 2592000, + "id_token": null, + "refresh_token": null, + "scope": "openid cloud sync", + "token_type": "bearer" + }, + "user_info": { + "handle": "${user_handle}", + "name": "${user_handle}", + "user_id": "${user_id}" + } + } + } + } +} +EOF +} + +wait_for_diffs() { +# Since namespace diffs are computed asynchronously, we just block here until there are no diffs left in +# the causal_diff_queue. +for _ in {1..5}; do + if [[ $(pg_sql "select count(*) from causal_diff_queue;") -ne 0 ]]; then + sleep 1 + else + break + fi +done +} diff --git a/transcripts/transcript_helpers.sh b/transcripts/transcript_helpers.sh index 8dc61681..3d02478d 100755 --- a/transcripts/transcript_helpers.sh +++ b/transcripts/transcript_helpers.sh @@ -1,83 +1,11 @@ #!/usr/bin/env bash set -e +set -u -transcripts_dir=$(realpath "$(dirname "$0")") -ucm_xdg_data_dir=$(mktemp -d) -mkdir -p "${ucm_xdg_data_dir}/unisonlanguage" -ucm_credentials_file="${ucm_xdg_data_dir}/unisonlanguage/credentials.json" +transcripts_dir="${SHARE_PROJECT_ROOT}/transcripts" +source "${transcripts_dir}/transcript_functions.sh" -# Executable to use when running unison transcripts -export UCM_PATH="${1:-"$(which ucm)"}" -export empty_causal_hash='sg60bvjo91fsoo7pkh9gejbn0qgc95vra87ap6l5d35ri0lkaudl7bs12d71sf3fh6p23teemuor7mk1i9n567m50ibakcghjec5ajg' -export echo_server_port=9999 -export echo_server="http://localhost:${echo_server_port}" -# if CI=true, change the echo server url to a url from the docker network. -if [ "${CI:-false}" = "true" ]; then - export echo_server="http://http-echo:${echo_server_port}" -fi - -# UCM to use within transcripts -transcript_ucm() { - XDG_DATA_HOME="${ucm_xdg_data_dir}" UNISON_SHARE_HOST="http://localhost:5424" "${UCM_PATH}" "$@" -} - -cookie_jar_dir=$(mktemp -d) -export cookie_jar_dir - -cookie_jar_for_user_id () { - if [ -z "$1" ]; then - echo "cookie_jar_for_user_id requires a user id" >&2 - exit 1 - fi - echo "${cookie_jar_dir}/$1" -} - -# SQL stuff - -# Run sql against the local pg -pg_sql () { - PGPASSWORD="sekrit" psql -q -U postgres -p 5432 -h localhost -t -A -c "$1" -} - - -# Reset all the fixtures to the state in `inserts.sql` -pg_reset_fixtures () { - PGPASSWORD="sekrit" psql -U postgres -p 5432 -h localhost -f "${transcripts_dir}/sql/clean.sql" > /dev/null - PGPASSWORD="sekrit" psql -U postgres -p 5432 -h localhost -f "${transcripts_dir}/sql/inserts.sql" > /dev/null -} - -user_id_from_handle () { - if [ -z "$1" ]; then - echo "user_id_from_handle requires a handle" >&2 - exit 1 - fi - handle="$1" - pg_sql "SELECT 'U-' || id FROM users WHERE handle = '${handle}';" -} - -project_id_from_handle_and_slug () { - if [ -z "$1" ]; then - echo "project_id_from_handle_and_slug requires a handle" >&2 - exit 1 - fi - if [ -z "$2" ]; then - echo "project_id_from_handle_and_slug requires a slug" >&2 - exit 1 - fi - handle="$1" - slug="$2" - pg_sql "SELECT 'P-' || p.id FROM projects p JOIN users u ON p.owner_user_id = u.id WHERE u.handle = '${handle}' AND p.slug = '${slug}';" -} - -# Creates a user and returns the user id -create_user () { - handle="$1" - uid=$(pg_sql "INSERT INTO users (handle, primary_email, email_verified) VALUES ('${handle}', '${handle}@example.com', true) RETURNING 'U-' || id;") - # Log the user in, storing credentials in their cookie jar. - curl -s --cookie-jar "$(cookie_jar_for_user_id "$uid")" "http://localhost:5424/local/user/${handle}/login" > /dev/null - echo "$uid" -} # Set up users so we can auth against them. pg_reset_fixtures @@ -108,141 +36,4 @@ curl -s --cookie-jar "$(cookie_jar_for_user_id "$unauthorized_user")" http://loc unauthenticated_user="$(cookie_jar_for_user_id 'unauthenticated')" export unauthenticated_user - -clean_for_transcript() { - # Replace all ISO8601 in stdin with the string "" - sed -E 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z?//g' | \ - # Replace all uuids in stdin with the string "" - sed -E 's/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}//g' | \ - # Replace all cursors in stdin with the string "" - sed -E 's/Cursor": ?"[^"]+"/Cursor": ""/g' | \ - # Replace all JWTs in stdin with the string "" - sed -E 's/eyJ([[:alnum:]_.-]+)//g' | \ - # In docker we network things together with host names, but locally we just use localhost; so - # this normalizes everything in transcripts. - sed -E 's/(localhost|http-echo):/:/g' -} - -fetch() { - testname="$3" - result_file="$(mktemp)" - status_code_file="$(mktemp)" - api_path="$4" - echo "${testname}" "${api_path}" - fetch_data "$@" 2> "${status_code_file}" | clean_for_transcript > "${result_file}" - # Try embedding the json response as-is, but if it's not valid json (e.g. it's an error message instead), embed it as a string. - jq --sort-keys -n --slurpfile status "${status_code_file}" --slurpfile body "${result_file}" '{"status": $status, "body": ($body | .[0])}' > "./$testname.json" 2> /dev/null || { - jq --sort-keys -n --slurpfile status "${status_code_file}" --rawfile body "${result_file}" '{"status": $status, "body": $body}' > "./$testname.json" - } - jq_exit=$? - if [ $jq_exit -ne 0 ]; then - echo "Failed to parse JSON response for test ${testname} with jq exit code ${jq_exit}" >&2 - echo "Response body: $(cat "${result_file}")" >&2 - exit 1 - fi -} - -# fetch which returns the result, -# stderr gets '{"status_code:xxx"}' -# stdout gets the body -fetch_data() { - if [ "$#" -lt 4 ]; then - echo "fetch requires at least 4 arguments: user_id, method, testname, api_path, [data]" >&2 - exit 1 - fi - if [ -z "$1" ]; then - echo "fetch requires a user id" >&2 - exit 1 - fi - cookie_jar="$(cookie_jar_for_user_id "$1")" - method="$2" - testname="$3" - api_path="$4" - data="$5" - url="http://localhost:5424${api_path}" - result_file="$(mktemp)" - status_code_file="$(mktemp)" - - case $method in - GET) - curl --request "GET" -L -s --cookie "$cookie_jar" -H "Accept: application/json" -w '%{stderr} {"status_code":%{http_code}}' "$url" - ;; - *) - curl --request "$method" -L -s --cookie "$cookie_jar" -H "Accept: application/json" -H "Content-Type: application/json" --data-raw "$data" -w '%{stderr} {"status_code":%{http_code}}' "$url" - ;; - esac -} - -fetch_data_jq() { - if [ "$#" -lt 5 ]; then - echo "fetch requires at least 5 arguments: user_id, method, testname, api_path, jq_pattern, [data]" >&2 - exit 1 - fi - if [ -z "$1" ]; then - echo "fetch requires a user id" >&2 - exit 1 - fi - cookie_jar="$1" - method="$2" - testname="$3" - api_path="$4" - jq_pattern="$5" - data="$6" - fetch_data "$cookie_jar" "$method" "$testname" "$api_path" "$data" 2> /dev/null | \ - jq --sort-keys -r "$jq_pattern" -} - -# Credentials setup - -login_user_for_ucm() { - if [ -z "$1" ]; then - echo "login_user_for_ucm requires a user handle" >&2 - exit 1 - fi - user_handle="$1" - user_id=$(user_id_from_handle "$user_handle") - access_token=$(curl -L -s "http://localhost:5424/local/user/${user_handle}/access-token" ) - now=$(date -u "+%F") - - # Save the credentials to a file so that UCM can find them -cat << EOF > "${ucm_credentials_file}" -{ - "active_profile": "default", - "credentials": { - "default": { - "localhost:5424": { - "discovery_uri": "http://localhost:5424/.well-known/openid-configuration", - "fetch_time": "${now}T00:00:00.000000Z", - "tokens": { - "access_token": "${access_token}", - "expires_in": 2592000, - "id_token": null, - "refresh_token": null, - "scope": "openid cloud sync", - "token_type": "bearer" - }, - "user_info": { - "handle": "${user_handle}", - "name": "${user_handle}", - "user_id": "${user_id}" - } - } - } - } -} -EOF -} - login_user_for_ucm 'transcripts' - -wait_for_diffs() { -# Since namespace diffs are computed asynchronously, we just block here until there are no diffs left in -# the causal_diff_queue. -for i in {1..5}; do - if [[ $(pg_sql "select count(*) from causal_diff_queue;") -ne 0 ]]; then - sleep 1 - else - break - fi -done -}