Run separate web server, registry watcher and build servers in docker-compose

Nemo157 · Nemo157 · commit 80a5011e8888 · 2024-03-19T15:46:42.000+01:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -44,13 +44,8 @@ jobs:
 
       - name: Launch postgres
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db
 
       - name: run database migrations
         run: cargo run -- database migrate
@@ -68,9 +63,6 @@ jobs:
             --database-url $DOCSRS_DATABASE_URL \
             --target-version 0
 
-      - name: Clean up the database
-        run: docker compose down --volumes
-
   test:
     env:
       SQLX_OFFLINE: 1
@@ -92,13 +84,8 @@ jobs:
 
       - name: Launch postgres and min.io
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db s3
       
       - name: run workspace tests
         run: |
@@ -110,9 +97,6 @@ jobs:
         run: |
           cargo test --locked -- --ignored --test-threads=1
 
-      - name: Clean up the database
-        run: docker compose down --volumes
-
   GUI_test:
     runs-on: ubuntu-latest
     steps:
@@ -129,19 +113,12 @@ jobs:
 
       - name: Launch postgres and min.io
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db s3
 
       - name: Run GUI tests
         run: ./dockerfiles/run-gui-tests.sh
 
-      - name: Clean up the database
-        run: docker compose down --volumes
 
   fmt:
     name: Rustfmt
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -9,5 +9,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Build the Docker image
-        run: docker build -t docs-rs -f dockerfiles/Dockerfile .
+      - run: docker build --target web-server -f dockerfiles/Dockerfile .
+      - run: docker build --target build-server -f dockerfiles/Dockerfile .
+      - run: docker build --target registry-watcher -f dockerfiles/Dockerfile .
+      - run: docker build --target cli -f dockerfiles/Dockerfile .
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 /ignored
 /.env
+/.docker.env
 /src/web/badge/Cargo.lock
 target
 *.css
diff --git a/README.md b/README.md
@@ -64,7 +64,7 @@ mkdir -p ignored/cratesfyi-prefix/crates.io-index
 # Builds the docs.rs binary
 cargo build
 # Start the external services.
-docker compose up -d db s3
+docker compose up --wait db s3
 # anything that doesn't run via docker-compose needs the settings defined in
 # .env. Either via `. ./.env` as below, or via any dotenv shell integration.
 . ./.env
@@ -115,21 +115,32 @@ which uses docker-compose for the web server as well.
 This will not cache dependencies - in particular, you'll have to rebuild all 400 whenever the lockfile changes -
 but makes sure that you're in a known environment so you should have fewer problems getting started.
 
-You can also use the `web` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows):
+You'll need to `touch .docker.env` first, this file can have any environment
+variable overrides you want to use in docker containers.
+
+You can also use the `builder-a` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows):
+
 ```sh
+# update the toolchain
+docker compose run --rm builder-a build update-toolchain
 # run a build for a single crate
-docker compose run web build crate regex 1.3.1
-# or build essential files
-docker compose run web build add-essential-files
-# rebuild the web container when you changed code.
-docker compose up -d web --build
+docker compose run --rm builder-a build crate regex 1.3.1
+# rebuild containers when you changed code.
+docker compose up --wait --build
 ```
 
-You can also run other commands like the setup above from within the container:
+You can also run other non-build commands like the setup steps above, or queueing crates for the background builders from within the `cli` container:
 
 ```sh
 docker compose run --rm cli database migrate
-docker compose run --rm cli build update-toolchain
+docker compose run --rm cli queue add regex 1.3.1
+```
+
+If the command needs the crates.io-index clone then it must be run from within
+a `registry-watcher` container:
+
+```sh
+docker compose run --rm registry-watcher queue set-last-seen-reference --head
 ```
 
 Note that running tests is not supported when using pure docker-compose.
@@ -151,7 +162,7 @@ Three services are defined:
 
 #### Rebuilding Containers
 
-To rebuild the site, run `docker compose build`.
+To rebuild the site, run `docker compose --profile all build`.
 Note that docker-compose caches the build even if you change the source code,
 so this will be necessary anytime you make changes.
 
@@ -172,7 +183,7 @@ This is probably because you have `git.autocrlf` set to true,
 
 ##### I see the error `/opt/rustwide/cargo-home/bin/cargo: cannot execute binary file: Exec format error` when running builds.
 
-You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run web build [...]` workaround described above.
+You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run --rm builder-a build [...]` workaround described above.
 
 See [rustwide#41](https://github.com/rust-lang/rustwide/issues/41) for more details about supporting more platforms directly.
 
@@ -200,11 +211,11 @@ cargo run -- start-web-server
 ```sh
 # Builds <CRATE_NAME> <CRATE_VERSION> and adds it into database
 # This is the main command to build and add a documentation into docs.rs.
-# For example, `docker compose run web build crate regex 1.1.6`
+# For example, `docker compose run --rm builder-a build crate regex 1.1.6`
 cargo run -- build crate <CRATE_NAME> <CRATE_VERSION>
 
-# alternatively, via the web container
-docker compose run web build crate <CRATE_NAME> <CRATE_VERSION>
+# alternatively, within docker-compose containers
+docker compose run --rm builder-a build crate <CRATE_NAME> <CRATE_VERSION>
 
 # Builds every crate on crates.io and adds them into database
 # (beware: this may take months to finish)
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,35 +1,116 @@
-version: "3"
+version: "3.4"
+
+x-healthcheck: &healthcheck-interval
+  interval: 1s
+  timeout: 1s
+  start_period: 10s
+  # TODO: https://github.com/docker/compose/issues/10461
+  # interval: 10s
+  # start_interval: 1s
+
+x-environment: &environment
+    RUST_BACKTRACE: true
+
+    DOCSRS_PREFIX: /opt/docsrs/prefix
+
+    DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
+    DOCSRS_MIN_POOL_SIZE: 2
+    DOCSRS_MAX_POOL_SIZE: 10
+
+    DOCSRS_STORAGE_BACKEND: s3
+
+    S3_ENDPOINT: http://s3:9000
+    AWS_ACCESS_KEY_ID: cratesfyi
+    AWS_SECRET_ACCESS_KEY: secret_key
+
+    DOCSRS_RENDER_THREADS: 2
+
+    DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
+    DOCSRS_DOCKER: true
+    DOCSRS_DOCKER_IMAGE: ghcr.io/rust-lang/crates-build-env/linux-micro
+    DOCSRS_BUILD_CPU_LIMIT: 2
+    DOCSRS_INCLUDE_DEFAULT_TARGETS: false
+
+x-builder: &builder
+    build:
+        context: .
+        dockerfile: ./dockerfiles/Dockerfile
+        target: build-server
+    depends_on:
+        - db
+        - s3
+    environment: *environment
+    env_file:
+        - .docker.env
+    healthcheck:
+        << : *healthcheck-interval
+        test: curl --silent --fail localhost:3000/about/metrics
+
 services:
     web:
         build:
             context: .
             dockerfile: ./dockerfiles/Dockerfile
+            target: web-server
         platform: "linux/amd64"
         depends_on:
             - db
             - s3
         ports:
-            - "3000:3000"
-        # for metrics
-        expose: ["3000"]
+            - "3000:80"
+        environment: *environment
+        env_file:
+            - .docker.env
+        healthcheck:
+            << : *healthcheck-interval
+            test: curl --silent --fail localhost:80/about/metrics
+
+    # Include the registry watcher with `docker compose --profile watch up --build --wait`
+    registry-watcher:
+        build:
+            context: .
+            dockerfile: ./dockerfiles/Dockerfile
+            target: registry-watcher
+        platform: "linux/amd64"
+        depends_on:
+            - db
         volumes:
-            - "/var/run/docker.sock:/var/run/docker.sock"
-            - ".rustwide-docker:/opt/docsrs/rustwide"
             - "cratesio-index:/opt/docsrs/prefix/crates.io-index"
-        environment:
-            DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
-            DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
-            DOCSRS_STORAGE_BACKEND: s3
-            S3_ENDPOINT: http://s3:9000
-            AWS_ACCESS_KEY_ID: cratesfyi
-            AWS_SECRET_ACCESS_KEY: secret_key
+        environment: *environment
         env_file:
-            - .env
+            - .docker.env
+        profiles:
+            - watch
+            - all
         healthcheck:
-            test: ["CMD", "curl", "--silent", "--fail", "localhost:3000"]
-            interval: 10s
-            timeout: 5s
-            retries: 10
+            << : *healthcheck-interval
+            test: curl --silent --fail localhost:3000/about/metrics
+
+    builder-a:
+        << : *builder
+        volumes:
+            - ".rustwide-docker/builder-a:/opt/docsrs/rustwide"
+            - "/var/run/docker.sock:/var/run/docker.sock"
+
+    builder-b:
+        << : *builder
+        volumes:
+            - ".rustwide-docker/builder-b:/opt/docsrs/rustwide"
+            - "/var/run/docker.sock:/var/run/docker.sock"
+
+    cli:
+        build:
+            context: .
+            dockerfile: ./dockerfiles/Dockerfile
+            target: cli
+        depends_on:
+            - db
+            - s3
+        environment: *environment
+        env_file:
+            - .docker.env
+        profiles:
+            - all
 
     db:
         build:
@@ -44,10 +125,8 @@ services:
             # Use a non-standard port on the host to avoid conflicting with existing postgres servers
             - "127.0.0.1:15432:5432"
         healthcheck:
-            test: ["CMD", "pg_isready", "--username", "cratesfyi"]
-            interval: 10s
-            timeout: 5s
-            retries: 10
+            << : *healthcheck-interval
+            test: pg_isready --username cratesfyi
 
     s3:
         image: minio/minio
@@ -65,17 +144,8 @@ services:
             MINIO_ROOT_USER: cratesfyi
             MINIO_ROOT_PASSWORD: secret_key
         healthcheck:
-            test:
-                [
-                    "CMD",
-                    "curl",
-                    "--silent",
-                    "--fail",
-                    "localhost:9000/minio/health/ready",
-                ]
-            interval: 10s
-            timeout: 5s
-            retries: 10
+            << : *healthcheck-interval
+            test: mc ready local
 
     prometheus:
         build:
@@ -84,11 +154,8 @@ services:
         ports:
             - "127.0.0.1:9090:9090"
         healthcheck:
-            test:
-                ["CMD", "curl", "--silent", "--fail", "localhost:9090/-/ready"]
-            interval: 10s
-            timeout: 5s
-            retries: 10
+            << : *healthcheck-interval
+            test: promtool check healthy
 
     gui_tests:
         build:
@@ -99,6 +166,8 @@ services:
             - "host.docker.internal:host-gateway"
         volumes:
             - "${PWD}:/build/out"
+        profiles:
+            - all
 
 volumes:
     postgres-data: {}
diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile
diff --git a/dockerfiles/entrypoint.sh b/dockerfiles/entrypoint.sh
