Overhaul the Docker setup (#117)

This patch rewrites the Docker setup used to create a deployable
container running a production LNT server.

- Split the Docker image into two, a basic image with LNT installed
  and an image with the actual production server as an entry point.
- Relocate all of the Docker-related files under docker/.
- Document the various Docker-related files.
- Improve input validation in the docker entrypoint.
- Using proper Docker secrets to transmit sensitive information to
  the LNT webserver entry point and the Postgres database.
- Update to Postgres 18. We might as well use the latest version
  available since we're standing this up from scratch.

With this setup, I am able to spin up a local LNT server instance with:

     docker compose --file docker/compose.yaml --env-file <secrets> up

An example of a secrets file would be

     LNT_DB_PASSWORD=foo
     LNT_AUTH_TOKEN=bar

Author: ldionne

.github/workflows/build-docker.yaml

@@ -32,4 +32,5 @@ jobs:
       with:
         push: ${{ github.ref == 'refs/heads/main' }} # only push to ghcr.io on the main branch
         tags: ghcr.io/${{github.repository}}:latest
+        file: docker/lnt.dockerfile
         context: . # use the current directory as context, as checked out by actions/checkout -- needed for setuptools_scm

Dockerfile

@@ -0,0 +1,65 @@
+# This file composes a full service running LNT. A LNT service is comprised
+# of a container running a Postgres database and a container running a
+# production LNT webserver.
+#
+# In order to build the full service, some secrets are required. They are taken
+# as environment variables, which means they can be stored in a file and included
+# via `--env-file <secrets-file>`. These environment variables are:
+#
+#   LNT_DB_PASSWORD
+#     The password to use for logging into the database.
+#
+#   LNT_AUTH_TOKEN
+#     The authentication token used to require authentication to
+#     perform destructive actions.
+
+name: llvm-lnt
+
+services:
+  webserver:
+    container_name: webserver
+    build:
+      context: ../
+      dockerfile: docker/lnt.dockerfile
+    environment:
+      - DB_USER=lntuser
+      - DB_HOST=dbserver
+      - DB_NAME=lnt.db
+      - DB_PASSWORD_FILE=/run/secrets/lnt-db-password
+      - AUTH_TOKEN_FILE=/run/secrets/lnt-auth-token
+    secrets:
+      - lnt-db-password
+      - lnt-auth-token
+    depends_on:
+      - db
+    deploy:
+      restart_policy:
+        condition: on-failure
+    ports:
+      - "8000:8000"
+    volumes:
+      - instance:/var/lib/lnt
+      - logs:/var/log/lnt
+
+  db:
+    container_name: dbserver
+    image: docker.io/postgres:13-alpine
+    environment:
+      - POSTGRES_PASSWORD_FILE=/run/secrets/lnt-db-password
+      - POSTGRES_USER=lntuser
+      - POSTGRES_DB=lnt.db
+    secrets:
+      - lnt-db-password
+    volumes:
+      - database:/var/lib/postgresql
+
+volumes:
+  instance:
+  logs:
+  database:
+
+secrets:
+  lnt-db-password:
+    environment: "LNT_DB_PASSWORD"
+  lnt-auth-token:
+    environment: "LNT_AUTH_TOKEN"

docker-compose.yaml

@@ -1,21 +1,23 @@
 #!/bin/sh
 
-DB_PATH=${DB_ENGINE:-postgresql}://${DB_USER:-lntuser}:${DB_PWD:?}@${DB_HOST:?}
-DB_BASE=${DB_BASE:-lnt}
+set -u
 
-if [ ! -r /etc/lnt/lnt.cfg ]; then
-    DB_BASE_PATH="${DB_PATH}/${DB_BASE}" wait_db
+password="$(cat ${DB_PASSWORD_FILE})"
+token="$(cat ${AUTH_TOKEN_FILE})"
+DB_PATH="postgres://${DB_USER}:${password}@${DB_HOST}"
+
+# Set up the instance the first time this gets run.
+if [ ! -e /var/lib/lnt/instance/lnt.cfg ]; then
+    lnt-wait-db "${DB_PATH}/${DB_NAME}"
     lnt create /var/lib/lnt/instance    \
-        --config /etc/lnt/lnt.cfg       \
         --wsgi lnt_wsgi.py              \
         --tmp-dir /tmp/lnt              \
         --db-dir "${DB_PATH}"           \
-        --default-db "${DB_BASE}"
-    if [ -n "${LNT_AUTH_TOKEN:-}" ]; then
-        sed -i "s/# \(api_auth_token =\).*/\1 '${LNT_AUTH_TOKEN}'/" /etc/lnt/lnt.cfg
-    fi
+        --default-db "${DB_NAME}"
+    sed -i "s/# \(api_auth_token =\).*/\1 '${token}'/" /var/lib/lnt/instance/lnt.cfg
 fi
 
+# Run the server under gunicorn.
 cd /var/lib/lnt/instance
 exec gunicorn lnt_wsgi:application                      \
     --bind 0.0.0.0:8000                                 \
@@ -24,4 +26,4 @@ exec gunicorn lnt_wsgi:application                      \
     --name lnt_server                                   \
     --log-file /var/log/lnt/lnt.log                     \
     --access-logfile /var/log/lnt/gunicorn_access.log   \
-    --max-requests 250000 "$@"
+    --max-requests 250000

docker/compose.yaml

@@ -1,11 +1,13 @@
 #!/usr/bin/env python
 
-import os
-from sqlalchemy import create_engine
+import sys
+import sqlalchemy
 
-db_base_path = os.environ['DB_BASE_PATH']
+if len(sys.argv) != 2:
+    raise "Missing db path for lnt-wait-db"
 
-engine = create_engine(db_base_path)
+db = sys.argv[1]
+engine = sqlalchemy.create_engine(db)
 started = False
 
 while not started:

docker/docker-entrypoint.sh

@@ -0,0 +1,41 @@
+# This Dockerfile defines an image that contains a production LNT server.
+# This image is intended to be built from a Docker Compose file, as it
+# requires additional information passed as environment variables:
+#
+#   DB_USER
+#     The username to use for logging into the database.
+#
+#   DB_HOST
+#     The hostname to use to access the database.
+#
+#   DB_NAME
+#     The name of the database on the server.
+#
+#   DB_PASSWORD_FILE
+#     File containing the password to use for logging into the database.
+#
+#   AUTH_TOKEN_FILE
+#     File containing the authentication token used to require authentication
+#     to perform destructive actions.
+
+FROM python:3.10-alpine
+
+# Install dependencies
+RUN apk update \
+  && apk add --no-cache --virtual .build-deps git g++ postgresql-dev yaml-dev \
+  && apk add --no-cache libpq
+
+# Install LNT itself, without leaving behind any sources inside the image.
+RUN --mount=type=bind,source=.,target=./lnt-source \
+    cp -R lnt-source /tmp/lnt-src && \
+    cd /tmp/lnt-src && \
+    pip3 install -r requirements.server.txt && apk --purge del .build-deps && \
+    rm -rf /tmp/lnt-src
+
+# Prepare volumes that will be used by the server
+VOLUME /var/lib/lnt /var/log/lnt
+
+# Set up the actual entrypoint that gets run when the container starts.
+COPY docker/docker-entrypoint.sh docker/lnt-wait-db /usr/local/bin/
+ENTRYPOINT ["docker-entrypoint.sh"]
+EXPOSE 8000

docker/wait_db renamed to docker/lnt-wait-db

@@ -110,3 +110,14 @@ To install the extra packages for the server config::
     gunicorn app_wrapper:app --bind 0.0.0.0:8000 --workers 8 --timeout 300 --name lnt_server --log-file /var/log/lnt/lnt.log --access-logfile /var/log/lnt/gunicorn_access.log --max-requests 250000
 
 
+Running a LNT Server via Docker
+-------------------------------
+
+We provide a Docker Compose setup with Docker containers that can be used to
+easily bring up a fully working production server within minutes. The container
+can be built and run with::
+
+   docker compose --file docker/compose.yaml --env-file <secrets> up
+
+``<secrets>`` should be the path to a file containing environment variables
+required by the containers. Please refer to the Docker Compose file for details.