Overhaul the Docker setup

This patch rewrites the Docker setup used to create a deployable
container running a production LNT server.

- Split the Docker image into two, a basic image with LNT installed
  and an image with the actual production server as an entry point.
- Relocate all of the Docker-related files under docker/.
- Document the various Docker-related files.
- Improve input validation in the docker entrypoint.
- Using proper Docker secrets to transmit sensitive information to
  the LNT webserver entry point and the Postgres database.
- Update to Postgres 18. We might as well use the latest version
  available since we're standing this up from scratch.

With this setup, I am able to spin up a local LNT server instance with:

     docker compose --file docker/compose.yaml --env-file <secrets> up

An example of a secrets file would be

     LNT_DB_PASSWORD=foo
     LNT_AUTH_TOKEN=bar

Author: ldionne

.github/workflows/build-docker.yaml

@@ -32,4 +32,6 @@ jobs:
       with:
         push: ${{ startsWith(github.ref, 'refs/tags/') }} # only push to ghcr.io on tags
         tags: ghcr.io/${{github.repository}}:latest
+        file: docker/lnt.dockerfile
+        target: llvm-lnt
         context: . # use the current directory as context, as checked out by actions/checkout -- needed for setuptools_scm

Dockerfile

@@ -0,0 +1,64 @@
+# This file composes a full service running LNT. A LNT service is comprised
+# of a container running a Postgres database and a container running a
+# production LNT webserver.
+#
+# In order to build the full service, some secrets are required. They are taken
+# as environment variables, which means they can be stored in a file and included
+# via `--env-file <secrets-file>`. These environment variables are:
+#
+#   LNT_DB_PASSWORD
+#     The password to use for logging into the database.
+#
+#   LNT_AUTH_TOKEN
+#     The authentication token used to require authentication to
+#     perform destructive actions.
+
+name: llvm-lnt-prod
+
+services:
+  webserver:
+    container_name: webserver
+    build:
+      context: ../
+      dockerfile: docker/lnt.dockerfile
+      target: llvm-lnt-prod
+      args:
+        DB_USER: lntuser
+        DB_HOST: dbserver
+        DB_NAME: lnt.db
+    secrets:
+      - lnt-db-password
+      - lnt-auth-token
+    depends_on:
+      - db
+    deploy:
+      restart_policy:
+        condition: on-failure
+    ports:
+      - "8000:8000"
+    volumes:
+      - instance:/var/lib/lnt
+      - logs:/var/log/lnt
+
+  db:
+    container_name: dbserver
+    image: docker.io/postgres:18-alpine
+    environment:
+      - POSTGRES_PASSWORD_FILE=/run/secrets/lnt-db-password
+      - POSTGRES_USER=lntuser
+      - POSTGRES_DB=lnt.db
+    secrets:
+      - lnt-db-password
+    volumes:
+      - database:/var/lib/postgresql
+
+volumes:
+  instance:
+  logs:
+  database:
+
+secrets:
+  lnt-db-password:
+    environment: "LNT_DB_PASSWORD"
+  lnt-auth-token:
+    environment: "LNT_AUTH_TOKEN"

docker-compose.yaml

@@ -1,21 +1,46 @@
 #!/bin/sh
 
-DB_PATH=${DB_ENGINE:-postgresql}://${DB_USER:-lntuser}:${DB_PWD:?}@${DB_HOST:?}
-DB_BASE=${DB_BASE:-lnt}
+if [ -z ${DB_USER+x} ]; then
+    echo "Missing DB_USER environment variable"
+    exit 1
+fi
+
+if [ -z ${DB_HOST+x} ]; then
+    echo "Missing DB_HOST environment variable"
+    exit 1
+fi
+
+if [ -z ${DB_NAME+x} ]; then
+    echo "Missing DB_NAME environment variable"
+    exit 1
+fi
+
+if [ ! -f /run/secrets/lnt-db-password ]; then
+    echo "Missing secret lnt-db-password"
+    exit 1
+fi
+DB_PASSWORD="$(cat /run/secrets/lnt-db-password)"
+
+if [ ! -f /run/secrets/lnt-auth-token ]; then
+    echo "Missing secret lnt-auth-token"
+    exit 1
+fi
+AUTH_TOKEN="$(cat /run/secrets/lnt-auth-token)"
+
+DB_PATH="postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}"
 
-if [ ! -r /etc/lnt/lnt.cfg ]; then
-    DB_BASE_PATH="${DB_PATH}/${DB_BASE}" wait_db
+# Set up the instance the first time this gets run.
+if [ ! -e /var/lib/lnt/instance/lnt.cfg ]; then
+    lnt-wait-db "${DB_PATH}/${DB_NAME}"
     lnt create /var/lib/lnt/instance    \
-        --config /etc/lnt/lnt.cfg       \
         --wsgi lnt_wsgi.py              \
         --tmp-dir /tmp/lnt              \
         --db-dir "${DB_PATH}"           \
-        --default-db "${DB_BASE}"
-    if [ -n "${LNT_AUTH_TOKEN:-}" ]; then
-        sed -i "s/# \(api_auth_token =\).*/\1 '${LNT_AUTH_TOKEN}'/" /etc/lnt/lnt.cfg
-    fi
+        --default-db "${DB_NAME}"
+    sed -i "s/# \(api_auth_token =\).*/\1 '${AUTH_TOKEN}'/" /var/lib/lnt/instance/lnt.cfg
 fi
 
+# Run the server under gunicorn.
 cd /var/lib/lnt/instance
 exec gunicorn lnt_wsgi:application                      \
     --bind 0.0.0.0:8000                                 \
@@ -24,4 +49,4 @@ exec gunicorn lnt_wsgi:application                      \
     --name lnt_server                                   \
     --log-file /var/log/lnt/lnt.log                     \
     --access-logfile /var/log/lnt/gunicorn_access.log   \
-    --max-requests 250000 "$@"
+    --max-requests 250000

docker/compose.yaml

@@ -1,11 +1,13 @@
 #!/usr/bin/env python
 
-import os
-from sqlalchemy import create_engine
+import sys
+import sqlalchemy
 
-db_base_path = os.environ['DB_BASE_PATH']
+if len(sys.argv) != 2:
+    raise "Missing db path for lnt-wait-db"
 
-engine = create_engine(db_base_path)
+db = sys.argv[1]
+engine = sqlalchemy.create_engine(db)
 started = False
 
 while not started:

docker/docker-entrypoint.sh

@@ -0,0 +1,52 @@
+# This Dockerfile defines a basic 'llvm-lnt' image that contains an installed
+# copy of LNT. That image can be built and run with:
+#
+#   $ docker build --file docker/lnt.dockerfile --target llvm-lnt .
+#   $ docker run -it <sha> /bin/sh
+#
+# It also defines a 'llvm-lnt-prod' image which is set up to run a production
+# LNT server. This image is intended to be built from a Docker Compose file,
+# as it requires additional information like secrets and build arguments:
+#
+#   ARG DB_USER
+#     The username to use for logging into the database.
+#
+#   ARG DB_HOST
+#     The hostname to use to access the database.
+#
+#   ARG DB_NAME
+#     The name of the database on the server.
+#
+#   secret: lnt-db-password
+#     The password to use for logging into the database.
+#
+#   secret: lnt-auth-token
+#     The authentication token used to require authentication to
+#     perform destructive actions.
+
+FROM python:3.10-alpine AS llvm-lnt
+
+# Install dependencies
+RUN apk update \
+  && apk add --no-cache --virtual .build-deps git g++ postgresql-dev yaml-dev \
+  && apk add --no-cache libpq
+
+# Install LNT itself
+COPY . /var/tmp/lnt
+WORKDIR /var/tmp/lnt
+RUN pip3 install -r requirements.server.txt && apk --purge del .build-deps
+
+
+FROM llvm-lnt AS llvm-lnt-prod
+
+# Prepare volumes that will be used by the server
+VOLUME /var/lib/lnt /var/log/lnt
+
+# Set up the actual entrypoint that gets run when the container starts.
+COPY docker/docker-entrypoint.sh docker/lnt-wait-db /usr/local/bin/
+ARG DB_USER DB_HOST DB_NAME
+ENV DB_USER=${DB_USER}
+ENV DB_HOST=${DB_HOST}
+ENV DB_NAME=${DB_NAME}
+ENTRYPOINT ["docker-entrypoint.sh"]
+EXPOSE 8000

docker/wait_db renamed to docker/lnt-wait-db

@@ -110,3 +110,14 @@ To install the extra packages for the server config::
     gunicorn app_wrapper:app --bind 0.0.0.0:8000 --workers 8 --timeout 300 --name lnt_server --log-file /var/log/lnt/lnt.log --access-logfile /var/log/lnt/gunicorn_access.log --max-requests 250000
 
 
+Running a LNT Server via Docker
+-------------------------------
+
+We provide a Docker Compose setup with Docker containers that can be used to
+easily bring up a fully working production server within minutes. The container
+can be built and run with::
+
+   docker compose --file docker/compose.yaml --env-file <secrets> up
+
+``<secrets>`` should be the path to a file containing environment variables
+required by the containers. Please refer to the Docker Compose file for details.