Getting Dockerflow ready

Author: Bryan Sieber

Makefile

@@ -0,0 +1,54 @@
+# Set these in the environment to override them. This is helpful for
+# development if you have file ownership problems because the user
+# in the container doesn't match the user on your host.
+_UID ?= 10001
+_GID ?= 10001
+
+.PHONY: help
+help:
+	@echo "Usage: make RULE"
+	@echo ""
+	@echo "JBI make rules:"
+	@echo ""
+	@echo "  build   - build docker containers"
+	@echo "  lint    - lint check for code"
+	@echo "  start   - run the API service"
+	@echo ""
+	@echo "  test        - run test suite"
+	@echo "  shell       - open a shell in the web container"
+	@echo "  test-shell  - open a shell in test environment"
+	@echo ""
+	@echo "  help    - see this text"
+
+
+.PHONY: build
+build:
+	docker-compose -f ./docker-compose.yaml -f ./tests/infra/docker-compose.test.yaml build \
+		--build-arg userid=${_UID} --build-arg groupid=${_GID}
+
+.PHONY: lint
+lint:
+	docker-compose -f ./docker-compose.yaml -f ./tests/infra/docker-compose.lint.yaml build \
+		--build-arg userid=${_UID} --build-arg groupid=${_GID} lint
+
+
+.PHONY: shell
+shell:
+	docker-compose -f ./docker-compose.yaml run web
+
+.PHONY: start
+start:
+	docker-compose up
+
+.PHONY: test
+test:
+	docker-compose -f ./docker-compose.yaml -f ./tests/infra/docker-compose.test.yaml run tests
+ifneq (1, ${MK_KEEP_DOCKER_UP})
+	# Due to https://github.com/docker/compose/issues/2791 we have to explicitly
+	# rm all running containers
+	docker-compose down
+endif
+
+.PHONY: test-shell
+test-shell: .env
+	docker-compose -f ./docker-compose.yaml -f ./tests/infra/docker-compose.test.yaml run web

docker-compose.yaml

@@ -0,0 +1,19 @@
+version: "3.8"
+services:
+  web:
+    build:
+      context: .
+      dockerfile: ./infra/Dockerfile
+      target: development
+    volumes:
+      - type: bind
+        source: .
+        target: /app
+    ports:
+      - ${PORT:-8000}:${PORT:-8000}
+    # Let the init system handle signals for us.
+    # among other things this helps shutdown be fast
+    init: true
+#    env_file:
+#      - ./infra/config/local_dev.env
+#      - .env

infra/Dockerfile

@@ -0,0 +1,118 @@
+# Creating a python base with shared environment variables
+FROM python:3.9.9-slim as python-base
+ENV PYTHONPATH=/app \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    POETRY_NO_INTERACTION=1 \
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv" \
+    PORT=8000
+
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+# Set up user and group
+ARG userid=10001
+ARG groupid=10001
+WORKDIR /app
+RUN groupadd --gid $groupid app && \
+    useradd -g app --uid $userid --shell /usr/sbin/nologin --create-home app
+
+RUN mkdir -p $POETRY_HOME && \
+    chown app:app /opt/poetry && \
+    mkdir -p $PYSETUP_PATH && \
+    chown app:app $PYSETUP_PATH && \
+    mkdir -p /app && \
+    chown app:app /app
+
+RUN apt-get update && \
+    apt-get install --assume-yes apt-utils && \
+    echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \
+    apt-get install --no-install-recommends -y \
+      libpq5
+
+# builder-base is used to build dependencies
+FROM python-base as builder-base
+RUN apt-get install --no-install-recommends -y \
+      curl \
+      build-essential \
+      libpq-dev
+
+# Install Poetry - respects $POETRY_VERSION & $POETRY_HOME
+USER app
+ENV POETRY_VERSION=1.1.5
+RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python
+
+# We copy our Python requirements here to cache them
+# and install only runtime deps using poetry
+WORKDIR $PYSETUP_PATH
+COPY --chown=app:app ./poetry.lock ./pyproject.toml ./
+RUN poetry install --no-dev --no-root
+
+
+# 'development' stage installs all dev deps and can be used to develop code.
+# For example using docker-compose to mount local volume under /app
+FROM python-base as development
+ENV FASTAPI_ENV=development
+
+# Copying poetry and venv into image
+USER app
+COPY --from=builder-base $POETRY_HOME $POETRY_HOME
+COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
+
+# Copying in our entrypoint
+COPY --chown=app:app ./infra/docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint.sh
+
+# venv already has runtime deps installed we get a quicker install
+WORKDIR $PYSETUP_PATH
+RUN poetry install --no-root
+
+WORKDIR /app
+COPY --chown=app:app . .
+
+EXPOSE $PORT
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD uvicorn src.app.api:app --reload --host=0.0.0.0 --port=$PORT
+
+
+# 'lint' stage runs similar checks to pre-commit / scripts/lint.sh
+# running in check mode means build will fail if any linting errors occur
+FROM development AS lint
+RUN bandit -lll --recursive src --exclude "src/poetry.lock,src/.venv,src/.mypy,src/build"
+RUN mypy src
+RUN black --config ./pyproject.toml --check src tests
+RUN isort --recursive --settings-path ./pyproject.toml --check-only src
+RUN pylint src tests/unit
+CMD ./infra/lint.sh
+
+
+# 'test' stage runs our unit tests with pytest and
+# coverage.  Build will fail if test coverage is under 80%
+FROM development AS test
+CMD ./infra/test.sh
+
+
+# 'production' stage uses the clean 'python-base' stage and copyies
+# in only our runtime deps that were installed in the 'builder-base'
+FROM python-base as production
+ENV FASTAPI_ENV=production \
+    IS_GUNICORN=1 \
+    PROMETHEUS_MULTIPROC=1
+
+COPY --from=builder-base $VENV_PATH $VENV_PATH
+COPY ./infra/gunicorn_conf.py /gunicorn_conf.py
+
+COPY ./infra/docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint.sh
+
+COPY . /app
+WORKDIR /app
+
+EXPOSE $PORT
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD gunicorn -k uvicorn.workers.UvicornWorker -c /gunicorn_conf.py src.app.api:app

infra/docker-entrypoint.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+# activate our virtual environment here
+. /opt/pysetup/.venv/bin/activate
+
+# setup an empty directory for Prometheus metrics
+if [ ${PROMETHEUS_MULTIPROC:-0} -eq 1 ]; then
+    prometheus_multiproc_dir=`mktemp -td prometheus.XXXXXXXXXX` || exit 1
+    export prometheus_multiproc_dir
+fi
+
+# Evaluating passed command:
+exec "$@"
+
+if [ -z ${prometheus_multiproc_dir+x} ]; then
+    rm -rf ${prometheus_multiproc_dir}
+fi

infra/gunicorn_conf.py

@@ -0,0 +1,76 @@
+# From: https://github.com/tiangolo/uvicorn-gunicorn-docker/blob/2daa3e3873c837d5781feb4ff6a40a89f791f81b/docker-images/gunicorn_conf.py
+# pylint: disable=invalid-name
+
+import json
+import multiprocessing
+import os
+
+from prometheus_client import multiprocess
+
+workers_per_core_str = os.getenv("WORKERS_PER_CORE", "1")
+max_workers_str = os.getenv("MAX_WORKERS")
+use_max_workers = None
+if max_workers_str:
+    use_max_workers = int(max_workers_str)
+web_concurrency_str = os.getenv("WEB_CONCURRENCY", None)
+
+host = os.getenv("HOST", "0.0.0.0")
+port = os.getenv("PORT", "80")
+bind_env = os.getenv("BIND", None)
+use_loglevel = os.getenv("LOG_LEVEL", "info")
+if bind_env:
+    use_bind = bind_env
+else:
+    use_bind = f"{host}:{port}"
+
+cores = multiprocessing.cpu_count()
+workers_per_core = float(workers_per_core_str)
+default_web_concurrency = workers_per_core * cores
+if web_concurrency_str:
+    web_concurrency = int(web_concurrency_str)
+    assert web_concurrency > 0
+else:
+    web_concurrency = max(int(default_web_concurrency), 2)
+    if use_max_workers:
+        web_concurrency = min(web_concurrency, use_max_workers)
+accesslog_var = os.getenv("ACCESS_LOG", "-")
+use_accesslog = accesslog_var or None
+errorlog_var = os.getenv("ERROR_LOG", "-")
+use_errorlog = errorlog_var or None
+graceful_timeout_str = os.getenv("GRACEFUL_TIMEOUT", "120")
+timeout_str = os.getenv("TIMEOUT", "120")
+keepalive_str = os.getenv("KEEP_ALIVE", "5")
+
+# Gunicorn config variables
+loglevel = use_loglevel
+workers = web_concurrency
+bind = use_bind
+errorlog = use_errorlog
+worker_tmp_dir = "/dev/shm"
+accesslog = use_accesslog
+graceful_timeout = int(graceful_timeout_str)
+timeout = int(timeout_str)
+keepalive = int(keepalive_str)
+
+
+# For debugging and testing
+log_data = {
+    "loglevel": loglevel,
+    "workers": workers,
+    "bind": bind,
+    "graceful_timeout": graceful_timeout,
+    "timeout": timeout,
+    "keepalive": keepalive,
+    "errorlog": errorlog,
+    "accesslog": accesslog,
+    # Additional, non-gunicorn variables
+    "workers_per_core": workers_per_core,
+    "use_max_workers": use_max_workers,
+    "host": host,
+    "port": port,
+}
+print(json.dumps(log_data))
+
+
+def child_exit(server, worker):
+    multiprocess.mark_process_dead(worker.pid)

infra/lint.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -e
+
+CURRENT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+BASE_DIR="$(dirname "$CURRENT_DIR")"
+HAS_GIT="$(command -v git || echo '')"
+echo $HAS_GIT
+
+bandit -lll --recursive "${BASE_DIR}" --exclude "${BASE_DIR}/poetry.lock,${BASE_DIR}/.venv,${BASE_DIR}/.mypy,${BASE_DIR}/build"
+
+if [ -n "$HAS_GIT" ]; then
+    # Scan only files checked into the repo, omit poetry.lock
+    SECRETS_TO_SCAN=`git ls-tree --full-tree -r --name-only HEAD | grep -v poetry.lock`
+    detect-secrets-hook $SECRETS_TO_SCAN --baseline .secrets.baseline
+fi
+
+mypy "${BASE_DIR}"
+black --config "${BASE_DIR}/pyproject.toml" "${BASE_DIR}"
+isort --recursive --settings-path "${BASE_DIR}/pyproject.toml" "${BASE_DIR}"
+pylint src tests/unit

infra/test.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+CURRENT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+BASE_DIR="$(dirname "$CURRENT_DIR")"
+
+coverage run --rcfile "${BASE_DIR}/pyproject.toml" -m pytest
+coverage report --rcfile "${BASE_DIR}/pyproject.toml" -m --fail-under 80
+coverage html --rcfile "${BASE_DIR}/pyproject.toml"

poetry.lock

@@ -11,6 +11,7 @@ fastapi = "^0.73.0"
 pydantic = "^1.9.0"
 uvicorn = {extras = ["standard"], version = "^0.17.4"}
 gunicorn = "^20.1.0"
+prometheus-client = "^0.13.1"
 
 [tool.poetry.dev-dependencies]
 pre-commit = "^2.17.0"
@@ -39,6 +40,7 @@ testpaths = [
         "C0114", #missing-module-docstring
         "C0115", #missing-class-docstring
         "C0116", #missing-function-docstring
+        "C0301", #line-too-long
         "R0903", #too-few-public-methods
         "W0613", #unused-argument
     ]