Remove builder-base and development image stages, run linting and tests outside Docker (#111)

* Remove `builder-base` and `development` image stages

Now, we only have two Dockerfile stages:
- `base`, where we install dependencies with poetry
- `production`, where we copy the source files and run the app

We use environment variables to run the app in a "nonprod" or "prod"
mode

We also add a .dockerignore file to limit the files that make it into the
production container

Small fixes:

* Make docker entrypoint executable

* Move gunicorn_conf.py to config dir

* Refactor Make rules
- add `docker-` prefix for shell and start rules
- add a local start rule
- modify help text
- Run lint, format, and test outside of Docker

Author: grahamalama

.dockerignore

@@ -0,0 +1,9 @@
+# ignore everything
+*
+# use exceptions to create an "allow list"
+!/config
+!/src
+!/bin
+!/poetry.lock
+!/pyproject.toml
+!/version.json

.secrets.baseline

@@ -115,16 +115,16 @@
         "filename": "config/local_dev.env",
         "hashed_secret": "4b9a4ce92b6a01a4cd6ee1672d31c043f2ae79ab",
         "is_verified": false,
-        "line_number": 5
+        "line_number": 7
       },
       {
         "type": "Secret Keyword",
         "filename": "config/local_dev.env",
         "hashed_secret": "77ea6398f252999314d609a708842a49fc43e055",
         "is_verified": false,
-        "line_number": 8
+        "line_number": 10
       }
     ]
   },
-  "generated_at": "2022-07-13T09:34:14Z"
+  "generated_at": "2022-07-14T17:13:15Z"
 }

Dockerfile

@@ -1,101 +1,46 @@
 # Creating a python base with shared environment variables
-FROM python:3.10.5-slim as python-base
-ENV PYTHONPATH=/app \
-    PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_NO_CACHE_DIR=off \
-    PIP_DISABLE_PIP_VERSION_CHECK=on \
+FROM python:3.10.5 as base
+ENV PIP_NO_CACHE_DIR=off \
     PIP_DEFAULT_TIMEOUT=100 \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
     POETRY_HOME="/opt/poetry" \
-    POETRY_VIRTUALENVS_IN_PROJECT=true \
     POETRY_NO_INTERACTION=1 \
-    PYSETUP_PATH="/opt/pysetup" \
-    VENV_PATH="/opt/pysetup/.venv" \
-    PORT=8000
-
-ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
-
-# Set up user and group
-ARG userid=10001
-ARG groupid=10001
-WORKDIR /app
-RUN groupadd --gid $groupid app && \
-    useradd -g app --uid $userid --shell /usr/sbin/nologin --create-home app
-
-RUN mkdir -p $POETRY_HOME && \
-    chown app:app /opt/poetry && \
-    mkdir -p $PYSETUP_PATH && \
-    chown app:app $PYSETUP_PATH && \
-    mkdir -p /app && \
-    chown app:app /app
-
-RUN apt-get update && \
-    apt-get install --assume-yes apt-utils && \
-    echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    PYSETUP_PATH="/opt/pysetup"
 
-# builder-base is used to build dependencies
-FROM python-base as builder-base
-RUN apt-get install --no-install-recommends -y \
-      curl \
-      build-essential
+ENV PATH="$POETRY_HOME/bin:$PATH"
 
 # Install Poetry - respects $POETRY_VERSION & $POETRY_HOME
-USER app
 ENV POETRY_VERSION=1.1.14
 RUN curl -sSL https://install.python-poetry.org | python3 -
 
 # We copy our Python requirements here to cache them
 # and install only runtime deps using poetry
 WORKDIR $PYSETUP_PATH
-COPY --chown=app:app ./poetry.lock ./pyproject.toml ./
+COPY ./poetry.lock ./pyproject.toml ./
 RUN poetry install --no-dev --no-root
 
+# `production` stage uses the dependencies downloaded in the `base` stage
+FROM python:3.10.5-slim as production
+ENV PROMETHEUS_MULTIPROC=1 \
+    PORT=8000 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    VIRTUAL_ENV=/opt/pysetup/.venv
 
-# 'development' stage installs all dev deps and can be used to develop code.
-# For example using docker-compose to mount local volume under /app
-FROM python-base as development
-# to run detect-secrets
-RUN apt-get install --no-install-recommends -y git
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-ENV FASTAPI_ENV=development
+COPY --from=base $VIRTUAL_ENV $VIRTUAL_ENV
 
-# Copying poetry and venv into image
+ARG userid=10001
+ARG groupid=10001
+RUN groupadd --gid $groupid app && \
+    useradd -g app --uid $userid --shell /usr/sbin/nologin --create-home app
 USER app
-COPY --from=builder-base $POETRY_HOME $POETRY_HOME
-COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
-
-# Copying in our entrypoint
-COPY --chown=app:app ./bin/docker-entrypoint.sh /docker-entrypoint.sh
-RUN chmod +x /docker-entrypoint.sh
-
-# venv already has runtime deps installed we get a quicker install
-WORKDIR $PYSETUP_PATH
-RUN poetry install --no-root
-
-WORKDIR /app
-COPY --chown=app:app . .
-
-EXPOSE $PORT
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD uvicorn src.app.api:app --reload --host=0.0.0.0 --port=$PORT
-
-
-# 'production' stage uses the clean 'python-base' stage and copyies
-# in only our runtime deps that were installed in the 'builder-base'
-FROM python-base as production
-ENV FASTAPI_ENV=production \
-    IS_GUNICORN=1 \
-    PROMETHEUS_MULTIPROC=1
-
-COPY --from=builder-base $VENV_PATH $VENV_PATH
-COPY ./bin/gunicorn_conf.py /gunicorn_conf.py
-
-COPY ./bin/docker-entrypoint.sh /docker-entrypoint.sh
-RUN chmod +x /docker-entrypoint.sh
 
-COPY . /app
 WORKDIR /app
+COPY . .
 
 EXPOSE $PORT
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD gunicorn -k uvicorn.workers.UvicornWorker -c /gunicorn_conf.py src.app.api:app
+ENTRYPOINT ["bin/docker-entrypoint.sh"]
+CMD ["gunicorn", "-k", "uvicorn.workers.UvicornWorker", "-c", "config/gunicorn_conf.py", "src.app.api:app"]

Makefile

@@ -4,45 +4,61 @@
 _UID ?= 10001
 _GID ?= 10001
 
+VENV := $(shell echo $${VIRTUAL_ENV-.venv})
+INSTALL_STAMP = $(VENV)/.install.stamp
+POETRY_VIRTUALENVS_IN_PROJECT = true
+
 .PHONY: help
 help:
 	@echo "Usage: make RULE"
 	@echo ""
 	@echo "JBI make rules:"
 	@echo ""
-	@echo "  build   - build docker containers"
-	@echo "  lint    - lint check for code"
-	@echo "  format  - run formatters (black, isort), fix in place"
-	@echo "  start   - run the API service"
+	@echo "Local"
+	@echo "  format        - run formatters (black, isort), fix in place"
+	@echo "  lint          - run linters"
+	@echo "  start         - run the API service locally"
+	@echo "  test          - run test suite"
 	@echo ""
-	@echo "  test        - run test suite"
-	@echo "  shell       - open a shell in the web container"
+	@echo "Docker"
+	@echo "  build         - build docker container"
+	@echo "  docker-start  - run the API service through docker"
+	@echo "  docker-shell  - open a shell in the web container"
 	@echo ""
-	@echo "  help    - see this text"
+	@echo "  help          - see this text"
 
+install: $(INSTALL_STAMP)
+$(INSTALL_STAMP): poetry.lock
+	@if [ -z $(shell command -v poetry 2> /dev/null) ]; then echo "Poetry could not be found. See https://python-poetry.org/docs/"; exit 2; fi
+	poetry install --no-root
+	touch $(INSTALL_STAMP)
 
 .PHONY: build
 build:
 	docker-compose build \
 		--build-arg userid=${_UID} --build-arg groupid=${_GID}
 
 .PHONY: format
-format:
+format: $(INSTALL_STAMP)
 	bin/lint.sh black --fix
 	bin/lint.sh isort --fix
 
 .PHONY: lint
-lint:
-	docker-compose run --rm web bin/lint.sh
-
-.PHONY: shell
-shell:
-	docker-compose run web /bin/sh
+lint: $(INSTALL_STAMP)
+	bin/lint.sh
 
 .PHONY: start
 start:
+	poetry run python -m src.app.api
+
+.PHONY: docker-shell
+docker-shell:
+	docker-compose run --rm web /bin/sh
+
+.PHONY: docker-start
+docker-start:
 	docker-compose up
 
 .PHONY: test
-test:
-	docker-compose run --rm web bin/test.sh
+test: $(INSTALL_STAMP)
+	bin/test.sh

bin/docker-entrypoint.sh

@@ -2,9 +2,6 @@
 
 set -e
 
-# activate our virtual environment here
-. /opt/pysetup/.venv/bin/activate
-
 # setup an empty directory for Prometheus metrics
 if [ ${PROMETHEUS_MULTIPROC:-0} -eq 1 ]; then
     prometheus_multiproc_dir=`mktemp -td prometheus.XXXXXXXXXX` || exit 1

bin/gunicorn_conf.py renamed to config/gunicorn_conf.py

@@ -1,5 +1,7 @@
 # Environment variables for the local development environment
 
+ENV=nonprod
+
 # Jira API Secrets
 JIRA_USERNAME="fake_jira_username"
 JIRA_API_KEY="fake_jira_api_key"

config/local_dev.env

@@ -1,15 +1,14 @@
 version: "3.8"
 services:
   web:
-    build:
-      context: .
-      target: development
-    volumes:
-      - .:/app
-    ports:
-      - ${PORT:-8000}:${PORT:-8000}
+    build: .
+    command: python -m src.app.api
+    env_file:
+      - config/local_dev.env
     # Let the init system handle signals for us.
     # among other things this helps shutdown be fast
     init: true
-    env_file:
-      - config/local_dev.env
+    ports:
+      - ${PORT:-8000}:${PORT:-8000}
+    volumes:
+      - .:/app

docker-compose.yaml

@@ -1,7 +1,7 @@
 import pytest
 from pydantic import ValidationError
 
-from bin.gunicorn_conf import GunicornSettings
+from config.gunicorn_conf import GunicornSettings
 
 
 def test_set_bind(monkeypatch):