From 787a5a5d2aee9ef1f74e1b6279ec43e585ab1969 Mon Sep 17 00:00:00 2001 From: rohsyl Date: Fri, 28 Mar 2025 00:06:03 +0100 Subject: [PATCH 1/9] build: cd workflow and and move to non-root image --- .github/workflows/cd.yml | 80 ++++++++++++++++++++++++++++++++++++++++ docker-compose.prod.yml | 7 +--- docker/Dockerfile | 39 ++++++++++++-------- web_entry_point.sh | 1 + 4 files changed, 106 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/cd.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..c548eef --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,80 @@ +name: Build and Deploy + +on: + workflow_dispatch: + push: + branches: + - 'main' + tags: + - 'v*' + pull_request: + branches: + - 'main' + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/Dockerfile + target: 'prod' + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + deploy: + runs-on: ubuntu-latest + needs: build + if: github.ref == 'refs/heads/main' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Copy docker-compose file over ssh + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.SSH_HOST }} + username: ${{ secrets.SSH_USER }} + key: ${{ secrets.SSH_PRIVATE_KEY }} + source: "docker-compose.prod.yml" + target: "~/staging.mapdb.cncnet.org" + + - name: SSH into server and deploy + uses: appleboy/ssh-action@v1.2.1 + with: + host: ${{ secrets.SSH_HOST }} + username: ${{ secrets.SSH_USER }} + key: ${{ secrets.SSH_PRIVATE_KEY }} + script: | + cd ~/staging.mapdb.cncnet.org + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + docker compose -f docker-compose.prod.yml pull + docker compose -f docker-compose.prod.yml down + docker compose -f docker-compose.prod.yml up -d \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 68222e3..517e3c5 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -16,10 +16,7 @@ services: django: container_name: mapdb-django - build: - context: . - dockerfile: docker/Dockerfile - target: prod + image: ghcr.io/cncnet/cncnet-map-db:latest volumes: - ${HOST_MEDIA_ROOT}:/data/cncnet_silo # django will save user-uploaded files here. MEDIA_ROOT - ${HOST_STATIC_ROOT}:/data/cncnet_static # django will gather static files here. STATIC_ROOT @@ -39,6 +36,6 @@ services: - ${HOST_STATIC_ROOT}:/usr/share/nginx/html/static # website static assets. - ${HOST_MEDIA_ROOT}:/usr/share/nginx/html/silo # website user-uploaded files. ports: - - "80:80" + - "${EXPOSED_PORT}:80" depends_on: - django diff --git a/docker/Dockerfile b/docker/Dockerfile index 3b80d99..c30c3c0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,33 +1,40 @@ FROM python:3.12-bookworm AS base +# Base environment configuration ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 + +# Configurable user setup +ENV USER=cncnet +ENV UID=1000 + WORKDIR /cncnet-map-api -# This is just here to crash the build if you don't make a .env file. -COPY .env /cncnet-map-api/ +# Install system dependencies +RUN apt-get update && apt-get install -y liblzo2-dev libmagic1 + +# Create non-root user with configurable name and UID +RUN useradd -m -u ${UID} ${USER} -# Copy files needed for build +# Copy necessary files for the build COPY requirements.txt /cncnet-map-api COPY requirements-dev.txt /cncnet-map-api COPY web_entry_point.sh /cncnet-map-api +COPY .env /cncnet-map-api/ + +# Set permissions and make script executable +RUN chmod +x /cncnet-map-api/web_entry_point.sh && \ + chown -R ${USER}:${USER} /cncnet-map-api -# liblzo2 is a compression library used by westwood. -# libmagic1 is used for detecting file mime types by analyzing the file contents. -RUN apt-get update && apt-get install -y liblzo2-dev libmagic1 RUN pip install --upgrade pip -RUN chmod +x /cncnet-map-api/web_entry_point.sh FROM base AS dev -# The cflags are needed to build the lzo library on Apple silicon. -RUN CFLAGS=-I$(brew --prefix)/include LDFLAGS=-L$(brew --prefix)/lib pip install -r ./requirements-dev.txt -ENTRYPOINT "/cncnet-map-api/web_entry_point.sh" +RUN pip install -r ./requirements-dev.txt +USER ${USER} +ENTRYPOINT ["/cncnet-map-api/web_entry_point.sh"] FROM base AS prod -# The cflags are needed to build the lzo library on Apple silicon. -RUN CFLAGS=-I$(brew --prefix)/include LDFLAGS=-L$(brew --prefix)/lib pip install -r ./requirements.txt - COPY . /cncnet-map-api - -ENTRYPOINT "/cncnet-map-api/web_entry_point.sh" - +RUN pip install -r ./requirements.txt +USER ${USER} +ENTRYPOINT ["/cncnet-map-api/web_entry_point.sh"] diff --git a/web_entry_point.sh b/web_entry_point.sh index 3019f9d..a363cc3 100755 --- a/web_entry_point.sh +++ b/web_entry_point.sh @@ -1,3 +1,4 @@ +#!/bin/sh # Don't run this file manually. It's the entry point for the dockerfile. python manage.py collectstatic --noinput python manage.py migrate From 73c6060e6abe5fe235d8e342f1ba94671a6036cc Mon Sep 17 00:00:00 2001 From: rohsyl Date: Wed, 9 Apr 2025 20:39:43 +0200 Subject: [PATCH 2/9] build: cd wip --- .github/workflows/cd.yml | 38 ++++++++++++++++++++++++++++++-------- docker/nginx.prod.conf | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 docker/nginx.prod.conf diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index c548eef..e78b09c 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -5,11 +5,13 @@ on: push: branches: - 'main' + - 'staging' tags: - 'v*' pull_request: branches: - 'main' + - 'staging' jobs: build: @@ -50,21 +52,41 @@ jobs: deploy: runs-on: ubuntu-latest + permissions: + contents: read + packages: read needs: build - if: github.ref == 'refs/heads/main' + strategy: + matrix: + include: + - environment: staging + branch: staging + target_path: "~/staging.mapdb.cncnet.org" + compose_file: "docker-compose.prod.yml" + nginx_conf: "docker/nginx.prod.conf" +# disabled for now +# - environment: production +# branch: main +# target_path: "~/prod2.mapdb.cncnet.org" +# compose_file: "docker-compose.prod.yml" +# nginx_conf: "docker/nginx.prod.conf" steps: + - name: "Exit if not matching branch" + if: github.ref != format('refs/heads/{0}', matrix.branch) + run: echo "Not target branch for this deployment. Skipping..." && exit 0 + - name: Checkout repository uses: actions/checkout@v4 - - name: Copy docker-compose file over ssh + - name: Copy docker-compose and nginx config over ssh uses: appleboy/scp-action@v0.1.7 with: host: ${{ secrets.SSH_HOST }} username: ${{ secrets.SSH_USER }} key: ${{ secrets.SSH_PRIVATE_KEY }} - source: "docker-compose.prod.yml" - target: "~/staging.mapdb.cncnet.org" + source: "${{ matrix.compose_file }},${{ matrix.nginx_conf }}" + target: "${{ matrix.target_path }}" - name: SSH into server and deploy uses: appleboy/ssh-action@v1.2.1 @@ -73,8 +95,8 @@ jobs: username: ${{ secrets.SSH_USER }} key: ${{ secrets.SSH_PRIVATE_KEY }} script: | - cd ~/staging.mapdb.cncnet.org + cd ${{ matrix.target_path }} echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - docker compose -f docker-compose.prod.yml pull - docker compose -f docker-compose.prod.yml down - docker compose -f docker-compose.prod.yml up -d \ No newline at end of file + docker compose -f ${{ matrix.compose_file }} pull + docker compose -f ${{ matrix.compose_file }} down + docker compose -f ${{ matrix.compose_file }} up -d \ No newline at end of file diff --git a/docker/nginx.prod.conf b/docker/nginx.prod.conf new file mode 100644 index 0000000..40340e0 --- /dev/null +++ b/docker/nginx.prod.conf @@ -0,0 +1,34 @@ +user nginx; +worker_processes 4; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + server { + listen 80; + + # Serve static files: js, static images, etc. + location /static/ { + alias /usr/share/nginx/html/static/; # The nginx container's mounted volume. + expires 30d; + add_header Cache-Control public; + } + + # Serve user uploaded files + location /silo/ { + alias /usr/share/nginx/html/silo/; # The container's mounted volume. + } + + # Proxy requests to the Django app running in gunicorn + location / { + proxy_pass http://django:8000; # The Django app is exposed on the `django` container on port 8000 + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +} From 04ea75df5c657234d9120502bd08691d5ada916d Mon Sep 17 00:00:00 2001 From: rohsyl Date: Wed, 9 Apr 2025 20:41:41 +0200 Subject: [PATCH 3/9] build: cd wip --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c30c3c0..8249e54 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,6 @@ RUN useradd -m -u ${UID} ${USER} COPY requirements.txt /cncnet-map-api COPY requirements-dev.txt /cncnet-map-api COPY web_entry_point.sh /cncnet-map-api -COPY .env /cncnet-map-api/ # Set permissions and make script executable RUN chmod +x /cncnet-map-api/web_entry_point.sh && \ From fc8f07857dbbec9bad02ff12eda645e5e6ec8f5e Mon Sep 17 00:00:00 2001 From: rohsyl Date: Wed, 9 Apr 2025 20:48:53 +0200 Subject: [PATCH 4/9] build: cd wip --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 517e3c5..e181f46 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -32,7 +32,7 @@ services: container_name: mapdb-nginx image: nginx:latest volumes: - - ./nginx.conf:/etc/nginx/nginx.conf:ro + - ./docker/nginx.prod.conf:/etc/nginx/nginx.conf:ro - ${HOST_STATIC_ROOT}:/usr/share/nginx/html/static # website static assets. - ${HOST_MEDIA_ROOT}:/usr/share/nginx/html/silo # website user-uploaded files. ports: From 44f70165bcb31b06d3576dd7d07d0a8087d47b98 Mon Sep 17 00:00:00 2001 From: rohsyl Date: Wed, 9 Apr 2025 21:33:21 +0200 Subject: [PATCH 5/9] build: cd wip --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e181f46..7b759d8 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -16,7 +16,7 @@ services: django: container_name: mapdb-django - image: ghcr.io/cncnet/cncnet-map-db:latest + image: ghcr.io/cncnet/cncnet-map-api:latest volumes: - ${HOST_MEDIA_ROOT}:/data/cncnet_silo # django will save user-uploaded files here. MEDIA_ROOT - ${HOST_STATIC_ROOT}:/data/cncnet_static # django will gather static files here. STATIC_ROOT From 4a86ef06c22daadbbf367d974f0e1e35ac0fb370 Mon Sep 17 00:00:00 2001 From: rohsyl Date: Wed, 9 Apr 2025 21:40:42 +0200 Subject: [PATCH 6/9] build: cd wip --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 7b759d8..073aa14 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -16,7 +16,7 @@ services: django: container_name: mapdb-django - image: ghcr.io/cncnet/cncnet-map-api:latest + image: ghcr.io/cncnet/cncnet-map-api:${APP_TAG} volumes: - ${HOST_MEDIA_ROOT}:/data/cncnet_silo # django will save user-uploaded files here. MEDIA_ROOT - ${HOST_STATIC_ROOT}:/data/cncnet_static # django will gather static files here. STATIC_ROOT From 15d3605ac65cb922481ba1e257420f6a63d047ff Mon Sep 17 00:00:00 2001 From: rohsyl Date: Wed, 9 Apr 2025 22:04:27 +0200 Subject: [PATCH 7/9] build: cd wip --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 4937bb4..a58e77e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ gunicorn>=22.0.0,<23.0.0 django-filter==25.1 orjson>=3.10.15,<4.0 structlog>=25.1.0,<26.0.0 +drf-spectacular[sidecar] From ce5752f48671e407d5bd5900fc086b5ddfdaac70 Mon Sep 17 00:00:00 2001 From: rohsyl Date: Wed, 9 Apr 2025 22:22:54 +0200 Subject: [PATCH 8/9] build: expose only necessary ports on prod --- docker-compose.prod.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 073aa14..c8d5a12 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -10,9 +10,9 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} env_file: - .env - ports: - - "127.0.0.1:${POSTGRES_PORT}:${POSTGRES_PORT}" command: -p ${POSTGRES_PORT} + networks: + - mapdb-network django: container_name: mapdb-django @@ -20,12 +20,12 @@ services: volumes: - ${HOST_MEDIA_ROOT}:/data/cncnet_silo # django will save user-uploaded files here. MEDIA_ROOT - ${HOST_STATIC_ROOT}:/data/cncnet_static # django will gather static files here. STATIC_ROOT - ports: - - "8000:8000" env_file: - .env depends_on: - db + networks: + - mapdb-network nginx-server: # nginx proxies requests to django via gunicorn. @@ -39,3 +39,9 @@ services: - "${EXPOSED_PORT}:80" depends_on: - django + networks: + - mapdb-network + +networks: + mapdb-network: + driver: bridge \ No newline at end of file From c4a78a8d33546de4babba5c9dc5a552bd598850f Mon Sep 17 00:00:00 2001 From: rohsyl Date: Wed, 9 Apr 2025 22:44:03 +0200 Subject: [PATCH 9/9] build: add missing allowed hosts --- kirovy/settings/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kirovy/settings/_base.py b/kirovy/settings/_base.py index 58ce179..0f0b661 100644 --- a/kirovy/settings/_base.py +++ b/kirovy/settings/_base.py @@ -31,7 +31,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = get_env_var("DEBUG", False, validation_callback=not_allowed_on_prod) -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = get_env_var("ALLOWED_HOSTS", "localhost,mapdb-nginx").split(",") MAX_UPLOADED_FILE_SIZE_MAP = file_utils.ByteSized(mega=25)