Feat/docker setup improvements (#74)

Setup:
* updated README.md for auto linked www.local.conf
* check if nginx config www.local.conf symlink/file exits, otherwise create it automatically
* set default SECRET_KEY for test or dev setup
* remove need for symlink to node_modules, updated docker vue image fixes security issues, updated to docker image gemeindescan-webui-vue:0.3.8
* fixes import order error
* needs optionals in docker image, updated docker vue image, vue composition api was also added before
* renamed docker-compose.yml to docker-compose.dev.yml
* added docker-compose.yml to ignore
* added _DEV env vars to docker-compose.dev.yml for dev out of the box settings
* Setup symlink for docker compose dev file to default
* new alpine openresty image which has arm64 and amd64 images

Testing:
* added DOCKER_NOTTY option to Makefiles for test runs
* added init-test to Makefile for noinput test setup, use bash in Makefile for more features (return true)
* run django and vue tests in two separate steps for different reports
* Fixed django & vue tests

Cleanup:
* deleted old not used make command for deployment

Co-authored-by: André Glatzl <[email protected]>

Author: yvess

.github/workflows/tests.yml

@@ -27,8 +27,10 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Setup
-        run: touch env.hosts.prod && docker-compose pull && make up && make init
+        run: ln -s docker-compose.dev.yml docker-compose.yml && docker-compose pull && make up && DOCKER_NOTTY=YES make init-test
 
-      # Runs a single command using the runners shell
-      - name: Run tests
-        run: make tests
+      - name: Run django tests
+        run: cd django && DOCKER_NOTTY=YES make tests
+
+      - name: Run vue tests
+        run: cd vue && DOCKER_NOTTY=YES make tests

.gitignore

@@ -6,3 +6,4 @@ server.yml
 docker-prod.yml
 docker-compose-prod.yml
 __pycache__/
+docker-compose.yml

Makefile

@@ -2,7 +2,6 @@ SHELL = /bin/bash
 DOCKER_EXEC_DJANGO=$(shell command -v docker > /dev/null && echo "docker-compose exec django")
 DOCKER_EXEC_VUE=$(shell command -v docker > /dev/null && echo "docker-compose exec vue")
 DOCKER_EXEC_WWW=$(shell command -v docker > /dev/null && echo "docker-compose exec www")
-
 DOCKER_CRON_VUE=$(shell command -v docker > /dev/null && echo "docker-compose exec -T vue")
 DOCKER_CRON_DJANGO=$(shell command -v docker > /dev/null && echo "docker-compose exec -T django")
 
@@ -11,7 +10,10 @@ DOCKER_CRON_DJANGO=$(shell command -v docker > /dev/null && echo "docker-compose
 
 init:
 	cd django && make init
-	cd vue && make init
+
+init-test:
+	sleep 10 # wait for db setup
+	cd django && make init-test
 
 up:
 	docker-compose up -d
@@ -38,7 +40,7 @@ enter_vue:
 	$(DOCKER_EXEC_VUE) ash
 
 enter_www:
-	$(DOCKER_EXEC_WWW) ash
+	$(DOCKER_EXEC_WWW) bash
 
 start_all: up
 	$(DOCKER_CRON_VUE) make &
@@ -57,23 +59,6 @@ start_screenshotservice:
 reload_www:
 	$(DOCKER_EXEC_WWW) sh -c 'openresty -t & openresty -s reload'
 
-
-deploy_prod:
-	cd vue && make build
-	source env.hosts.prod && ssh $$DJANGO_PROD_HOST -t "cd $$DJANGO_PROD_PATH && git pull -v"
-	source env.hosts.prod && ssh $$DJANGO_PROD_HOST -t "cd $$DJANGO_PROD_PATH && COMPOSE_FILE=$$COMPOSE_PROD docker-compose up -d"
-	source env.hosts.prod && rsync -av --delete vue/dist $$DJANGO_PROD_HOST:$$VUE_PROD_PATH
-	source env.hosts.prod && ssh $$DJANGO_PROD_HOST -t "cd $$DJANGO_PROD_PATH && COMPOSE_FILE=$$COMPOSE_PROD docker-compose exec django make migrate"
-	source env.hosts.prod && ssh $$DJANGO_PROD_HOST -t "cd $$DJANGO_PROD_PATH && COMPOSE_FILE=$$COMPOSE_PROD docker-compose exec django killall -TERM gunicorn"
-
-deploy_dev:
-	cd vue && make build
-	source env.hosts.prod && ssh $$DJANGO_DEV_HOST -t "cd $$DJANGO_DEV_PATH && git pull -v"
-	source env.hosts.prod && ssh $$DJANGO_DEV_HOST -t "cd $$DJANGO_DEV_PATH && COMPOSE_FILE=$$COMPOSE_DEV docker-compose up -d"
-	source env.hosts.prod && rsync -av --delete vue/dist $$DJANGO_DEV_HOST:$$VUE_DEV_PATH
-	source env.hosts.prod && ssh $$DJANGO_DEV_HOST -t "cd $$DJANGO_DEV_PATH && COMPOSE_FILE=$$COMPOSE_DEV docker-compose exec django make migrate"
-	source env.hosts.prod && ssh $$DJANGO_DEV_HOST -t "cd $$DJANGO_DEV_PATH && COMPOSE_FILE=$$COMPOSE_DEV docker-compose exec django killall -TERM gunicorn"
-
 deploy_local:
 	docker-compose up -d
 	make -f vue/Makefile build-cron

README.md

@@ -33,11 +33,8 @@ For a tour of the app and a testing protocol see [`docs/testing.md`](./docs/test
 (1) Prepare local configuration
 
 - Add `www.local` and `django` to the 127.0.0.1 entry in `/etc/hosts` (for screenshot service).
-- For local development add a symlink to the nginx configuration file, for production use a dedicated copy:
-```bash
-ln -s etc/nginx/www.local.dev etc/nginx/www.local.conf
-```
-- Create an `.env` file with the following environment variables:
+- A symlink to the nginx configuration file in the www container will be created automatically (`/etc/nginx/conf.d/www.local.conf`) for production use a dedicated copy
+- for production or custom settings add an `.env` file with the following environment variables:
 ```bash
 DJANGO_SECRET=some.R4nd0m_k3y
 MAPBOX_TOKEN=a.mapbox.api.token
@@ -64,10 +61,6 @@ python3 manage.py createsuperuser
 make import-gemeinden-json
 # exit django container
 exit
-# enter vue container
-maker enter_vue
-ln -s /node_modules ./
-```
 
 For development we use a named docker volume for the `var` folder inside the container, this named
 volume will be created automatically. This makes setup under windows easier.

django/Makefile

@@ -1,6 +1,9 @@
 DJANGO_VERSION=3.1.4
 DJANGO_VERSION_PREV=0.3.4
-DOCKER_EXEC=$(shell command -v docker > /dev/null && echo "docker-compose exec django")
+NOTTY=$(shell command [ "$$DOCKER_NOTTY" = YES ] && echo "-T")
+DOCKER_EXEC=$(shell command -v docker > /dev/null && echo "docker-compose exec $(NOTTY) django")
+SHELL := /bin/bash
+
 all: dev run-tests run-tests-dev
 .PHONY: all
 default: dev
@@ -26,11 +29,14 @@ runprod:
 adminuser:
 	$(DOCKER_EXEC) python3 manage.py createsuperuser
 
+adminuser-noinput:
+	$(DOCKER_EXEC) python3 manage.py createsuperuser --noinput --username admin --email [email protected]
+
 collectstatic:
 	$(DOCKER_EXEC) python3 manage.py collectstatic --noinput
 
 symlinks:
-	$(DOCKER_EXEC) sh -c "cd /var/services/django; ln -s media downloads"
+	$(DOCKER_EXEC) bash -c "mkdir -p /var/services/django/media && cd /var/services/django; [ ! -e /var/services/django/downloads ] && ln -s media downloads" || true
 
 migrate:
 	$(DOCKER_EXEC) python3 manage.py migrate
@@ -40,7 +46,7 @@ createcachetable:
 	$(DOCKER_EXEC) python3 manage.py createcachetable
 
 tests:
-	$(DOCKER_EXEC) pytest --create-db
+	$(DOCKER_EXEC) pytest --create-db -vvv
 
 tests-dev:
 	$(DOCKER_EXEC) pytest --reuse-db -vvv --nomigrations
@@ -60,3 +66,10 @@ init:
 	make createcachetable
 	make adminuser
 	make import-gemeinden-json
+
+init-test:
+	make symlinks
+	make migrate
+	make createcachetable
+	make adminuser-noinput
+	make import-gemeinden-json

django/conftest.py

@@ -5,7 +5,7 @@
 @pytest.fixture(scope='session')
 def django_db_setup(django_db_setup, django_db_blocker):
     with django_db_blocker.unblock():
-        call_command('loaddata', 'fixtures-tests/sites.site.json')
-        call_command('loaddata', 'fixtures-tests/gsuser.user.json')
-        call_command('loaddata', 'fixtures-tests/gsmap.municipality.json')
-        call_command('loaddata', 'fixtures-tests/gsmap.snapshot.json')
+        call_command('loaddata', '/opt/app/fixtures-tests/sites.site.json', verbosity=2)
+        call_command('loaddata', '/opt/app/fixtures-tests/gsuser.user.json', verbosity=2)
+        call_command('loaddata', '/opt/app/fixtures-tests/gsmap.municipality.json', verbosity=2)
+        call_command('loaddata', '/opt/app/fixtures-tests/gsmap.snapshot.json', verbosity=2)

django/gsmap/tests.py

@@ -127,7 +127,7 @@ def test_snapshots_exists():
                         'pk': 'R4RPGC',
                         'title': 'test snapshot',
                         'topic': 'test topic',
-                        'screenshot': ''
+                        'screenshot': None
                     }
                 }]
             }

django/main/settings.py

@@ -1,6 +1,7 @@
 import os
 
-SECRET_KEY = os.getenv('DJANGO_SECRET_KEY')
+# use your own secret_key, default for testing and dev
+SECRET_KEY = os.getenv('DJANGO_SECRET_KEY') or os.getenv('DJANGO_SECRET_KEY_DEV')
 DEBUG = os.getenv('DJANGO_DEBUG') == 'True'
 USE_HTTPS = os.getenv('DJANGO_HTTPS') == 'True'
 

docker-compose.yml renamed to docker-compose.dev.yml

@@ -28,7 +28,9 @@ services:
       - TERM=xterm-256color
       - START_SCREENSHOTSERVICE=NO
       - VUE_APP_MAPBOX_ACCESSTOKEN=${MAPBOX_TOKEN}
+      - VUE_APP_MAPBOX_ACCESSTOKEN_DEV=pk.eyJ1IjoiZ2VtZWluZGVzY2FuIiwiYSI6ImNrNXdrOXV2ZjEwdXcza28wcXNoYXE1ZGMifQ.P-nCvxNRK4AOs7YdRbHCLg
       - VUE_APP_MAPBOX_DEFAULT_STYLES=${MAPBOX_STYLE}
+      - VUE_APP_MAPBOX_DEFAULT_STYLES_DEV=mapbox://styles/gemeindescan/ck6rp249516tg1iqkmt48o4pz
       - VUE_APP_GRAPHQL_URI=http://www:8000/graphql/
       - VUE_APP_DJANGOBASEURL=http://www:8000
       - PUBLIC_URL=http://www:8000
@@ -50,6 +52,7 @@ services:
       - TERM=linux
       - START_DJANGO=NO
       - DJANGO_SECRET_KEY=${DJANGO_SECRET}
+      - DJANGO_SECRET_KEY_DEV=adddxxccxx4rt0ioywg3yf5kjq64-t5
       - DJANGO_DB_NAME=webui
       - DJANGO_DB_USER=webui
       - DJANGO_DB_PASSWORD=pgpass
@@ -73,11 +76,15 @@ services:
     # command: -f /dev/null
 
   www:
-    image: openresty/openresty:1.15.8.3-alpine
-    hostname: www
+    extends:
+      file: docker-images.yml
+      service: www
     volumes:
       - ./etc/nginx:/etc/nginx/conf.d:Z
       - var-gemeindescan-webui:/var/services
     ports:
       - 8000:8000
+    command: >
+      sh -c '[ ! -e /etc/nginx/conf.d/www.local.conf ] && ln -s /etc/nginx/conf.d/www.local.dev /etc/nginx/conf.d/www.local.conf;
+      /usr/local/openresty/bin/openresty -g "daemon off;"'
     # command: tail -f /dev/null

docker-images.yml

@@ -6,9 +6,13 @@ services:
     hostname: pdb
 
   vue:
-    image: smartuse/gemeindescan-webui-vue:0.3.7
+    image: smartuse/gemeindescan-webui-vue:0.3.8
     hostname: vue
 
   django:
     image: smartuse/gemeindescan-webui-django:3.1.4
     hostname: django
+
+  www:
+    image: openresty/openresty:1.19.3.1-alpine-fat
+    hostname: www