Merge branch 'master' into stage

Author: yvess

Makefile

@@ -1,4 +1,8 @@
 SHELL = /bin/bash
+DOCKER_EXEC_DJANGO=$(shell command -v docker > /dev/null && echo "docker-compose exec -T django")
+DOCKER_EXEC_VUE=$(shell command -v docker > /dev/null && echo "docker-compose exec vue")
+DOCKER_EXEC_WWW=$(shell command -v docker > /dev/null && echo "docker-compose exec www")
+
 .PHONY: tests
 
 init:
@@ -17,10 +21,26 @@ push:
 	cd vue && make push
 
 enter_django:
-	cd django && make enter
+	$(DOCKER_EXEC_DJANGO) ash
 
 enter_vue:
-	cd vue && make enter
+	$(DOCKER_EXEC_VUE) ash
+
+enter_www:
+	$(DOCKER_EXEC_WWW) ash
+
+start_vue:
+	$(DOCKER_EXEC_VUE) make
+
+start_django:
+	$(DOCKER_EXEC_DJANGO) make
+
+start_screenshotservice:
+	$(DOCKER_EXEC_VUE) make screenshotservice
+
+reload_www:
+	$(DOCKER_EXEC_WWW) sh -c 'openresty -t & openresty -s reload'
+
 
 deploy_prod:
 	cd vue && make build

README.md

@@ -24,7 +24,7 @@ For a tour of the app and a testing protocol see [`docs/testing.md`](/docs/testi
 
 ## setup project
 
-Add `django` to the 127.0.0.1 entry in `/etc/hosts` (for screenshot service).
+Add `www.local` and `django` to the 127.0.0.1 entry in `/etc/hosts` (for screenshot service).
 
 ```bash
 touch env.hosts.prod # required file, can be empty and edited later
@@ -66,6 +66,19 @@ Database files and static generated files are stored in `var` folder.
 make up
 ```
 
+**start services**
+
+```bash
+# 1. shell
+make start_vue
+# 2. shell
+make start_django
+# 3. shell
+make start_screenshotservice
+```
+
+You can also start the vue / django service from the vscode shell (see editor).
+
 ### tests
 
 ```bash

django/gsmap/admin.py

@@ -4,13 +4,25 @@
 from django_json_widget.widgets import JSONEditorWidget
 from django.utils.html import mark_safe
 from django.contrib import messages
+from django.forms.widgets import Textarea
 import requests
 from sortedm2m_filter_horizontal_widget.forms import SortedFilteredSelectMultiple
 from gsmap.models import Municipality, Snapshot, Workspace
 
 
 class MunicipalityAdmin(admin.OSMGeoAdmin):
-    pass
+    readonly_fields = ('bfs_number',)
+    fields = ('bfs_number', 'name', 'canton', 'perimeter')
+    list_display = (
+        'name',
+        'bfs_number',
+    )
+    list_filter = ('canton',)
+    search_fields = ('id', 'name', 'canton')
+
+    def get_map_widget(self, db_field):
+        return Textarea
+
 
 
 class SnapshotAdmin(admin.OSMGeoAdmin):
@@ -92,7 +104,17 @@ def save_model(self, request, obj, form, change):
         try:
             obj.save()
         except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e:
-            messages.error(request, "Couldn't create the screenshots, screenshot server problem.")
+            print(repr(e))
+            messages.error(
+                request,
+                "Couldn't create the screenshots, screenshot server problem. (ReadTimeout, ConnectionError)"
+            )
+        except Exception as e:
+            print(repr(e))
+            messages.error(
+                request,
+                "Couldn't create the screenshots, screenshot server problem. (Other Error)"
+            )
 
 
 class WorkspaceAdmin(admin.OSMGeoAdmin):

django/gsmap/models.py

@@ -70,6 +70,10 @@ class Meta:
     def fullname(self):
         return f'{self.name} ({self.canton})'
 
+    @property
+    def bfs_number(self):
+        return self.id
+
     def __str__(self):
         return self.fullname
 
@@ -195,20 +199,6 @@ def create_screenshot_file(self, is_thumbnail=False):
         )
         return screenshot_file
 
-    def save(self, *args, **kwargs):
-        def test_exists(pk):
-            if list(self.__class__.objects.filter(pk=pk)):
-                new_id = create_slug_hash_6()
-                test_exists(new_id)
-            else:
-                return pk
-
-        if self._state.adding:
-            self.id = create_slug_hash_6()
-            self.id = test_exists(self.id)
-
-        super().save(*args, **kwargs)
-
     def image_twitter(self):
         if bool(self.screenshot):
             return get_thumbnail(
@@ -231,53 +221,58 @@ def __str__(self):
         else:
             return self.title
 
+    def save(self, *args, **kwargs):
+        def test_exists(pk):
+            if list(self.__class__.objects.filter(pk=pk)):
+                new_id = create_slug_hash_6()
+                test_exists(new_id)
+            else:
+                return pk
+
+        if self._state.adding:
+            self.id = create_slug_hash_6()
+            self.id = test_exists(self.id)
 
-@receiver(post_save, sender=Snapshot)
-def save_screenshot_handler(sender, **kwargs):
-    instance = kwargs.get('instance')
+        if self.data:
+            storage = OverwriteStorage()
+            if self.permission is int(SnapshotPermission.PUBLIC):
+                self.create_meta(storage)
+            else:
+                storage.delete(f'snapshot-meta/{self.id}.html')
 
-    def save_screenshot():
-        post_save.disconnect(save_screenshot_handler, sender=Snapshot)
+        super().save(*args, **kwargs)
+
+        if hasattr(settings, 'SAVE_SCREENSHOT_ENABLED') and settings.SAVE_SCREENSHOT_ENABLED is True:
+            self.create_screenshot()
+
+        super().save(*args, **kwargs)
+
+    def create_screenshot(self):
         # only create snapshot if data changed
-        if instance.data_changed([
+        if self.data_changed([
                 'data', 'screenshot_generated', 'thumbnail_generated'
-        ]) or not bool(instance.thumbnail_generated):
-            if not 'resources' in instance.data:
-                return
-            try:
-                # disconnect to break save recursive loop
-                post_save.disconnect(save_screenshot_handler, sender=Snapshot)
-                screenshot_file = instance.create_screenshot_file()
-                thumbnail_file = instance.create_screenshot_file(is_thumbnail=True)
-                instance.screenshot_generated = screenshot_file
-                instance.thumbnail_generated = thumbnail_file
-                instance.save()
-            finally:
-                # always reconnect signal
-                post_save.connect(save_screenshot_handler, sender=Snapshot)
-
-    def save_meta(storage):
+        ]) or not bool(self.thumbnail_generated):
+            print('resources', 'resources' in self.data)
+            if not 'resources' in self.data:
+                raise ValueError('no resources key in data')
+
+            screenshot_file = self.create_screenshot_file()
+            thumbnail_file = self.create_screenshot_file(is_thumbnail=True)
+            self.screenshot_generated = screenshot_file
+            self.thumbnail_generated = thumbnail_file
+
+    def create_meta(self, storage):
         domain = Site.objects.get_current().domain
         proto = 'https' if settings.USE_HTTPS else 'http'
         meta = f'''
-<meta property="og:title" content="{instance.title_data}">
-<meta property="og:description" content="{instance.description_data}">
+<meta property="og:title" content="{self.title_data}">
+<meta property="og:description" content="{self.description_data}">
 <meta property="og:type" content="website">
-<meta property="og:url" content="{proto}://{domain}{instance.get_absolute_url()}">
-<meta property="og:image" content="{proto}://{domain}/{instance.image_facebook()}">
-<meta name="twitter:image" content="{proto}://{domain}/{instance.image_twitter()}">
+<meta property="og:url" content="{proto}://{domain}{self.get_absolute_url()}">
+<meta property="og:image" content="{proto}://{domain}/{self.image_facebook()}">
+<meta name="twitter:image" content="{proto}://{domain}/{self.image_twitter()}">
 '''
-        storage.save(f'snapshot-meta/{instance.id}.html', ContentFile(meta))
-
-    if hasattr(settings, 'SAVE_SCREENSHOT_ENABLED') and settings.SAVE_SCREENSHOT_ENABLED is True:
-        save_screenshot()
-
-    if instance.data:
-        storage = OverwriteStorage()
-        if instance.permission is int(SnapshotPermission.PUBLIC):
-            save_meta(storage)
-        else:
-            storage.delete(f'snapshot-meta/{instance.id}.html')
+        storage.save(f'snapshot-meta/{self.id}.html', ContentFile(meta))
 
 
 class Workspace(models.Model):

django/main/settings.py

@@ -129,6 +129,8 @@
 CORS_ORIGIN_WHITELIST = [
     "http://localhost:8080",
     "http://localhost:8081",
+    "http://www:8000",
+    "http://www.local:8000",
 ]
 CORS_ALLOW_CREDENTIALS = True
 CACHES = {

docker-compose.yml

@@ -27,14 +27,16 @@ services:
     environment:
       - TERM=xterm-256color
       - START_SCREENSHOTSERVICE=NO
-      - PUBLIC_URL=http://localhost:8080
+      - PUBLIC_URL=http://www:8000
+      - VUE_APP_MAPBOX_ACCESSTOKEN=pk.eyJ1IjoiZ2VtZWluZGVzY2FuIiwiYSI6ImNrNXdrOXV2ZjEwdXcza28wcXNoYXE1ZGMifQ.P-nCvxNRK4AOs7YdRbHCLg
+      - VUE_APP_MAPBOX_DEFAULT_STYLES=mapbox://styles/gemeindescan/ck6rp249516tg1iqkmt48o4pz
     working_dir: /opt/vue
     volumes:
       - ./vue:/opt/vue
       - var-gemeindescan-webui:/var/services/
       - vscode-remote-vue:/root/.vscode-server
     ports:
-      # - "9222:9222" # debug chrome
+      - "9222:9222" # debug chrome
       - "8079:8079" # snapshost service
       - "10001:10001" # vscode
 
@@ -53,14 +55,12 @@ services:
       - DJANGO_HTTPS=False
       - DJANGO_EMAIL_HOST=maildev
       - DJANGO_SECRET_KEY=adddxxccxx4rt0ioywg3yf5kjq64-t5
-      - DJANGO_ALLOWED_HOSTS=local.gemeindescan.ch,localhost,django
+      - DJANGO_ALLOWED_HOSTS=www,www.local,localhost,django
     working_dir: /opt/app
     volumes:
       - ./django:/opt/app
       - var-gemeindescan-webui:/var/services
       - vscode-remote-django:/root/.vscode-server
-    links:
-     - pdb
     ports:
       - "10002:10002" # vscode
     # entrypoint: tail
@@ -72,9 +72,6 @@ services:
     volumes:
       - ./etc/nginx:/etc/nginx/conf.d
       - var-gemeindescan-webui:/var/services
-    links:
-      - django
-      - vue
     ports:
-      - 8080:80
-    command: tail -f /dev/null
+      - 8000:8000
+    # command: tail -f /dev/null

etc/nginx/localhost.conf renamed to etc/nginx/www.local.conf

@@ -7,11 +7,11 @@ upstream vue-dev {
 }
 
 server {
-    listen 80;
-    server_name localhost;
+    listen 8000;
+    server_name www www.local;
 
-    access_log /var/log/localhost.access.log;
-    error_log /var/log/localhost.error.log error;
+    access_log /var/log/local.www.access.log;
+    error_log /var/log/local.www.error.log error;
 
     client_max_body_size 10M;
 
@@ -20,7 +20,7 @@ server {
     location = / {
         set $lang_sup "de,fr";
         set_by_lua_file $lang_accept /etc/nginx/conf.d/lang.lua $lang_sup;
-        return $scheme://$host:8080/$lang_accept;
+        return $scheme://$host:8000/$lang_accept;
     }
 
 
@@ -89,7 +89,7 @@ server {
 
     # vue dev server mode hotreload
     location /sockjs-node {
-        proxy_pass http://vue:8080;
+        proxy_pass http://vue-dev;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;

vue/.env

@@ -1,3 +1,2 @@
-VUE_APP_GRAPHQL_URI=http://localhost:8080/graphql/
-VUE_APP_MAPBOX_ACCESSTOKEN=pk.eyJ1IjoiZ2VtZWluZGVzY2FuIiwiYSI6ImNrNXdrOXV2ZjEwdXcza28wcXNoYXE1ZGMifQ.P-nCvxNRK4AOs7YdRbHCLg
-VUE_APP_DJANGOBASEURL=http://localhost:8080
+VUE_APP_GRAPHQL_URI=http://www:8000/graphql/
+VUE_APP_DJANGOBASEURL=http://www:8000

vue/.env.production

@@ -1,3 +1,2 @@
 VUE_APP_GRAPHQL_URI=/graphql/
-VUE_APP_MAPBOX_ACCESSTOKEN=pk.eyJ1IjoiZ2VtZWluZGVzY2FuIiwiYSI6ImNrNXdrOXV2ZjEwdXcza28wcXNoYXE1ZGMifQ.P-nCvxNRK4AOs7YdRbHCLg
 VUE_APP_DJANGOBASEURL=

vue/screenshot-service/screenshot-server.js

@@ -18,7 +18,10 @@ const browserFactory = {
         devtools: false,
         args: [
           '--no-sandbox',
-          '--disable-dev-shm-usage'
+          '--disable-dev-shm-usage',
+          '--disable-setuid-sandbox',
+          '--disable-accelerated-2d-canvas',
+          '--disable-gpu'
           // '--remote-debugging-port=9222',
           // '--remote-debugging-address=0.0.0.0'
         ]
@@ -71,9 +74,12 @@ app.get('*', async (req, res) => {
 
   const page = await browserPool.acquire();
   page.setViewport(pageOptions);
-  await page.goto(screenshotUrl);
+  await page.goto(screenshotUrl, {
+    waitLoad: true,
+    waitNetworkIdle: true
+  });
   await page.waitForSelector('#mapinfo .v-list-item__title');
-  await page.waitFor(1000); // extra 1sec wait, for map tiles
+  // await page.waitFor(1000); // extra 1sec wait, for map tiles
   const screenshotBuffer = await page.screenshot({ encoding: 'binary' });
   // await browser.close();
   await browserPool.release(page);