Merged in stage (pull request #26)

release 0.5.0

Author: yvess

Makefile

@@ -1,4 +1,8 @@
 SHELL = /bin/bash
+DOCKER_EXEC_DJANGO=$(shell command -v docker > /dev/null && echo "docker-compose exec django")
+DOCKER_EXEC_VUE=$(shell command -v docker > /dev/null && echo "docker-compose exec vue")
+DOCKER_EXEC_WWW=$(shell command -v docker > /dev/null && echo "docker-compose exec www")
+
 .PHONY: tests
 
 init:
@@ -17,10 +21,26 @@ push:
 	cd vue && make push
 
 enter_django:
-	cd django && make enter
+	$(DOCKER_EXEC_DJANGO) ash
 
 enter_vue:
-	cd vue && make enter
+	$(DOCKER_EXEC_VUE) ash
+
+enter_www:
+	$(DOCKER_EXEC_WWW) ash
+
+start_vue:
+	$(DOCKER_EXEC_VUE) make
+
+start_django:
+	$(DOCKER_EXEC_DJANGO) make
+
+start_screenshotservice:
+	$(DOCKER_EXEC_VUE) make screenshotservice
+
+reload_www:
+	$(DOCKER_EXEC_WWW) sh -c 'openresty -t & openresty -s reload'
+
 
 deploy_prod:
 	cd vue && make build

README.md

@@ -24,7 +24,7 @@ For a tour of the app and a testing protocol see [`docs/testing.md`](/docs/testi
 
 ## setup project
 
-Add `django` to the 127.0.0.1 entry in `/etc/hosts` (for screenshot service).
+Add `www.local` and `django` to the 127.0.0.1 entry in `/etc/hosts` (for screenshot service).
 
 ```bash
 touch env.hosts.prod # required file, can be empty and edited later
@@ -66,6 +66,30 @@ Database files and static generated files are stored in `var` folder.
 make up
 ```
 
+**start services**
+
+```bash
+# 1. shell
+make start_vue
+# 2. shell
+make start_django
+# 3. shell
+make start_screenshotservice
+```
+
+You can also start the vue / django service from the vscode shell (see editor).
+
+### tests
+
+```bash
+make tests
+```
+
+This command runs first the backend tests and afterwards the frontend test.
+
+For the backend tests a new test database is generated and the fixtures are loaded.
+
+
 ### editor
 
 The project is setup to work with visual studio code as editor. Any other editor is also fine.
@@ -101,4 +125,4 @@ Type `make deploy_prod` in the project root. The file `env.hosts.prod` needs to
 - [django](/docs/django.md)
 - [links](/docs/links.md)
 - [grahpql](/docs/graphql.md)
-- [troubleshooting](/docs/troubleshooting.md)
+- [troubleshooting](/docs/troubleshooting.md)

django/Makefile

@@ -1,6 +1,6 @@
 GEMEINDESCAN_WEBUI_DJANGO=0.3.4
 DOCKER_EXEC=$(shell command -v docker > /dev/null && echo "docker-compose exec django")
-all: run-tests run-tests-dev
+all: dev run-tests run-tests-dev
 .PHONY: all
 default: dev
 
@@ -25,6 +25,9 @@ adminuser:
 collectstatic:
 	$(DOCKER_EXEC) python3 manage.py collectstatic --noinput
 
+symlinks:
+	$(DOCKER_EXEC) sh -c "cd /var/services/django; ln -s media downloads"
+
 migrate:
 	$(DOCKER_EXEC) python3 manage.py migrate
 	make collectstatic
@@ -45,6 +48,7 @@ import-gemeinden-json:
 	$(DOCKER_EXEC) rm ./tmp/gemeinden.geojson
 
 init:
+	make symlinks
 	make migrate
 	make adminuser
 	make import-gemeinden-json

django/gsmap/admin.py

@@ -4,13 +4,26 @@
 from django_json_widget.widgets import JSONEditorWidget
 from django.utils.html import mark_safe
 from django.contrib import messages
+from django.forms.widgets import Textarea
+from django.conf import settings
 import requests
 from sortedm2m_filter_horizontal_widget.forms import SortedFilteredSelectMultiple
 from gsmap.models import Municipality, Snapshot, Workspace
 
 
 class MunicipalityAdmin(admin.OSMGeoAdmin):
-    pass
+    readonly_fields = ('bfs_number',)
+    fields = ('bfs_number', 'name', 'canton', 'perimeter')
+    list_display = (
+        'name',
+        'bfs_number',
+    )
+    list_filter = ('canton',)
+    search_fields = ('id', 'name', 'canton')
+
+    def get_map_widget(self, db_field):
+        return Textarea
+
 
 
 class SnapshotAdmin(admin.OSMGeoAdmin):
@@ -92,7 +105,15 @@ def save_model(self, request, obj, form, change):
         try:
             obj.save()
         except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e:
-            messages.error(request, "Couldn't create the screenshots, screenshot server problem.")
+            messages.error(
+                request,
+                f"Couldn't create the screenshots, screenshot server problem. (ReadTimeout, ConnectionError) {repr(e)}"
+            )
+        except Exception as e:
+            messages.error(
+                request,
+                f"Couldn't create the screenshots, screenshot server problem. (Other Error) {repr(e)}"
+            )
 
 
 class WorkspaceAdmin(admin.OSMGeoAdmin):
@@ -120,3 +141,4 @@ def formfield_for_manytomany(self, db_field, request=None, **kwargs):
 admin.site.register(Municipality, MunicipalityAdmin)
 admin.site.register(Snapshot, SnapshotAdmin)
 admin.site.register(Workspace, WorkspaceAdmin)
+admin.site.site_header = settings.ADMIN_NAME

django/gsmap/models.py

@@ -2,6 +2,7 @@
 import string
 from enum import IntFlag
 import requests
+import os
 from django.utils import timezone
 from django.db.models.signals import post_save
 from django.dispatch import receiver
@@ -11,11 +12,21 @@
 from django.contrib.sites.models import Site
 from django.utils.html import format_html
 from django.conf import settings
+from django.core.files.base import ContentFile
+from django.core.files.storage import FileSystemStorage
 from sortedm2m.fields import SortedManyToManyField
-from sorl.thumbnail import ImageField
+from sorl.thumbnail import ImageField, get_thumbnail
 from gsuser.models import User
 
 
+class OverwriteStorage(FileSystemStorage):
+    def get_available_name(self, name, max_length=None):
+        # If the filename already exists, remove it as if it was a true file system
+        if self.exists(name):
+            os.remove(os.path.join(settings.MEDIA_ROOT, name))
+        return name
+
+
 class Municipality(models.Model):
     class Meta:
         verbose_name_plural = 'municipalities'
@@ -59,6 +70,10 @@ class Meta:
     def fullname(self):
         return f'{self.name} ({self.canton})'
 
+    @property
+    def bfs_number(self):
+        return self.id
+
     def __str__(self):
         return self.fullname
 
@@ -144,10 +159,25 @@ def screenshot(self):
     def thumbnail(self):
         return self.thumbnail_manual or self.thumbnail_generated
 
+    @property
+    def title_data(self):
+        try:
+            return self.data['views'][0]['spec']['title']
+        except KeyError:
+            return self.title
+
+    @property
+    def description_data(self):
+        try:
+            return self.data['views'][0]['spec']['description']
+        except KeyError:
+            return ''
+
     def get_absolute_link(self):
         domain = Site.objects.get_current().domain
+        proto = 'https' if settings.USE_HTTPS else 'http'
         return format_html(
-            f'<a href="//{domain}{self.get_absolute_url()}" target="_blank">'
+            f'<a href="{proto}://{domain}{self.get_absolute_url()}" target="_blank">'
             f'{domain}{self.get_absolute_url()}</a>'
         )
     get_absolute_link.short_description = "Snapshot Url"
@@ -169,6 +199,28 @@ def create_screenshot_file(self, is_thumbnail=False):
         )
         return screenshot_file
 
+    def image_twitter(self):
+        if bool(self.screenshot):
+            return get_thumbnail(
+                self.screenshot, '1200x630',
+                crop='bottom', format='PNG'
+            )
+        return ''
+
+    def image_facebook(self):
+        if bool(self.screenshot):
+            return get_thumbnail(
+                self.screenshot, '1200x630',
+                crop='bottom', format='PNG'
+            )
+
+    def __str__(self):
+        if self.municipality:
+            return f'{self.municipality.fullname}, {self.title}, ' \
+                f'{self.id} ({self.get_permission_display()})'
+        else:
+            return self.title
+
     def save(self, *args, **kwargs):
         def test_exists(pk):
             if list(self.__class__.objects.filter(pk=pk)):
@@ -181,40 +233,46 @@ def test_exists(pk):
             self.id = create_slug_hash_6()
             self.id = test_exists(self.id)
 
+        if self.data:
+            storage = OverwriteStorage()
+            if self.permission is int(SnapshotPermission.PUBLIC):
+                self.create_meta(storage)
+            else:
+                storage.delete(f'snapshot-meta/{self.id}.html')
+
         super().save(*args, **kwargs)
 
-    def __str__(self):
-        if self.municipality:
-            return f'{self.municipality.fullname}, {self.title}, ' \
-                f'{self.id} ({self.get_permission_display()})'
-        else:
-            return self.title
+        if hasattr(settings, 'SAVE_SCREENSHOT_ENABLED') and settings.SAVE_SCREENSHOT_ENABLED is True:
+            self.create_screenshot()
 
+        super().save(*args, **kwargs)
 
-@receiver(post_save, sender=Snapshot)
-def save_screenshot_handler(sender, **kwargs):
-    def save_screenshot():
-        post_save.disconnect(save_screenshot_handler, sender=Snapshot)
-        instance = kwargs.get('instance')
+    def create_screenshot(self):
         # only create snapshot if data changed
-        if instance.data_changed([
+        if self.data_changed([
                 'data', 'screenshot_generated', 'thumbnail_generated'
-        ]) or not bool(instance.thumbnail_generated):
-            if not 'resources' in instance.data:
-                return
-            try:
-                # disconnect to break save recursive loop
-                post_save.disconnect(save_screenshot_handler, sender=Snapshot)
-                screenshot_file = instance.create_screenshot_file()
-                thumbnail_file = instance.create_screenshot_file(is_thumbnail=True)
-                instance.screenshot_generated = screenshot_file
-                instance.thumbnail_generated = thumbnail_file
-                instance.save()
-            finally:
-                # always reconnect signal
-                post_save.connect(save_screenshot_handler, sender=Snapshot)
-    if hasattr(settings, 'SAVE_SCREENSHOT_ENABLED') and settings.SAVE_SCREENSHOT_ENABLED is True:
-        save_screenshot()
+        ]) or not bool(self.thumbnail_generated):
+            print('resources', 'resources' in self.data)
+            if not 'resources' in self.data:
+                raise ValueError('no resources key in data')
+
+            screenshot_file = self.create_screenshot_file()
+            thumbnail_file = self.create_screenshot_file(is_thumbnail=True)
+            self.screenshot_generated = screenshot_file
+            self.thumbnail_generated = thumbnail_file
+
+    def create_meta(self, storage):
+        domain = Site.objects.get_current().domain
+        proto = 'https' if settings.USE_HTTPS else 'http'
+        meta = f'''
+<meta property="og:title" content="{self.title_data}">
+<meta property="og:description" content="{self.description_data}">
+<meta property="og:type" content="website">
+<meta property="og:url" content="{proto}://{domain}{self.get_absolute_url()}">
+<meta property="og:image" content="{proto}://{domain}/{self.image_facebook()}">
+<meta name="twitter:image" content="{proto}://{domain}/{self.image_twitter()}">
+'''
+        storage.save(f'snapshot-meta/{self.id}.html', ContentFile(meta))
 
 
 class Workspace(models.Model):
@@ -234,9 +292,10 @@ class Meta:
     snapshots = SortedManyToManyField(Snapshot)
 
     def get_absolute_link(self):
+        proto = 'https' if settings.USE_HTTPS else 'http'
         domain = Site.objects.get_current().domain
         return format_html(
-            f'<a href="//{domain}{self.get_absolute_url()}" target="_blank">'
+            f'<a href="{proto}://{domain}{self.get_absolute_url()}" target="_blank">'
             f'{domain}{self.get_absolute_url()}</a>'
         )
     get_absolute_link.short_description = "Workspace Url"
@@ -245,7 +304,6 @@ def get_absolute_url(self):
         first_id = self.snapshots.all().first().id
         return f'/{self.id}/{first_id}/'
 
-
     def save(self, *args, **kwargs):
         def test_exists(pk):
             if self.__class__.objects.filter(pk=pk):

django/gsmap/schema.py

@@ -8,7 +8,6 @@
 from graphene_django.types import DjangoObjectType
 from graphene_django.filter import DjangoFilterConnectionField
 from graphene_django.converter import convert_django_field
-from sorl.thumbnail import get_thumbnail
 from gsmap.models import Municipality, Snapshot, SnapshotPermission, Workspace
 
 
@@ -71,18 +70,10 @@ def resolve_thumbnail(self, info):
         return self.thumbnail
 
     def resolve_screenshot_facebook(self, info):
-        screenshot = get_thumbnail(self.screenshot,
-                                   '1200x630',
-                                   crop='bottom',
-                                   format='PNG')
-        return screenshot
+        return self.image_facebook()
 
     def resolve_screenshot_twitter(self, info):
-        screenshot = get_thumbnail(self.screenshot,
-                                   '1200x600',
-                                   crop='bottom',
-                                   format='PNG')
-        return screenshot
+        return self.image_twitter()
 
 
 class MunicipalityNode(DjangoObjectType):