Merge pull request #83 from nuest/main

Author: nuest

.gitignore

@@ -134,3 +134,6 @@ geodjango/
 tests-ui/screenshots/
 
 static/
+
+
+static/

README.md

@@ -42,7 +42,7 @@ The folder `/fixtures` contains some test data, either as an SQL command to inse
 [`jq`](https://stedolan.github.io/jq/) is used for pretty-printing of the output.
 
 ```bash
-# create dump after running test_data.sql:
+# create dump after creating/harvesting test data:
 python manage.py dumpdata --exclude=auth --exclude=contenttypes | jq > fixtures/test_data.json
 
 # load:
@@ -70,6 +70,9 @@ docker run --name optimetaPortalDB -p 5432:5432 -e POSTGRES_USER=optimeta -e POS
 python manage.py makemigrations
 python manage.py migrate
 
+# create cache table
+python manage.py createcachetable
+
 # collect static files
 python manage.py collectstatic --noinput
 
@@ -80,7 +83,7 @@ python manage.py runserver
 OPTIMAP_CACHE=dummy OPTIMAP_DEBUG=True python manage.py runserver
 ```
 
-Now open a browser at <http://127.0.0.1:8000/publications/map/> for the map and <http://127.0.0.1:8000/publications/api/> for the API.
+Now open a browser at <http://127.0.0.1:8000/>.
 
 ### Debug with VS Code
 
@@ -115,6 +118,17 @@ Configuration for debugging with VS Code:
 
 Add `EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend` to the `.env` file to have emails printed to the console instead of sent via SMTP.
 
+Alternatively, you can run a local STMP server with the following command and configuration:
+
+```bash
+python -m smtpd -c DebuggingServer -n localhost:5587
+```
+
+```env
+OPTIMAP_EMAIL_HOST=localhost
+OPTIMAP_EMAIL_PORT=5587
+```
+
 ### Create superusers/admin
 
 Superusers/admin can be created  using the createsuperuser command:
@@ -149,8 +163,6 @@ python -Wa manage.py test
 # running UI tests needs either compose configuration or a manage.py runserver in a seperate shell
 docker-compose up --build
 
-# TODO insert test data
-
 python -Wa manage.py test tests-ui
 ```
 

fixtures/test_data.json

@@ -10,7 +10,10 @@
       "url": null,
       "geometry": "SRID=4326;GEOMETRYCOLLECTION (POINT (7.595730774920725 51.96944097112328), POLYGON ((7.599984296478425 51.984257653537384, 7.5715788777530975 51.97057414651397, 7.570122189613329 51.950602187631205, 7.580319006590855 51.93825551711683, 7.609054957094401 51.93035649564658, 7.659674869951374 51.942256350721436, 7.6833460522228165 51.968514669138415, 7.665137450475669 51.99229098076532, 7.626171042736502 51.98982421450293, 7.599984296478425 51.984257653537384)))",
       "creationDate": "2022-10-24T12:10:53.086Z",
-      "lastUpdate": "2022-10-24T12:10:53.086Z"
+      "lastUpdate": "2022-10-24T12:10:53.086Z",
+      "journal": "OPTIMAP Test Journal",
+      "timeperiod_startdate": "[\"2020-02-02\"]",
+      "timeperiod_enddate": "[\"2022-02-20\"]"
     }
   },
   {
@@ -24,7 +27,10 @@
       "url": "http://paper.url/two",
       "geometry": "SRID=4326;GEOMETRYCOLLECTION (LINESTRING (9.754609563397707 52.36630414438588, 9.813062794192035 52.41569645624003, 10.141300167111496 52.36904961184797, 10.518997966087937 52.330597538337116, 10.838242534270051 52.311358956793185, 11.058566250338231 52.220550088821824, 11.535184901427073 52.15714903642342, 12.272594889905236 52.24258143981572, 12.618817872299417 52.35532056817789, 12.911084026269464 52.2976119913985, 13.144896949445211 52.50063147184562, 13.396695482095708 52.517051586549286))",
       "creationDate": "2022-10-24T12:10:53.086Z",
-      "lastUpdate": "2022-10-24T12:10:53.086Z"
+      "lastUpdate": "2022-10-24T12:10:53.086Z",
+      "journal": "OPTIMAP Test Journal",
+      "timeperiod_startdate": "[\"2010-01-01\"]",
+      "timeperiod_enddate": "[\"2012-12-12\"]"
     }
   }
 ]

fixtures/test_data.sql

@@ -10,8 +10,8 @@ curl -L https://fly.io/install.sh | sh
 
 ## Create PostGIS
 
-- https://community.fly.io/t/deploying-postgis/3530
-- https://fly.io/docs/reference/postgres-on-nomad/
+- <https://community.fly.io/t/deploying-postgis/3530>
+- <https://fly.io/docs/reference/postgres-on-nomad/>
 
 ```bash
 flyctl postgres create 
@@ -45,9 +45,9 @@ fly launch --dockerfile Dockerfile
 
 Now say YES when asked if you want to create a Postgres DB.
 
-> We recommend using the database_url(pip install dj-database-url) to parse the DATABASE_URL from os.environ['DATABASE_URL']
+> We recommend using the database_url (`pip install dj-database-url`) to parse the DATABASE_URL from os.environ['DATABASE_URL']
 >
-> For detailed documentation, see https://fly.dev/docs/django/
+> For detailed documentation, see <https://fly.dev/docs/django/>
 
 Why not - changed configuration style to use `dj-database-uri`.
 
@@ -142,9 +142,10 @@ and
 
 <https://optimap.science/>
 
-## Update allowed hosts
+## Update allowed hosts and configure CSRF
 
 - <https://learndjango.com/tutorials/deploy-django-postgresql-flyio>
+- See <https://github.com/ifgi/optimetaPortal/issues/42> for links and issue description around CSRF
 
 Add to `tly.toml`:
 
@@ -154,7 +155,15 @@ Add to `tly.toml`:
 
 Then `flyctl deploy`.
 
+## Connect to database
+
+```bash
+fly proxy 15432:5432 -a optimap-db
+```
+
+Connect to database locally at port `15432`, e.g., with pgAdmin.
+
 ## Future
 
 - Database backups, see <https://www.joseferben.com/posts/django-on-flyio/>
-- Health check endpoint, see <https://www.joseferben.com/posts/django-on-flyio/>
+- Health check endpoint, see <https://www.joseferben.com/posts/django-on-flyio/>

fly.io.md

@@ -9,18 +9,22 @@ processes = []
   dockerfile = "Dockerfile"
 
 [deploy]
-  release_command = "python manage.py migrate"
+  release_command = "sh release_command.sh"
 
 [env]
-  OPTIMAP_ALLOWED_HOST = "*"
+  OPTIMAP_ALLOWED_HOST = "optimap.science,optimap.fly.dev"
   PORT = "8000"
   OPTIMAP_DEBUG = false
   OPTIMAP_EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend"
-  OPTIMAP_EMAIL_HOST= "smtp.ionos.com"
-  OPTIMAP_EMAIL_PORT = "587"
+  OPTIMAP_EMAIL_HOST= "smtp.ionos.de"
+  OPTIMAP_EMAIL_PORT_SMTP = "587"
   OPTIMAP_EMAIL_HOST_USER = "login@optimap.science"
   OPTIMAP_EMAIL_USE_TLS= true
+  OPTIMAP_EMAIL_HOST_IMAP = "imap.ionos.de"
+  OPTIMAP_EMAIL_PORT_IMAP = "993"
+  OPTIMAP_EMAIL_IMAP_SENT_FOLDER = "\"Gesendete Objekte\""
   CSRF_TRUSTED_ORIGINS = "https://optimap.science"
+  OPTIMAP_LOGGING_CONSOLE_LEVEL = "DEBUG"
 
 [experimental]
   allowed_public_ports = []

fly.toml

@@ -11,7 +11,12 @@ OPTIMAP_DB_PORT=5432
 
 OPTIMAP_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend
 OPTIMAP_EMAIL_HOST=smtp.gmail.com
-OPTIMAP_EMAIL_PORT=587
+OPTIMAP_EMAIL_PORT_SMTP=587
 OPTIMAP_EMAIL_HOST_USER=...
 OPTIMAP_EMAIL_HOST_PASSWORD=...
+OPTIMAP_EMAIL_HOST_IMAP=imap.ionos.de
+OPTIMAP_EMAIL_PORT_IMAP=993
 OPTIMAP_EMAIL_USE_TLS=True
+OPTIMAP_EMAIL_IMAP_SENT_FOLDER=""
+
+OPTIMAP_LOGGING_CONSOLE_LEVEL=INFO

optimetaPortal/.env.example

@@ -31,15 +31,17 @@
 
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = env('SECRET_KEY', default='django-insecure-@(y&etxu!n5qkeyim8ineufd*c*0o20k6$q^$89md-i%qcdk57')
+SECRET_KEY = env('SECRET_KEY', default='django-insecure')
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env('OPTIMAP_DEBUG', default=True)
 
-ALLOWED_HOSTS = env('OPTIMAP_ALLOWED_HOST', default=[])
+ALLOWED_HOSTS = [i.strip('[]') for i in env('OPTIMAP_ALLOWED_HOST', default='*').split(',')]
 
 OPTIMAP_SUPERUSER_EMAILS = [i.strip('[]') for i in env('OPTIMAP_SUPERUSER_EMAILS', default='').split(',')]
 
+TEST_HARVESTING_ONLINE = env('OPTIMAP_TEST_HARVESTING_ONLINE', default=False)
+
 ROOT_URLCONF = 'optimetaPortal.urls'
 
 AUTHENTICATION_BACKENDS = [
@@ -61,10 +63,13 @@
     'django_q',
     'drf_spectacular',
     'drf_spectacular_sidecar',
+    'leaflet' # used in admin site
 ]
 
 REST_FRAMEWORK = {
     'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema',
+    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
+	'PAGE_SIZE': 999,
 }
 
 # https://github.com/tfranzel/drf-spectacular
@@ -139,10 +144,10 @@
 }
 
 CACHES = {
-    # defaults to local-memory caching, see https://docs.djangoproject.com/en/4.1/topics/cache/#local-memory-caching
+    # defaults to database caching to persist across processes, see https://docs.djangoproject.com/en/4.1/topics/cache/#local-memory-caching
     'default': {
-        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': 'unique-snowflake',
+        'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
+        'LOCATION': 'cache',
     },
 
     # use for development
@@ -166,11 +171,15 @@
 
 # for testing email sending EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
 EMAIL_BACKEND =       env('OPTIMAP_EMAIL_BACKEND', default='django.core.mail.backends.console.EmailBackend')
-EMAIL_HOST =          env('OPTIMAP_EMAIL_HOST', default='smtp.gmail.com')
-EMAIL_PORT =          env('OPTIMAP_EMAIL_PORT', default=587)
-EMAIL_HOST_USER =     env('OPTIMAP_EMAIL_HOST_USER', default=False)
-EMAIL_HOST_PASSWORD = env('OPTIMAP_EMAIL_HOST_PASSWORD', default=False)
-EMAIL_USE_TLS =       env('OPTIMAP_EMAIL_USE_TLS', default=True)
+EMAIL_HOST =          env('OPTIMAP_EMAIL_HOST', default='optimeta.dev')
+EMAIL_PORT =          env('OPTIMAP_EMAIL_PORT_SMTP', default=587)
+EMAIL_HOST_IMAP =     env('OPTIMAP_EMAIL_HOST_IMAP', default='optimeta.imap')
+EMAIL_PORT_IMAP =     env('OPTIMAP_EMAIL_PORT_IMAP', default=993)
+EMAIL_HOST_USER =     env('OPTIMAP_EMAIL_HOST_USER', default='optimap@dev')
+EMAIL_HOST_PASSWORD = env('OPTIMAP_EMAIL_HOST_PASSWORD', default='')
+EMAIL_USE_TLS =       env('OPTIMAP_EMAIL_USE_TLS', default=False)
+EMAIL_USE_SSL =       env('OPTIMAP_EMAIL_USE_SSL', default=False)
+EMAIL_IMAP_SENT_FOLDER = env('OPTIMAP_EMAIL_IMAP_SENT_FOLDER', default='')
 
 MIDDLEWARE = [
     'django.middleware.cache.UpdateCacheMiddleware',
@@ -273,7 +282,7 @@
     },
     'handlers': {
         'console': {
-            'level': 'INFO',
+            'level': 'DEBUG',
             'filters': ['require_debug_true'],
             'class': 'logging.StreamHandler',
             'formatter': 'simple'
@@ -290,12 +299,18 @@
             'handlers': ['console', 'mail_admins'],
             'level': 'INFO',
         },
+        'publications': {
+            'handlers': ['console', 'mail_admins'],
+            'level': env('OPTIMAP_LOGGING_CONSOLE_LEVEL', default='INFO'),
+        },
         'django.request': {
             'handlers': ['mail_admins'],
             'level': 'WARNING',
             'propagate': False,
-        },
+        }
     }
 }
 
+CSRF_TRUSTED_ORIGINS = [i.strip('[]') for i in env('CSRF_TRUSTED_ORIGINS', default='https://localhost:8000').split(',')]
+
 ADMINS = [('OPTIMAP', 'login@optimap.science')]

optimetaPortal/settings.py

@@ -1,14 +1,9 @@
 from django.contrib import admin
-"""Markers admin."""
-
-from django.contrib.gis import admin
-
+from leaflet.admin import LeafletGeoAdmin
 from publications.models import Publication
 
-
 @admin.register(Publication)
-class PublicationAdmin(admin.OSMGeoAdmin):
+class PublicationAdmin(LeafletGeoAdmin):
     """Publication Admin."""
 
     list_display = ("title", "publicationDate", "creationDate", "lastUpdate")
-# Register your models here.

publications/admin.py

@@ -2,9 +2,9 @@
 
 from rest_framework import routers
 
-from publications.viewsets import PublicationViewSet,SubscriptionViewset
+from publications.viewsets import PublicationViewSet, SubscriptionViewset
 
 router = routers.DefaultRouter()
 router.register(r"publications", PublicationViewSet)
-router.register(r"subscriptions",SubscriptionViewset,basename='Subscription')
+router.register(r"subscriptions", SubscriptionViewset, basename='Subscription')
 urlpatterns = router.urls