From d4c2627391b4da4c7dd17236e96287b8e74c9501 Mon Sep 17 00:00:00 2001 From: James B Date: Thu, 11 Sep 2025 12:01:39 +0100 Subject: [PATCH] Add Dev Container, demo app, page in docs --- .github/workflows/lint.yml | 7 +- Dockerfile | 4 + docs/develop.rst | 37 ++++ docs/index.rst | 1 + fjorddemo/__init__.py | 0 fjorddemo/app/__init__.py | 0 fjorddemo/app/admin.py | 0 fjorddemo/app/apps.py | 5 + fjorddemo/app/forms.py | 27 +++ fjorddemo/app/migrations/__init__.py | 0 fjorddemo/app/models.py | 0 fjorddemo/app/process.py | 25 +++ fjorddemo/app/templates/fjorddemo/base.html | 36 ++++ .../app/templates/fjorddemo/explore.html | 7 + fjorddemo/app/templates/fjorddemo/index.html | 29 ++++ fjorddemo/app/views.py | 84 +++++++++ fjorddemo/settings.py | 159 ++++++++++++++++++ fjorddemo/urls.py | 24 +++ fjorddemo/wsgi.py | 16 ++ manage.py | 15 ++ setup.py | 7 +- 21 files changed, 479 insertions(+), 4 deletions(-) create mode 100644 Dockerfile create mode 100644 docs/develop.rst create mode 100644 fjorddemo/__init__.py create mode 100644 fjorddemo/app/__init__.py create mode 100644 fjorddemo/app/admin.py create mode 100644 fjorddemo/app/apps.py create mode 100644 fjorddemo/app/forms.py create mode 100644 fjorddemo/app/migrations/__init__.py create mode 100644 fjorddemo/app/models.py create mode 100644 fjorddemo/app/process.py create mode 100644 fjorddemo/app/templates/fjorddemo/base.html create mode 100644 fjorddemo/app/templates/fjorddemo/explore.html create mode 100644 fjorddemo/app/templates/fjorddemo/index.html create mode 100644 fjorddemo/app/views.py create mode 100644 fjorddemo/settings.py create mode 100644 fjorddemo/urls.py create mode 100644 fjorddemo/wsgi.py create mode 100755 manage.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 673b331..5610414 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,8 @@ jobs: python-version: 3.9 architecture: x64 - run: pip install -e .[dev] - - run: isort --check-only libfjordweb setup.py - - run: black --check libfjordweb setup.py - - run: flake8 libfjordweb setup.py + - run: isort --check-only libfjordweb fjorddemo setup.py + - run: black --check libfjordweb fjorddemo setup.py + - run: flake8 libfjordweb fjorddemo setup.py - run: mypy --install-types --non-interactive -p libfjordweb + - run: mypy --install-types --non-interactive -p fjorddemo diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..20b4d1b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.9 + +CMD [ "/bin/bash", "-c", "--", "while true; do sleep 30; done;" ] + diff --git a/docs/develop.rst b/docs/develop.rst new file mode 100644 index 0000000..5cbfb67 --- /dev/null +++ b/docs/develop.rst @@ -0,0 +1,37 @@ +Develop this library +==================== + +The git repository comes with a dev container and a sample Django app to see the library running and help you develop it. + + +To set up, run the dev container and: + +.. code-block:: + + pip install -e .[dev] + python manage.py migrate + + +To run the web server in one terminal: + +.. code-block:: + + python manage.py runserver 0.0.0.0:8000 + +Open another terminal, and run the worker: + +.. code-block:: + + celery -A libfjordweb.celery worker -l debug -c 1 + + +To lint code: + +.. code-block:: + + isort libfjordweb fjorddemo setup.py + black libfjordweb fjorddemo setup.py + flake8 libfjordweb fjorddemo setup.py + mypy --install-types --non-interactive -p libfjordweb + mypy --install-types --non-interactive -p fjorddemo + diff --git a/docs/index.rst b/docs/index.rst index 617afed..338d5dd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,4 +28,5 @@ The application consists of: migration-from-lib-cove-web.rst hosting/index.rst used-by.rst + develop.rst diff --git a/fjorddemo/__init__.py b/fjorddemo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fjorddemo/app/__init__.py b/fjorddemo/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fjorddemo/app/admin.py b/fjorddemo/app/admin.py new file mode 100644 index 0000000..e69de29 diff --git a/fjorddemo/app/apps.py b/fjorddemo/app/apps.py new file mode 100644 index 0000000..d36f30b --- /dev/null +++ b/fjorddemo/app/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class FjordDemoAppConfig(AppConfig): + name = "fjorddemo.app" diff --git a/fjorddemo/app/forms.py b/fjorddemo/app/forms.py new file mode 100644 index 0000000..7eed6a5 --- /dev/null +++ b/fjorddemo/app/forms.py @@ -0,0 +1,27 @@ +from django import forms +from django.conf import settings + + +class NewJSONUploadForm(forms.Form): + file_field_names = ["file_upload"] + file_upload = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_JSON_CONTENT_TYPES + + settings.ALLOWED_JSON_EXTENSIONS + ) + } + ), + label="", + ) + + +class NewJSONTextForm(forms.Form): + file_field_names: list = [] + paste = forms.CharField(label="Paste (JSON only)", widget=forms.Textarea) + + +class NewJSONURLForm(forms.Form): + file_field_names: list = [] + url = forms.URLField(label="URL") diff --git a/fjorddemo/app/migrations/__init__.py b/fjorddemo/app/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fjorddemo/app/models.py b/fjorddemo/app/models.py new file mode 100644 index 0000000..e69de29 diff --git a/fjorddemo/app/process.py b/fjorddemo/app/process.py new file mode 100644 index 0000000..9b432c4 --- /dev/null +++ b/fjorddemo/app/process.py @@ -0,0 +1,25 @@ +import os + +from libfjordweb.process.common_tasks.task_with_state import TaskWithState + + +class DetailsOnJSON(TaskWithState): + """""" + + state_filename: str = "details_on_json.json" + + def process_get_state(self, process_data: dict): + + out: dict = {} + + supplied_data_json_files = [ + i for i in self.supplied_data_files if i.content_type == "application/json" + ] + if len(supplied_data_json_files) == 1: + out["data_size"] = os.path.getsize( + supplied_data_json_files[0].upload_dir_and_filename() + ) + else: + raise Exception("Can't find JSON original data!") + + return out, process_data diff --git a/fjorddemo/app/templates/fjorddemo/base.html b/fjorddemo/app/templates/fjorddemo/base.html new file mode 100644 index 0000000..619819e --- /dev/null +++ b/fjorddemo/app/templates/fjorddemo/base.html @@ -0,0 +1,36 @@ +{% extends 'libfjordweb/base.html' %} +{% load i18n %} +{% load static %} + +{% block after_head %} +{% endblock %} + +{% block banner %} +{% endblock %} + +{% block page_header %} +{% endblock %} + +{% block full_width_header %} + +

Lib Fjord Web Demo App

+ +{% endblock %} + +{% block link %} +{% endblock %} + +{% block bottomcontent1 %} +{% endblock %} + +{% block topcontent1 %} +{% endblock %} + +{% block bottomcontent3 %} +{% endblock %} + +{% block about %} +{% endblock %} + +{% block version_link %} +{% endblock %} diff --git a/fjorddemo/app/templates/fjorddemo/explore.html b/fjorddemo/app/templates/fjorddemo/explore.html new file mode 100644 index 0000000..4ef893e --- /dev/null +++ b/fjorddemo/app/templates/fjorddemo/explore.html @@ -0,0 +1,7 @@ +{% extends request.current_app_base_template %} + +{% block content %} + +

The data you uploaded is {{ data_size }} size.

+ +{% endblock %} diff --git a/fjorddemo/app/templates/fjorddemo/index.html b/fjorddemo/app/templates/fjorddemo/index.html new file mode 100644 index 0000000..8f6408f --- /dev/null +++ b/fjorddemo/app/templates/fjorddemo/index.html @@ -0,0 +1,29 @@ +{% extends request.current_app_base_template %} + +{% block content %} + +

Upload

+ +
+ {% csrf_token %} + {{ forms.json.upload_form }} + +
+ +

Enter Directly

+ +
+ {% csrf_token %} + {{ forms.json.text_form }} + +
+ +

URL

+ +
+ {% csrf_token %} + {{ forms.json.url_form }} + +
+ +{% endblock %} diff --git a/fjorddemo/app/views.py b/fjorddemo/app/views.py new file mode 100644 index 0000000..8b4b8af --- /dev/null +++ b/fjorddemo/app/views.py @@ -0,0 +1,84 @@ +import logging + +from django.conf import settings +from django.shortcuts import render + +from fjorddemo.app.forms import ( + NewJSONTextForm, + NewJSONUploadForm, + NewJSONURLForm, +) +from libfjordweb.models import SuppliedDataFile +from libfjordweb.views import ExploreDataView, InputDataView + +logger = logging.getLogger(__name__) + + +def index(request): + forms = { + "json": { + form_name: form_class() + for form_name, form_class in JSON_FORM_CLASSES.items() + }, + } + + return render(request, "fjorddemo/index.html", {"forms": forms}) + + +JSON_FORM_CLASSES: dict = { + "upload_form": NewJSONUploadForm, + "text_form": NewJSONTextForm, + "url_form": NewJSONURLForm, +} + + +class NewJSONInput(InputDataView): + form_classes: dict = JSON_FORM_CLASSES # type: ignore + input_template = "fjorddemo/index.html" # type: ignore + allowed_content_types = settings.ALLOWED_JSON_CONTENT_TYPES + content_type_incorrect_message = "This does not appear to be a JSON file." + allowed_file_extensions = settings.ALLOWED_JSON_EXTENSIONS + file_extension_incorrect_message = "This does not appear to be a JSON file." + supplied_data_format = "json" # type: ignore + + def get_active_form_key(self, forms, request_data): + if "paste" in request_data: + return "text_form" + elif "url" in request_data: + return "url_form" + else: + return "upload_form" + + def save_file_content_to_supplied_data( + self, form_name, form, request, supplied_data + ): + if form_name == "upload_form": + supplied_data.save_file(request.FILES["file_upload"]) + elif form_name == "text_form": + supplied_data.save_file_contents( + "input.json", + form.cleaned_data["paste"], + "application/json", + None, + ) + elif form_name == "url_form": + supplied_data.save_file_from_source_url( + form.cleaned_data["url"], content_type="application/json" + ) + + +class ExploreView(ExploreDataView): + explore_template = "fjorddemo/explore.html" # type: ignore + + def default_explore_context(self, supplied_data): + return { + # Misc + "supplied_data_files": SuppliedDataFile.objects.filter( + supplied_data=supplied_data + ), + "created_datetime": supplied_data.created.strftime( + "%A, %d %B %Y %I:%M%p %Z" + ), + "created_date": supplied_data.created.strftime("%A, %d %B %Y"), + "created_time": supplied_data.created.strftime("%I:%M%p %Z"), + } diff --git a/fjorddemo/settings.py b/fjorddemo/settings.py new file mode 100644 index 0000000..ee3dfd7 --- /dev/null +++ b/fjorddemo/settings.py @@ -0,0 +1,159 @@ +""" +Django settings for demo project. + +Generated by 'django-admin startproject' using Django 2.1.3. + +For more information on this file, see +https://docs.djangoproject.com/en/2.1/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.1/ref/settings/ +""" + +import os + +import environ + +from libfjordweb import settings + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +env = environ.Env( # set default values and casting + DB_NAME=(str, os.path.join(BASE_DIR, "db.sqlite")), + CELERY_BROKER_URL=(str, ""), + REDIS_URL=(str, ""), +) + +# We can't take MEDIA_ROOT and MEDIA_URL from cove settings, +# ... otherwise the files appear under the BASE_DIR that is the Cove library install. +# That could get messy. We want them to appear in our directory. +MEDIA_ROOT = os.path.join(BASE_DIR, "media") +MEDIA_URL = "/media/" + +SECRET_KEY = settings.SECRET_KEY +DEBUG = settings.DEBUG +ALLOWED_HOSTS = settings.ALLOWED_HOSTS + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "bootstrap3", + "libfjordweb", + "fjorddemo.app.apps.FjordDemoAppConfig", +] + + +MIDDLEWARE = ( + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.locale.LocaleMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", + "libfjordweb.middleware.CoveConfigCurrentApp", +) + + +ROOT_URLCONF = "fjorddemo.urls" + +TEMPLATES = settings.TEMPLATES + +WSGI_APPLICATION = "fjorddemo.wsgi.application" + +# We can't take DATABASES from cove settings, +# ... otherwise the files appear under the BASE_DIR that is the Cove library install. +# That could get messy. We want them to appear in our directory. +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": env("DB_NAME"), + } +} + +# Password validation +# https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/2.1/topics/i18n/ + +LANGUAGE_CODE = settings.LANGUAGE_CODE +TIME_ZONE = settings.TIME_ZONE +USE_I18N = settings.USE_I18N +USE_L10N = settings.USE_L10N +USE_TZ = settings.USE_TZ + +LANGUAGES = (("en", "English"),) + +LOCALE_PATHS = (os.path.join(BASE_DIR, "fjorddemo", "locale"),) + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.1/howto/static-files/ + +# We can't take STATIC_URL and STATIC_ROOT from cove settings, +# ... otherwise the files appear under the BASE_DIR that is the Cove library install. +# and that doesn't work with our standard Apache setup. +STATIC_URL = "/static/" +STATIC_ROOT = os.path.join(BASE_DIR, "static") + +# Misc + +LOGGING = settings.LOGGING + +# Demo Config + +COVE_CONFIG = { + "app_name": "fjorddemo", + "app_base_template": "fjorddemo/base.html", + "app_verbose_name": "Fjord Demo", + "app_strapline": "Review your data.", + "root_list_path": "networks", + "root_id": "id", + "id_name": "id", + "convert_titles": False, + "input_methods": ["upload", "url", "text"], + "support_email": "openfibre@opendataservices.coop", +} + +DELETE_FILES_AFTER_DAYS = settings.DELETE_FILES_AFTER_DAYS + +# https://github.com/OpenDataServices/cove/issues/1098 +FILE_UPLOAD_PERMISSIONS = 0o644 + +ALLOWED_JSON_CONTENT_TYPES = settings.ALLOWED_JSON_CONTENT_TYPES +ALLOWED_JSON_EXTENSIONS = settings.ALLOWED_JSON_EXTENSIONS + +PROCESS_TASKS = [ + # Get data if not already on disk + ("libfjordweb.process.common_tasks.download_data_task", "DownloadDataTask"), + # Demo tasks + ("fjorddemo.app.process", "DetailsOnJSON"), +] + +CELERY_BROKER_URL = env("CELERY_BROKER_URL") or env("REDIS_URL") or "redis://redis" +CELERY_TASK_EAGER_PROPAGATES = CELERY_BROKER_URL == "memory://" +CELERY_TASK_ALWAYS_EAGER = CELERY_BROKER_URL == "memory://" diff --git a/fjorddemo/urls.py b/fjorddemo/urls.py new file mode 100644 index 0000000..be15d38 --- /dev/null +++ b/fjorddemo/urls.py @@ -0,0 +1,24 @@ +from django.conf import settings +from django.conf.urls.static import static +from django.urls import re_path + +import fjorddemo.app.views +import libfjordweb.views +from libfjordweb.urls import urlpatterns + +handler500 = "libfjordweb.views.handler500" + +urlpatterns += [ + re_path(r"^$", fjorddemo.app.views.index, name="index"), + re_path(r"^new_json$", fjorddemo.app.views.NewJSONInput.as_view(), name="new_json"), + re_path( + r"^data/([\w\-]+)$", fjorddemo.app.views.ExploreView.as_view(), name="explore" + ), + re_path( + r"^data/([\w\-]+)/processing_status_api$", + libfjordweb.views.ExploreDataProcessingStatusAPIView.as_view(), + name="explore_processing_status_api$", + ), +] + +urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/fjorddemo/wsgi.py b/fjorddemo/wsgi.py new file mode 100644 index 0000000..2de6af3 --- /dev/null +++ b/fjorddemo/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for demo project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fjorddemo.settings") + +application = get_wsgi_application() diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..76d9c95 --- /dev/null +++ b/manage.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == '__main__': + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fjorddemo.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) diff --git a/setup.py b/setup.py index b25eb8e..7c443ec 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,12 @@ version="0.5.0", author="Open Data Services", author_email="code@opendataservices.coop", - packages=find_packages(), + packages=find_packages( + exclude=( + "fjorddemo", + "fjorddemo.*", + ) + ), package_data={ "libfjordweb": [ "static/*",