Merge branch 'master' into feature/better_error_handling

Author: madnificent

.woodpecker/.feature.yml

@@ -1,10 +1,13 @@
-pipeline:
+steps:
   build-and-push:
-    image: plugins/docker
+    image: woodpeckerci/plugin-docker-buildx
     settings:
       repo: "${CI_REPO_OWNER##mu-}/${CI_REPO_NAME}"
       tags: "feature-${CI_COMMIT_BRANCH##feature/}"
-    secrets: [docker_username, docker_password]
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
 when:
-  event: push
-  branch: feature/*
+  - event: push
+    branch: [feature/*]

.woodpecker/.latest.yml

@@ -1,10 +1,13 @@
-pipeline:
+steps:
   build-and-push:
-    image: plugins/docker
+    image: woodpeckerci/plugin-docker-buildx
     settings:
       repo: "${CI_REPO_OWNER##mu-}/${CI_REPO_NAME}"
       tags: latest
-    secrets: [docker_username, docker_password]
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
 when:
-  event: push
-  branch: [master, main]
+  - event: push
+    branch: [master, main]

.woodpecker/.release.yml

@@ -1,10 +1,13 @@
-pipeline:
+steps:
   release:
-    image: plugins/docker
+    image: woodpeckerci/plugin-docker-buildx
     settings:
       repo: "${CI_REPO_OWNER##mu-}/${CI_REPO_NAME}"
       tags: "${CI_COMMIT_TAG##v}"
-    secrets: [ docker_username, docker_password ]
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
 when:
-  event: tag
-  tag: v*
+  - event: tag
+    ref: refs/tags/v*

Dockerfile

@@ -1,5 +1,5 @@
 FROM tiangolo/meinheld-gunicorn:python3.8
-MAINTAINER Michaël Dierick "[email protected]"
+LABEL maintainer="[email protected]"
 
 # Gunicorn Docker config
 ENV MODULE_NAME web
@@ -14,9 +14,11 @@ RUN chmod +x /start.sh
 # Template config
 ENV APP_ENTRYPOINT web
 ENV LOG_LEVEL info
+ENV LOG_SPARQL_ALL True
 ENV MU_SPARQL_ENDPOINT 'http://database:8890/sparql'
 ENV MU_SPARQL_UPDATEPOINT 'http://database:8890/sparql'
 ENV MU_APPLICATION_GRAPH 'http://mu.semte.ch/application'
+ENV MODE 'production'
 
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app

README.md

@@ -1,13 +1,13 @@
 # Mu Python template
 
-Template for [mu.semte.ch](http://mu.semte.ch)-microservices written in Python3. Based on the [Flask](https://palletsprojects.com/p/flask/)-framework.
+Template for [mu.semte.ch](http://mu.semte.ch)-microservices written in Python3.8. Based on the [Flask](https://palletsprojects.com/p/flask/)-framework.
 
 ## Quickstart
 
 Create a `Dockerfile` which extends the `semtech/mu-python-template`-image and set a maintainer.
 ```docker
 FROM semtech/mu-python-template:2.0.0-beta.2
-LABEL maintainer="sam.landuydt@gmail.com"
+LABEL maintainer="maintainer@example.com"
 ```
 
 Create a `web.py` entrypoint-file. (naming of the entrypoint can be configured through `APP_ENTRYPOINT`)
@@ -300,6 +300,7 @@ my-python:
 ```
 
 ### Environment variables
+#### General
 
 - `LOG_LEVEL` takes the same options as defined in the Python [logging](https://docs.python.org/3/library/logging.html#logging-levels) module.
 
@@ -317,7 +318,16 @@ my-python:
 
 - `MU_SPARQL_TIMEOUT` is used to configure the timeout (in seconds) for SPARQL queries.
 
+#### SPARQL Query Logging
+- `LOG_SPARQL_ALL`: Log *all* executed queries, read as well as update (default `True`)
 
+- `LOG_SPARQL_QUERIES`: Log *read* queries (default: `undefined`). Overrules `LOG_SPARQL_ALL`
+
+- `LOG_SPARQL_UPDATES`: Log *update* queries (default `undefined`). Overrules `LOG_SPARQL_ALL`.
+
+The string "true", ignoring casing, is considered `True`.  All other values are considered `False`.
+
+#### Meinheld Gunicorn Docker Variables
 Since this template is based on the meinheld-gunicorn-docker image, all possible environment config for that image is also available for the template. See [meinheld-gunicorn-docker#environment-variables](https://github.com/tiangolo/meinheld-gunicorn-docker#environment-variables) for more info. The template configures `WEB_CONCURRENCY` in particular to `1` by default.
 
 ### Production

escape_helpers.py

@@ -53,7 +53,7 @@ def sparql_escape_int(obj):
 
 def sparql_escape_float(obj):
     """Converts the given float to a SPARQL-safe RDF object string with the right RDF-datatype. """
-    if not isinstance(obj, int):
+    if not isinstance(obj, float):
         warn("You are escaping something that isn't a float with \
         the 'sparql_escape_float'-method. Implicit casting will occurr.")
         obj = str(float(obj))

helpers.py

@@ -42,6 +42,16 @@
 consoleHandler = logging.StreamHandler(stream=sys.stdout)# or stderr?
 logger.addHandler(consoleHandler)
 
+LOG_SPARQL_ALL_VAR = os.environ.get('LOG_SPARQL_ALL')
+LOG_SPARQL_QUERIES = os.environ.get(
+    'LOG_SPARQL_QUERIES',
+    default=LOG_SPARQL_ALL_VAR
+).lower() == 'true'
+LOG_SPARQL_UPDATES = os.environ.get(
+    'LOG_SPARQL_UPDATES',
+    default=LOG_SPARQL_ALL_VAR
+).lower() == 'true'
+
 def generate_uuid():
     """Generates a random unique user id (UUID) based on the host ID and current time"""
     return str(uuid.uuid1())
@@ -119,15 +129,20 @@ def validate_resource_type(expected_type, data):
 
 def query(the_query):
     """Execute the given SPARQL query (select/ask/construct) on the triplestore and returns the results in the given return Format (JSON by default)."""
-    log("execute query: \n" + the_query)
     for header in MU_HEADERS:
         if header in request.headers:
             sparqlQuery.customHttpHeaders[header] = request.headers[header]
         else: # Make sure headers used for a previous query are cleared
             if header in sparqlQuery.customHttpHeaders:
                 del sparqlQuery.customHttpHeaders[header]
     sparqlQuery.setQuery(the_query)
-    return sparqlQuery.query().convert()
+    if LOG_SPARQL_QUERIES:
+        log("Execute query: \n" + the_query)
+    try:
+        return sparqlQuery.query().convert()
+    except Exception as e:
+        log("Failed Query: \n" + the_query)
+        raise e
 
 
 def update(the_query):
@@ -140,7 +155,13 @@ def update(the_query):
                 del sparqlUpdate.customHttpHeaders[header]
     sparqlUpdate.setQuery(the_query)
     if sparqlUpdate.isSparqlUpdateRequest():
-        sparqlUpdate.query()
+        if LOG_SPARQL_UPDATES:
+            log("Execute query: \n" + the_query)
+        try:
+            sparqlUpdate.query()
+        except Exception as e:
+            log("Failed Query: \n" + the_query)
+            raise e
 
 
 def update_modified(subject, modified=datetime.datetime.now()):

start.sh

@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 set -eu
-if [ ${MODE:-""} == "development" ]; then 
+if [ "${MODE}" == "development" ]; then 
     if [ -f /app/requirements.txt ]; then pip install -r /app/requirements.txt; fi
     exec flask --app web.py run --debug --port 80 --host "0.0.0.0"
 else