Initial commit

Author: mdouchin

.docker/db-runner-entrypoint

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Basic entrypoint for executing shell commande
+
+exec bash -c "$@"

.docker/docker-compose.yml

@@ -0,0 +1,84 @@
+services:
+  qgis:
+    image: ${QGIS_IMAGE_TAG}
+    user: ${UID}:${GID}
+    command: /src/.docker/run-tests.sh
+    environment:
+      # Remove HOME variable if not using database
+      HOME: /home/qgis
+      QGIS_VERSION: ${QGIS_VERSION}
+    volumes:
+    - {type: bind, source: "..", target: /src}
+#
+# Remove everything from here if your are not using database
+#
+    - type: bind
+      source: ./pg_service.conf
+      target: /etc/postgresql-common/pg_service.conf
+    depends_on:
+      db:
+        condition: service_healthy
+    profiles:
+      - qgis
+
+  db:
+    image: 3liz/postgis:${POSTGIS_VERSION}
+    hostname: db
+    environment:
+      POSTGRES_DB: "gis"
+      POSTGRES_USER: "docker"
+      POSTGRES_PASSWORD: "docker"
+      POSTGRES_HOST: "db"
+    ports:
+      - "35432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U docker -d gis -q || exit 1"]
+      interval: 30s
+      timeout: 30s
+      start_interval: 2s
+      start_period: 10s 
+ 
+  db-runner:
+    build:
+      context: .
+      dockerfile_inline: |
+          FROM 3liz/postgis:${POSTGIS_VERSION}
+          RUN apt-get -y update && apt-get install -y rename
+    user: ${UID}:${GID}
+    command: ${DB_COMMAND}
+    working_dir: /src/.docker
+    entrypoint: /src/.docker/db-runner-entrypoint
+    stop_signal: SIGKILL
+    volumes:
+    - {type: bind, source: "..", target: /src}
+    - {type: bind, source: "../${MODULE_NAME}", target: /plugin}
+    - type: bind
+      source: ./pg_service.conf
+      target: /etc/postgresql-common/pg_service.conf
+    environment:
+      MODULE_NAME: ${MODULE_NAME}
+      SCHEMA: ${SCHEMA}
+      DB_CURRENT_VERSION: ${DB_CURRENT_VERSION}
+    depends_on:
+      db:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f /tmp/.db-runner-ok ] || exit 1"]
+      start_interval: 2s
+      start_period: 10s 
+    profiles:
+    - schemaspy
+    - dbrunner
+      
+  # Wait for db-runner to set the /tmp/.db-runner-ok file
+  schemaspy:
+    image: schemaspy/schemaspy:latest
+    volumes:
+    - {type: bind, source: "../docs/database", target: /output}
+    command: >-
+      -t pgsql -host db -db gis -u docker -p docker -port 5432 -s ${SCHEMA} -nopages
+    depends_on:
+      db-runner:
+        condition: service_healthy
+    profiles:
+    - schemaspy

.docker/export_database_structure_to_SQL.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+#
+# Explode PostgreSQL database dump into several files, one per type
+# LICENCE: GPL 2
+# AUTHOR: 3LIZ
+
+
+echo "# CHECK INPUT PARAMETERS service and schema"
+if [ -n "$1" ]; then
+  echo "# POSTGRESQL SERVICE: $1"
+  SERVICE=$1
+else
+  echo "ERROR: No PostgreSQL service given as second parameter";
+  exit;
+fi
+if [ -n "$2" ]; then
+  echo "# GIVEN SCHEMA: $2"
+  SCHEMA=$2
+else
+  SCHEMA="cartads"
+  echo "# DEFAULT SCHEMA: $SCHEMA";
+fi
+echo ""
+
+OUTDIR=$SCHEMA
+mkdir -p ./$OUTDIR
+
+# STRUCTURE
+# Dump database structure
+pg_dump service=$SERVICE --schema-only -n $SCHEMA --no-acl --no-owner -Fc -f "$OUTDIR/dump"
+
+# Loop through DB object types and extract SQL
+I=10
+for ITEM in FUNCTION "TABLE|SEQUENCE|DEFAULT" VIEW INDEX TRIGGER CONSTRAINT COMMENT; do
+    echo $ITEM
+    # Extract list of objects for current item
+    pg_restore --no-acl --no-owner -l $OUTDIR/dump | grep -E "$ITEM" > "$OUTDIR/$ITEM";
+    # Extract SQL for these objects
+    pg_restore -f "$OUTDIR"/"$I"_"$ITEM".sql --no-acl --no-owner -L "$OUTDIR/$ITEM" "$OUTDIR/dump";
+    # Remove file containing list of objects
+    rm "$OUTDIR/$ITEM";
+    # Simplify comments inside SQL files
+    perl -i -0pe 's/\n--\n-- Name: (TABLE )?(COLUMN )?(.+); Type:.+\n--\n\n/\n-- $3\n/g' "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Remove audit trigger (added afterwards)
+    if [ $ITEM = 'TRIGGER' ]
+    then
+        sed -i '/audit_trigger/d' "$OUTDIR"/"$I"_"$ITEM".sql;
+    fi
+    # Remove SET function to remove some compatibility issues between PostgreSQL versions
+    sed -i "s#SET idle_in_transaction_session_timeout = 0;##g" "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Remove as integer for sequences, to keep compatibility
+    sed -i -E "s#    AS integer##g" "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Remove SET search_path
+    sed -i "s#SELECT pg_catalog.set_config('search_path', '', false);##g" "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Remove default_table_access_method
+    sed -i "s#SET default_table_access_method = heap##g" "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Remove SET transaction_timeout = 0;
+    sed -i "s#SET transaction_timeout = 0;##g" "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Remove --- Dumped blah
+    sed -i 's#-- Dumped.*$##g' "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Replace FOR EACH ROW EXECUTE FUNCTION (pg13) by FOR EACH ROW EXECUTE PROCEDURE (still ok for Pg13)
+    sed -i "s#FOR EACH ROW EXECUTE FUNCTION#FOR EACH ROW EXECUTE PROCEDURE#g" "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Remove new pg_dump 17 restrict lines
+    sed -i -E "s#\\\(un)?restrict .*##g" "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Replace public.geometry by geometry
+    sed -i "s#public.geometry#geometry#g" "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Rename
+    rename -f 's#\|#_#g' "$OUTDIR"/"$I"_"$ITEM".sql;
+    # Increment I
+    I=$(($I+10));
+done
+
+# Remove dump
+rm "$OUTDIR/dump"
+
+# NOMENCLATURE
+echo "GLOSSARY"
+if [ $SCHEMA = 'cartads' ]
+then
+    pg_dump service=$SERVICE --data-only --inserts --column-inserts -n $SCHEMA --no-acl --no-owner --table "$SCHEMA.glossary_*" -f "$OUTDIR"/90_GLOSSARY.sql
+    sed -i "s#SET idle_in_transaction_session_timeout = 0;##g" "$OUTDIR"/"90_GLOSSARY.sql"
+    sed -i "s#SELECT pg_catalog.set_config('search_path', '', false);##g" "$OUTDIR"/"90_GLOSSARY.sql"
+    # Remove --- Dumped blah
+    sed -i 's#-- Dumped.*$##g' "$OUTDIR"/"90_GLOSSARY.sql";
+    # Remove SET transaction_timeout = 0;
+    sed -i "s#SET transaction_timeout = 0;##g" "$OUTDIR"/"90_GLOSSARY.sql";
+    # Remove new pg_dump 17 restrict lines
+    sed -i -E "s#\\\(un)?restrict .*##g" "$OUTDIR"/"90_GLOSSARY.sql";
+fi

.docker/install_db.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo 'Installation from latest version'
+psql service=test -c "DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE;" > /dev/null
+psql service=test -f /src/${MODULE_NAME}/install/sql/00_initialize_database.sql > /dev/null
+for sql_file in `ls -d -v /src/${MODULE_NAME}/install/sql/${SCHEMA}/*.sql`; do
+  echo "${sql_file}"
+  psql service=test -f ${sql_file} > /dev/null;
+done;

.docker/install_db_and_wait.sh

@@ -0,0 +1,5 @@
+set -e
+
+./install_db.sh
+touch /tmp/.db-runner-ok
+sleep infinity

.docker/install_migrate_generate.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+#
+# 1.Install the database with a stored previous version.
+# 2.Run the migrations scripts up to the current version.
+# 3.Output the migrate schema in tests/.tests-migration-<from>-to-<current>
+# 4.Generate a diff `sql.patch` between the current schema and the upgraded one
+#
+
+set -eu
+
+current_version=$DB_CURRENT_VERSION
+db_version=${DB_INSTALL_VERSION:-$((current_version-1))}
+
+install_dir=/src/tests/data/install-version-$db_version/sql
+
+if [[ ! -e $install_dir ]]; then
+    echo "No installation version for database version $db_version"
+    exit 1
+fi
+
+echo "== Installation from version $db_version"
+psql service=test -c "DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE;" > /dev/null
+psql service=test -f $install_dir/00_initialize_database.sql > /dev/null
+
+for sql_file in `ls -d -v $install_dir/${SCHEMA}/*.sql`; do
+  echo "${sql_file}"
+  psql service=test -f ${sql_file} > /dev/null
+done
+
+echo ""
+
+start_migration=$((db_version+1))
+echo "== Run SQL migrations stored from version $db_version to $current_version =="
+for ((i=start_migration;i<=current_version;i++)); do
+   migration=/src/${MODULE_NAME}/install/sql/upgrade/upgrade_to_$i.sql
+   if [[ ! -e $migration ]]; then
+       echo "Missing migration file $migration"
+       exit 1
+   fi
+   echo "Running migration:  ${migration}"
+   psql service=test -f ${migration} > /dev/null;
+done;
+echo ""
+
+cd /src
+
+echo '== Export update database as installation SQL files'
+destination_dir=tests/.test-migration-$db_version-to-$current_version
+rm -rf $destination_dir
+mkdir -p $destination_dir/sql
+pushd $destination_dir/sql
+/src/.docker/export_database_structure_to_SQL.sh test ${SCHEMA}
+popd
+
+
+# Generate a diff file between current and install version
+echo "== Creating patch file"
+set +e  # Suppress exit on erreur
+diff -urB $MODULE_NAME/install/sql/$SCHEMA  $destination_dir/sql/$SCHEMA > $destination_dir/sql.patch 
+
+[[ $? -eq 1 ]] && true
+echo "== Done"

.docker/pg_service.conf

@@ -0,0 +1,6 @@
+[test]
+host=db
+port=5432
+user=docker
+password=docker
+dbname=gis

.docker/processing_doc.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+export $(grep -v '^#' .env | xargs)
+
+#xhost +
+
+docker run -d \
+  --name qgis-testing-environment \
+  -v  $(pwd)/../${PLUGIN_NAME}:/tests_directory/${PLUGIN_NAME} \
+  -v  $(pwd)/../docs/processing:/processing \
+  -e DISPLAY=:99 \
+  qgis/qgis:release-3_34
+
+sleep 10
+
+echo "Setting up"
+docker exec -t qgis-testing-environment sh -c "qgis_setup.sh ${PLUGIN_NAME}"
+docker exec -t qgis-testing-environment sh \
+  -c "qgis_testrunner.sh ${PLUGIN_NAME}.qgis_plugin_tools.infrastructure.doc_processing.generate_processing_doc"
+
+docker kill qgis-testing-environment
+docker rm qgis-testing-environment

.docker/reformat_sql_install.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+echo 'Generating SQL files'
+cd /src/${MODULE_NAME}/install/sql/
+./export_database_structure_to_SQL.sh test ${SCHEMA}
+

.docker/run-tests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Run test in docker QGIS  image
+#
+
+set -e
+
+cd  /src
+
+VENV=/src/.docker-venv-$QGIS_VERSION
+
+python3 -m venv $VENV --system-site-package
+
+echo "Installing requirements..."
+$VENV/bin/pip install -q --no-cache -r requirements/tests.txt
+
+cd tests && $VENV/bin/python -m pytest -v
+
+