Merge branch 'main' into PLAT-235-logstash-dev-mount

Mark-Labuschagne · Mark-Labuschagne · commit d7cb88135b11 · 2022-05-25T16:27:29.000+02:00
diff --git a/.env.dev b/.env.dev
@@ -88,7 +88,7 @@ LOGSTASH_MEMORY_RESERVE=500M
 # ES_ELASTIC - Required for Logstash, set in the "Analytics Datastore - Elastic Search" section
 LS_JAVA_OPTS=-Xmx2g -Xms2g
 
-# Dashboard Visualiser - JS Reports
+# Dashboard Visualiser - JS Report
 
 JS_REPORT_INSTANCES=1
 JS_REPORT=dev_password_only
diff --git a/.env.local b/.env.local
@@ -52,11 +52,15 @@ LS_JAVA_OPTS=-Xmx2g -Xms2g
 LOGSTASH_DEV_MOUNT=false
 LOGSTASH_PACKAGE_PATH=
 
-# Dashboard Visualiser - JS Reports
+# Dashboard Visualiser - JS Report
 
 JS_REPORT_INSTANCES=1
 JS_REPORT=dev_password_only
 JS_REPORT_USERNAME=admin
 JS_REPORT_SECRET=dev_secret_only
 JS_REPORT_SSL=false
 JS_REPORT_CONFIG_FILE=export.jsrexport
+
+# MAKE SURE YOU HAVE RUN 'set-permissions.sh' SCRIPT BEFORE AND AFTER RUNNING JS REPORT
+JS_REPORT_DEV_MOUNT=false
+JS_REPORT_PACKAGE_PATH=
diff --git a/dashboard-visualiser-jsreport/.gitignore b/dashboard-visualiser-jsreport/.gitignore
@@ -0,0 +1,7 @@
+# JS Report metadata files
+fs.journal
+fs.version
+monitoring
+settings
+profiles
+storage
diff --git a/dashboard-visualiser-jsreport/README.md b/dashboard-visualiser-jsreport/README.md
@@ -5,3 +5,18 @@ JS Reports is a visualisation tool, which in our use case, is querying informati
 ## Accessing the services
 
 - JS Reports <http://localhost:5488/> - (u: admin p: dev_password_only) - for pixel-perfect PDF reporting
+
+## Developing the JS Report scripts/templates locally
+
+When seeking to make changes to the JS Report scripts/templates without having to repeatedly start and stop the service, one can set the `JS_REPORT_DEV_MOUNT`
+env var in your .env file to `true` to attach the service's content files to those on your local machine.
+
+- You have to run the `set-permissions.sh` script before launching JS Report when `JS_REPORT_DEV_MOUNT=true`.
+- `NB!!! REMEMBER TO EXPORT THE JSREXPORT FILE WHEN YOU'RE DONE EDITING THE SCRIPTS` (https://jsreport.net/learn/import-export)
+
+## Notes
+
+- With `JS_REPORT_DEV_MOUNT=true`, you have to set the `JS_REPORT_PACKAGE_PATH` variable with the absolute path to the JS Report package on your local machine, i.e., `LOGSTASH_PACKAGE_PATH=/home/user/Documents/Projects/platform/dashboard-visualiser-jsreport`.
+
+- Remember to shutdown JS Report before changing git branches if `JS_REPORT_DEV_MOUNT=true`, otherwise the dev mount will persist the JS Report scripts/templates across your branches.
+
diff --git a/dashboard-visualiser-jsreport/docker-compose.dev-mnt.yml b/dashboard-visualiser-jsreport/docker-compose.dev-mnt.yml
@@ -0,0 +1,6 @@
+version: '3.9'
+
+services:
+  dashboard-visualiser-jsreport:
+    volumes:
+      - ${JS_REPORT_PACKAGE_PATH}/scripts/:/app/jsreport/data/
diff --git a/dashboard-visualiser-jsreport/docker-compose.yml b/dashboard-visualiser-jsreport/docker-compose.yml
@@ -19,7 +19,8 @@ services:
       co.elastic.metrics/metricsets: 'cpu,memory,diskio,info,healthcheck,container'
     environment:
       allowLocalFilesAccess: 'true'
-      extensions_fs-store_dataDirectory: 'jsreport/data'
+      extensions_fsStore_dataDirectory: 'jsreport/data'
+      extensions_fsStore_externalModificationsSync: 'true'
       extensions_authentication_cookieSession_secret: ${JS_REPORT_SECRET:-dev_secret_only}
       extensions_authentication_admin_username: 'admin'
       extensions_authentication_admin_password: ${JS_REPORT:-dev_password_only}
diff --git a/dashboard-visualiser-jsreport/importer/export.jsrexport b/dashboard-visualiser-jsreport/importer/export.jsrexport
diff --git a/dashboard-visualiser-jsreport/package-metadata.json b/dashboard-visualiser-jsreport/package-metadata.json
@@ -16,6 +16,8 @@
     "JS_REPORT_CPU_LIMIT": "0.8",
     "JS_REPORT_MEMORY_LIMIT": "3G",
     "JS_REPORT_CPU_RESERVE": "0.05",
-    "JS_REPORT_MEMORY_RESERVE": "500M"
+    "JS_REPORT_MEMORY_RESERVE": "500M",
+    "JS_REPORT_DEV_MOUNT": "false",
+    "JS_REPORT_PACKAGE_PATH": ""
   }
 }
diff --git a/dashboard-visualiser-jsreport/scripts/elasticSearchConnTestScript/config.json b/dashboard-visualiser-jsreport/scripts/elasticSearchConnTestScript/config.json
@@ -0,0 +1,16 @@
+{
+    "name": "elasticSearchConnTestScript",
+    "scope": "global",
+    "shortid": "LWSXMjRQeQ",
+    "creationDate": {
+        "$$date": 1652944733948
+    },
+    "modificationDate": {
+        "$$date": 1652944733948
+    },
+    "inheritedReadPermissions": [],
+    "inheritedEditPermissions": [],
+    "_id": "4JSx1OQ519Ru98za",
+    "$entitySet": "scripts"
+}
+
diff --git a/dashboard-visualiser-jsreport/scripts/elasticSearchConnTestScript/content.js b/dashboard-visualiser-jsreport/scripts/elasticSearchConnTestScript/content.js
@@ -0,0 +1,18 @@
+// Use the "beforeRender" or "afterRender" hook
+// to manipulate and control the report generation
+const axios = require('axios')
+
+async function beforeRender (req, res) {
+    const resData = await axios({
+        method: 'get',
+        url: `http://analytics-datastore-elastic-search:9200/_cat/health/`,
+        headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Basic ${new Buffer(
+            `elastic:dev_password_only`
+            ).toString('base64')}`
+        }
+    })
+
+    req.data = Object.assign({}, {healthcheck: resData.data})
+}
diff --git a/dashboard-visualiser-jsreport/scripts/elasticSearchConnTestTemplate/config.json b/dashboard-visualiser-jsreport/scripts/elasticSearchConnTestTemplate/config.json
@@ -0,0 +1,20 @@
+{
+    "name": "elasticSearchConnTestTemplate",
+    "engine": "handlebars",
+    "recipe": "chrome-pdf",
+    "shortid": "FADVP6Lsc",
+    "chrome": {
+        "printBackground": true
+    },
+    "creationDate": {
+        "$$date": 1652944733941
+    },
+    "modificationDate": {
+        "$$date": 1652944733941
+    },
+    "inheritedReadPermissions": [],
+    "inheritedEditPermissions": [],
+    "_id": "wvi1DGuMBN75BpZE",
+    "$entitySet": "templates"
+
+}
diff --git a/dashboard-visualiser-jsreport/scripts/elasticSearchConnTestTemplate/content.handlebars b/dashboard-visualiser-jsreport/scripts/elasticSearchConnTestTemplate/content.handlebars
@@ -0,0 +1 @@
+<div>{{healthcheck}}</div>
diff --git a/dashboard-visualiser-jsreport/set-permissions.sh b/dashboard-visualiser-jsreport/set-permissions.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# This script is here to set and reset the file permissions that might have been altered when running
+# JS Report with the dev mount attached. Make sure that you run this script before and after running JS Report
+# with the dev mount attached.
+
+if [[ $(basename $(pwd)) != "dashboard-visualiser-jsreport" ]]; then
+    echo "Please change your working directory to dashboard-visualiser-jsreport's root directory before running the script. Exiting..."
+    exit 1
+fi
+
+sudo chmod 777 scripts/
+
+while read -r file; do
+    sudo chown 1000 "$file"
+    sudo chgrp 1000 "$file"
+    sudo chmod 777 "$file"
+done < <(find ./scripts)
+echo "Successfully set file permissions"
diff --git a/dashboard-visualiser-jsreport/swarm.sh b/dashboard-visualiser-jsreport/swarm.sh
@@ -18,56 +18,13 @@ ROOT_PATH="${COMPOSE_FILE_PATH}/.."
 
 if [[ "$MODE" == "dev" ]]; then
   echo "Running JS Reports package in DEV mode"
-  JsReportDevComposeParam="-c ${COMPOSE_FILE_PATH}/docker-compose.dev.yml"
+  js_report_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.dev.yml"
 else
   echo "Running JS Reports package in PROD mode"
-  JsReportDevComposeParam=""
+  js_report_dev_compose_param=""
 fi
 
-await_jsr_running() {
-  local startTime=$(date +%s)
-  until [[ $(docker service ls -f name=instant_dashboard-visualiser-jsreport --format "{{.Replicas}}") == *"${JS_REPORT_INSTANCES}/${JS_REPORT_INSTANCES}"* ]]; do
-    config::timeout_check $startTime "dashboard-visualiser-jsreport to start"
-    sleep 1
-  done
-
-  local awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
-  until [[ $awaitHelperState == *"Complete"* ]]; do
-    config::timeout_check $startTime "dashboard-visualiser-jsreport status check"
-    sleep 1
-
-    awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
-    if [[ $awaitHelperState == *"Failed"* ]] || [[ $awaitHelperState == *"Rejected"* ]]; then
-      echo "Fatal: Received error when trying to verify state of dashboard-visualiser-jsreport. Error:
-       $(docker service ps instant_await-helper --no-trunc --format \"{{.Error}}\")"
-      exit 1
-    fi
-  done
-
-  docker service rm instant_await-helper
-}
-
-remove_config_importer() {
-  local complete="false"
-  local startTime=$(date +%s)
-  local configImporterState=$(docker service ps instant_jsreport-config-importer --format "{{.CurrentState}}")
-  until [[ $configImporterState == *"Complete"* ]]; do
-    config::timeout_check $startTime "jsreport-config-importer to run"
-    sleep 1
-
-    configImporterState=$(docker service ps instant_jsreport-config-importer --format "{{.CurrentState}}")
-    if [[ $configImporterState == *"Failed"* ]] || [[ $configImporterState == *"Rejected"* ]]; then
-      echo "Fatal: JS Reports config importer failed with error:
-       $(docker service ps instant_jsreport-config-importer --no-trunc --format \"{{.Error}}\")"
-      exit 1
-    fi
-  done
-
-  docker service rm instant_jsreport-config-importer
-}
-
 configure_nginx() {
-
   if [[ "${INSECURE}" == "true" ]]; then
     docker config create --label name=nginx "${TIMESTAMP}-http-jsreport-insecure.conf" "${COMPOSE_FILE_PATH}/config/http-jsreport-insecure.conf"
     echo "Updating nginx service: adding jsreport config file..."
@@ -91,29 +48,39 @@ configure_nginx() {
   fi
 }
 
+if [[ "${JS_REPORT_DEV_MOUNT}" == "true" ]] && [[ "${ACTION}" == "init" ]]; then
+  if [[ -z "${JS_REPORT_PACKAGE_PATH}" ]]; then
+    echo "ERROR: JS_REPORT_PACKAGE_PATH environment variable not specified. Please specify JS_REPORT_PACKAGE_PATH as stated in the README."
+    exit 1
+  fi
+  echo "MAKE SURE YOU HAVE RUN 'set-permissions.sh' SCRIPT BEFORE AND AFTER RUNNING JS REPORT"
+
+  echo "Attaching dev mount..."
+  js_report_dev_mount_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.dev-mnt.yml"
+fi
+
 main() {
   if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
-    docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.yml $JsReportDevComposeParam instant
-
-    docker stack deploy -c "${COMPOSE_FILE_PATH}"/docker-compose.await-helper.yml instant
+    docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.yml $js_report_dev_compose_param $js_report_dev_mount_compose_param instant
 
-    echo "Verifying JS Reports service status"
-    await_jsr_running
+    if [[ "${JS_REPORT_DEV_MOUNT}" != "true" ]]; then
+      echo "Verifying JS Reports service status"
+      config::await_service_running "dashboard-visualiser-jsreport" "$COMPOSE_FILE_PATH"/docker-compose.await-helper.yml "$JS_REPORT_INSTANCES"
 
-    config::set_config_digests "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml
-    docker stack deploy -c "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml instant
+      config::set_config_digests "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml
+      docker stack deploy -c "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml instant
 
-    remove_config_importer
-    config::remove_stale_service_configs "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml "jsreport"
+      config::remove_config_importer "jsreport-config-importer"
+      config::remove_stale_service_configs "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml "jsreport" &>/dev/null
+    fi
 
     if [[ "${MODE}" != "dev" ]]; then
       configure_nginx "$@"
     fi
-
   elif [[ "${ACTION}" == "down" ]]; then
     docker service scale instant_dashboard-visualiser-jsreport=0
   elif [[ "${ACTION}" == "destroy" ]]; then
-    docker service rm instant_dashboard-visualiser-jsreport instant_jsreport-config-importer instant_await-helper
+    docker service rm instant_dashboard-visualiser-jsreport instant_jsreport-config-importer instant_await-helper &>/dev/null
   else
     echo "Valid options are: init, up, down, or destroy"
   fi

Original file line number	Diff line number	Diff line change
`@@ -16,6 +16,8 @@`
`16`	`16`	`"JS_REPORT_CPU_LIMIT": "0.8",`
`17`	`17`	`"JS_REPORT_MEMORY_LIMIT": "3G",`
`18`	`18`	`"JS_REPORT_CPU_RESERVE": "0.05",`
`19`		`- "JS_REPORT_MEMORY_RESERVE": "500M"`
	`19`	`+ "JS_REPORT_MEMORY_RESERVE": "500M",`
	`20`	`+ "JS_REPORT_DEV_MOUNT": "false",`
	`21`	`+ "JS_REPORT_PACKAGE_PATH": ""`
`20`	`22`	`}`
`21`	`23`	`}`