Merge pull request #52 from jembi/PLAT-235-logstash-dev-mount

PLAT-235 Logstash dev mount

Author: Mark-Labuschagne

.env.local

@@ -49,6 +49,9 @@ KIBANA_SSL=false
 # ES_ELASTIC - Required for Logstash, set in the "Analytics Datastore - Elastic Search" section
 LS_JAVA_OPTS=-Xmx2g -Xms2g
 
+LOGSTASH_DEV_MOUNT=false
+LOGSTASH_PACKAGE_PATH=
+
 # Dashboard Visualiser - JS Report
 
 JS_REPORT_INSTANCES=1

data-mapper-logstash/README.md

@@ -0,0 +1,11 @@
+# Data Mapper - Logstash
+
+## Developing the logstash configs locally
+
+When seeking to make changes to the logstash configs without having to repeatedly start and stop the service, one can set the `LOGSTASH_DEV_MOUNT`
+env var in your .env file to `true` to attach the service's config files to those on your local machine.
+
+## Notes
+
+- With `LOGSTASH_DEV_MOUNT=true`, you have to set the `LOGSTASH_PACKAGE_PATH` variable with the absolute path to package containing your Logstash config files, i.e., `LOGSTASH_PACKAGE_PATH=/home/user/Documents/Projects/platform/data-mapper-logstash`.
+- WARNING: do not edit the pipeline files from within the logstash container, or the group ID and user ID will change, and subsequently will result in file permission errors on your local file system.

data-mapper-logstash/dev-mnt-logstash.sh

@@ -0,0 +1,7 @@
+version: '3.9'
+
+services:
+  data-mapper-logstash:
+    volumes:
+      - ${LOGSTASH_PACKAGE_PATH}/pipeline/:/usr/share/logstash/pipeline/
+      - ${LOGSTASH_PACKAGE_PATH}/pipelines.yml:/usr/share/logstash/config/pipelines.yml

data-mapper-logstash/docker-compose.dev-mnt.yml

@@ -8,6 +8,8 @@
   "environmentVariables": {
     "ES_ELASTIC": "${ES_ELASTIC}",
     "LS_JAVA_OPTS": "-Xmx2g -Xms2g",
+    "LOGSTASH_DEV_MOUNT": "false",
+    "LOGSTASH_PACKAGE_PATH": "",
     "LOGSTASH_MEMORY_LIMIT": "3G",
     "LOGSTASH_MEMORY_RESERVE": "500M"
   },

data-mapper-logstash/package-metadata.json

@@ -4,6 +4,8 @@
 Action=$1
 Mode=$2
 
+readonly LOGSTASH_DEV_MOUNT=$LOGSTASH_DEV_MOUNT
+
 COMPOSE_FILE_PATH=$(
   cd "$(dirname "${BASH_SOURCE[0]}")" || exit
   pwd -P
@@ -61,16 +63,30 @@ else
   LogstashDevComposeParam=""
 fi
 
+if [[ "$LOGSTASH_DEV_MOUNT" == "true" ]]; then
+  if [[ -z $LOGSTASH_PACKAGE_PATH ]]; then
+    echo "ERROR: LOGSTASH_PACKAGE_PATH environment variable not specified. Please specify LOGSTASH_PACKAGE_PATH as stated in the README."
+    exit 1
+  fi
+
+  echo -e "\nRunning Data Mapper Logstash package with dev mount\n"
+  LogstashDevMountComposeParam="-c ${COMPOSE_FILE_PATH}/docker-compose.dev-mnt.yml"
+else
+  LogstashDevMountComposeParam=""
+fi
+
 if [[ "$Action" == "init" ]] || [[ "$Action" == "up" ]]; then
 
   config::set_config_digests "$COMPOSE_FILE_PATH"/docker-compose.yml
 
-  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.yml $LogstashDevComposeParam instant
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.yml $LogstashDevComposeParam $LogstashDevMountComposeParam instant
 
   AwaitContainerStartup
   AwaitContainerReady
 
-  config::copy_shared_configs "$COMPOSE_FILE_PATH"/package-metadata.json /usr/share/logstash/
+  if [[ "$LOGSTASH_DEV_MOUNT" != "true" ]]; then
+    config::copy_shared_configs "$COMPOSE_FILE_PATH"/package-metadata.json /usr/share/logstash/
+  fi
 
   echo "Removing stale configs..."
   config::remove_stale_service_configs "$COMPOSE_FILE_PATH"/docker-compose.yml "logstash"

data-mapper-logstash/swarm.sh

@@ -149,7 +149,7 @@ config::copy_shared_configs() {
 
     for sharedConfig in "${sharedConfigs[@]}"; do
         # TODO: (https://jembiprojects.jira.com/browse/PLAT-252) swap docker copy for a swarm compliant approach
-        docker cp "${packageBaseDir}""${sharedConfig//\"//}" "${containerId}":"${CONTAINER_DESTINATION}"
+        docker cp -a "${packageBaseDir}""${sharedConfig//\"//}" "${containerId}":"${CONTAINER_DESTINATION}"
     done
 }