Merge pull request #645 from geoadmin/fix_limit_memory

limit the available memory for the sphinx index creation

Author: ltclm

Makefile

@@ -73,11 +73,18 @@ endif
 
 PPID := $(shell echo $$PPID)
 
+# Docker resosource throttling
+# we allow 70% of the currently available memory to be used by sphinx indexer per job
+PERCENTAGE := 70
+free_mem := $(shell free -m | awk '/Mem:/{print $$4}')
+export DOCKER_FREE_MEM := $(shell echo $$(( ${free_mem} * ${PERCENTAGE} / 100 ))m)
+
 # Maintenance / Index Commands
 # EFS Index will be mounted as bind mount
 # DOCKER_EXEC will always check if a newer image exists on ecr -> develop.latest support
 export DOCKER_EXEC :=  docker run \
 				--rm \
+				--memory=$(DOCKER_FREE_MEM) \
 				-t \
 				-v $(SPHINX_EFS):/var/lib/sphinxsearch/data/index/ \
 				--env-file $(ENV_FILE) \
@@ -86,6 +93,7 @@ export DOCKER_EXEC :=  docker run \
 
 export DOCKER_EXEC_LOCAL :=  docker run \
 				--rm \
+				--memory=$(DOCKER_FREE_MEM) \
 				-t \
 				-v $(CURRENT_DIR)/conf/:/var/lib/sphinxsearch/data/index/ \
 				--env-file $(ENV_FILE) \
@@ -153,6 +161,7 @@ help:
 	@echo
 	@echo "- CPUS:                     ${YELLOW}${CPUS}${RESET}"
 	@echo "- DB_ACCESS:                ${YELLOW}${DB_ACCESS}${RESET}"
+	@echo "- DOCKER_FREE_MEM:          ${YELLOW}${DOCKER_FREE_MEM}${RESET}"
 
 
 # Build targets. Calling setup is all that is needed for the local files to be installed as needed.

scripts/pg2sphinx.sh

@@ -8,6 +8,16 @@ docker_is_logged_in() {
     docker pull "${DOCKER_IMG_LOCAL_TAG}" &> /dev/null
 }
 
+throw_error() {
+    error=$1
+    echo "index generation failed"
+    echo "the docker command was: ${DOCKER_EXEC}"
+    free -m
+    docker stats --no-stream
+    echo "original error code: ${error}"
+    exit "${error}"
+}
+
 # check if we have read-write access to the efs
 if ! /bin/test -d "${SPHINX_EFS}" -a -w "${SPHINX_EFS}"; then
     >&2 echo "no read-write access to folder ${SPHINX_EFS} available"
@@ -21,12 +31,12 @@ fi
 
 if [ -n "${DB:-}" ]; then
     # call pg2sphinx trigger with DATABASE pattern
-    ${DOCKER_EXEC} python3 pg2sphinx_trigger.py -s /etc/sphinxsearch/sphinx.conf -c update -d "${DB}"
+    ${DOCKER_EXEC} python3 pg2sphinx_trigger.py -s /etc/sphinxsearch/sphinx.conf -c update -d "${DB}" || { throw_error $?; }
 fi
 
 if [ -n "${INDEX:-}" ]; then
     # call pg2sphinx trigger with INDEX pattern
-    ${DOCKER_EXEC} python3 pg2sphinx_trigger.py -s /etc/sphinxsearch/sphinx.conf -c update -i "${INDEX}"
+    ${DOCKER_EXEC} python3 pg2sphinx_trigger.py -s /etc/sphinxsearch/sphinx.conf -c update -i "${INDEX}" || { throw_error $?; }
 fi
 
 mapfile -t array_config < <(${DOCKER_EXEC} cat /etc/sphinxsearch/sphinx.conf | grep -E "^[^#]+ path"  | awk -F"=" '{print $2}' | sed -n 's|^.*/||p' | sed 's/\r$//')