Skip to content

fix: avoid FailedPreStopHook events for Kubernetes #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pvbouwel wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: avoid FailedPreStopHook events for Kubernetes #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions docker/AlmaLinux/Dockerfile.creo
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ ARG JMX_PROMETHEUS_JAVAAGENT_VERSION=0.13.0
USER root

COPY docker/AlmaLinux/entrypoint.sh /opt/entrypoint.sh
# To avoid FailedPreStopHook
COPY docker/AlmaLinux/process_wrapper.sh /opt/process_wrapper.sh

COPY docker/AlmaLinux/decom.sh /opt/decom.sh
COPY --chmod=700 docker/AlmaLinux/addlocalnodeip /bin/addlocalnodeip

Expand All @@ -23,7 +26,7 @@ RUN adduser -u 18585 -d /opt/spark/work-dir spark && \
chown 18585:18585 /opt/spark/work-dir && \
printf "[global]\nextra-index-url = https://artifactory.vgt.vito.be/artifactory/api/pypi/python-openeo/simple\n" > /etc/pip.conf && \
curl -O https://artifactory.vgt.vito.be/artifactory/libs-release/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_PROMETHEUS_JAVAAGENT_VERSION}/jmx_prometheus_javaagent-${JMX_PROMETHEUS_JAVAAGENT_VERSION}.jar && \
chmod +x /opt/entrypoint.sh /opt/decom.sh && \
chmod +x /opt/entrypoint.sh /opt/process_wrapper.sh /opt/decom.sh && \
PYTHONPLATLIBDIR=lib64 /opt/venv/bin/python3 -m pip install -I --upgrade pip && \
chown -R 18585:18585 $SPARK_HOME/jars && \
PYTHONPLATLIBDIR=lib64 /opt/venv/bin/python3 -m pip install py4j boto3==1.35.99 kubernetes==12.0.1 PyYAML==5.3.1 Jinja2==3.1.4 spark_memlogger==0.6 && \
Expand All @@ -45,6 +48,6 @@ ENV LANG=en_US.UTF-8

WORKDIR /opt/spark/work-dir

ENTRYPOINT [ "/opt/entrypoint.sh" ]
ENTRYPOINT [ "/opt/process_wrapper.sh", "/opt/entrypoint.sh", "-o", "json", "--" ]

USER 18585
2 changes: 1 addition & 1 deletion docker/AlmaLinux/entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,4 +103,4 @@ case "$1" in
esac

# Execute the container CMD under tini for better hygiene
exec /usr/bin/tini -s -- "${CMD[@]}"
exec /usr/bin/tini -s -g -- "${CMD[@]}"
189 changes: 189 additions & 0 deletions docker/AlmaLinux/process_wrapper.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
#!/usr/bin/env bash
# A basic process wrapper which takes as first argument
# a path to a binary that is meant to be wrapped. Next it takes its own
# configs parameters up until an argument "--" after which all
# remaining arguments are passed on to the wrapped process as is.

TARGET_EXECUTABLE="$1"
shift 1

# Defaults
## How long to await in seconds between checking up on the wrapped process
POLL_TIME=3
## How long to sleep after the wrapped process has finished and before exiting ourselves
SLEEP_TIME=1
## At which verbosity to log
if [[ "$LOG_LEVEL" == "" ]]
then
LOG_LEVEL="ERROR"
fi

# Logging functions logging is possible in both JSON and text based

function utc_now() {
echo "$(date -u +%Y-%m-%dT"%T.%3N")"
}

function JSON_LOG() {
LEVEL="$1"
if [[ "${LEVEL}" == "DEBUG" ]] && [[ "${LOG_LEVEL}" != "DEBUG" ]]
then
return
fi
MESSAGE="$2"
shift 2

EXTRA_FIELDS=""
while (( "$#" ))
do
if [[ "$2" == "" ]]
then
LOG_WARNING "Bad log statement unmatched argument" "unmatched_key" "$1" "Original_message" "${MESSAGE}"
break
fi
EXTRA_FIELDS="${EXTRA_FIELDS}, \"$1\": \"$2\""

shift 2
done

echo "{\"ts\": \"`utc_now`\": \"levelname\": \"${LEVEL}\", \"message\": \"${MESSAGE}\", \"filename\": \"$0\", \"job_id\": \"$OPENEO_BATCH_JOB_ID\", \"user_id\": \"$OPENEO_USER_ID\"${EXTRA_FIELDS}}"
}

function TEXT_LOG() {
LEVEL="$1"
MESSAGE="$2"
shift 2
echo "${LEVEL}:`utc_now`:${MESSAGE} $@"
}

# By default LOG_DEBUG is a no-op
function LOG_DEBUG() {
if [ "$LOG_LEVEL" == "DEBUG" ]
then
TEXT_LOG "DEBUG" "$@"
fi
}

function LOG_INFO() {
if [[ "$LOG_LEVEL" == "DEBUG" ]] || [[ "$LOG_LEVEL" == "INFO" ]]
then
TEXT_LOG "INFO" "$@"
fi
}

function LOG_ERROR() {
TEXT_LOG "ERROR" "$@"
exit 1
}

# Process the arguments for the wrapper
ARGUMENT_DELIMITER_FOUND=false

while (( "$#" ))
do
case "$1" in
-o|--output-format) _LOG_FORMAT="$2"
shift 2
case "$_LOG_FORMAT" in
txt) LOG_DEBUG "Default is text log format keeping loggers"
;;
json)
function LOG_DEBUG() {
if [ "$LOG_LEVEL" == "DEBUG" ]
then
JSON_LOG "DEBUG" "$@"
fi
}
function LOG_INFO() {
if [[ "$LOG_LEVEL" == "DEBUG" ]] || [[ "$LOG_LEVEL" == "INFO" ]]
then
JSON_LOG "INFO" "$@"
fi
}
function LOG_ERROR() {
JSON_LOG "ERROR" "$@"
exit 1
}

esac
;;
-l|--log-level) LOG_LEVEL="$(echo $2 | tr '[:lower:]' '[:upper:]')"
shift 2
LOG_DEBUG "Loglevel set via CLI" "LOG_LEVEL" "${LOG_LEVEL}"
;;
-s|--sleep-seconds) SLEEP_TIME="$(echo $2 | grep "^[0-9][.0-9]*$")"
if [[ "$SLEEP_TIME" != "$2" ]]
then
LOG_ERROR "Sleep time must be number of seconds so a valid numeric value" "bad_value" "$2"
fi
shift 2
LOG_DEBUG "Sleep time for after process finish set via CLI" "SLEEP_SECONDS" "$SLEEP_TIME"
;;
-p|--poll-seconds) POLL_TIME="$(echo $2 | grep "^[0-9][.0-9]*$")"
if [[ "$POLL_TIME" != "$2" ]]
then
LOG_ERROR "Poll time must be number of seconds so a valid numeric value" "bad_value" "$2"
fi
shift 2
LOG_DEBUG "Poll time between checks of the wrapped process set via CLI" "POLL_SECONDS" "$POLL_TIME"
;;
--) ARGUMENT_DELIMITER_FOUND=true
shift
break
;;

*) LOG_ERROR "Unsupported argument $1" "remaining args" "$@"
;;
esac
done


# Start the wrapping logic

if [[ -x "$TARGET_EXECUTABLE" ]]
then
LOG_INFO "Wrapping '$TARGET_EXECUTABLE'" "Args" "$@"
else
LOG_ERROR "Cannot wrap '$TARGET_EXECUTABLE' since it is not a path to an executable file"
fi

#Started the wrapped process in the background and store it's PID
$TARGET_EXECUTABLE "$@" &
WRAPPED_PID="$!"

# Now that we know the PID of the wrapped process create a helper function
# to send signals to it

function send_signal_to_child() {
SIGNAL_NUMBER="$1"

kill -s ${SIGNAL_NUMBER} ${WRAPPED_PID} 2>/dev/null
}

# Install signal handlers for all signals that can be trapped
# SIGKIL (9) and SIGSTOP (19) cannot
for sig in `/bin/kill -l | /bin/sed -e 's/KILL\s//' -e 's/STOP\s//'`;
do
#Installing signal sender ${sig}
trap "send_signal_to_child ${sig}" ${sig} 2>/dev/null
done

# Sleep and wake up to not do blocked waiting
# Get the process ID and make sure no whitespace from ps output
while [ "$(ps -p ${WRAPPED_PID} -o pid= | tr -d ' ')" == "${WRAPPED_PID}" ]
do
if [ "${LOG_LEVEL}" == "DEBUG" ]
then
LOG_DEBUG "Wrapped process still exists" "PID" "$WRAPPED_PID"
fi
sleep "$POLL_TIME"
done

# Blocking wait to get exit code
wait ${WRAPPED_PID}
STATUS="$?"

LOG_INFO "Wrapped process ended" "PID" "$WRAPPED_PID" "exit_code" "$STATUS"

sleep $SLEEP_TIME
exit ${STATUS}