datadog-agent.init

#!/bin/sh

### BEGIN INIT INFO
# Provides: dd-agent
# Short-Description: Start and stop dd-agent
# Description: dd-agent is the monitoring Agent component for Datadog
# Required-Start: $remote_fs
# Required-Stop: $remote_fs
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
### END INIT INFO

. /lib/lsb/init-functions
PATH=$PATH:/sbin # add the location of start-stop-daemon on Debian

AGENTPATH="/opt/datadog-agent/agent/agent.py"
AGENTCONF="/etc/dd-agent/datadog.conf"
DOGSTATSDPATH="/opt/datadog-agent/agent/dogstatsd.py"
KILL_PATH="/opt/datadog-agent/embedded/bin/kill"
AGENTUSER="dd-agent"
FORWARDERPATH="/opt/datadog-agent/agent/ddagent.py"
TRACEAGENTPATH="/opt/datadog-agent/bin/trace-agent"
NAME="datadog-agent"
DESC="Datadog Agent"
SUPERVISOR_PIDFILE="/opt/datadog-agent/run/datadog-supervisord.pid"
SUPERVISOR_FILE="/etc/dd-agent/supervisor.conf"
SUPERVISOR_SOCK="/opt/datadog-agent/run/datadog-supervisor.sock"
SUPERVISORCTL_PATH="/opt/datadog-agent/bin/supervisorctl"
SUPERVISORD_PATH="/opt/datadog-agent/bin/supervisord"
COLLECTOR_PIDFILE="/opt/datadog-agent/run/dd-agent.pid"
SYSTEM_PATH=/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:$PATH

# Captures processes in the supervisord group that are 'optional' i.e. not essential for the agent to run
DD_OPT_PROC_REGEX="dogstatsd|jmxfetch|go-metro|trace-agent"

if [ ! -x $AGENTPATH ]; then
    echo "$AGENTPATH not found. Exiting."
    exit 0
fi

check_status() {

    QUERY="all"
    if [ -n "$1" ] && [ "$1" != "all" ] && [ "$1" != "essential" ] && [ "$1" != "optional" ]; then
        echo "Valid arguments for $0: all | essential | optional"
    elif [ -n "$1" ]; then
        QUERY=$1
    fi

    ESSENTIAL=0
    OPTIONAL=0
    FAIL=1

    # If the socket exists, we can use supervisorctl
    if [ -e $SUPERVISOR_SOCK ]; then
        # If we're using supervisor, check the number of datadog processes
        # supervisor is currently controlling, and make sure that it's the
        # same as the number of programs specified in the supervisor config
        # file:

        supervisor_processes=$($SUPERVISORCTL_PATH -c $SUPERVISOR_FILE status)

        # Number of RUNNING supervisord programs (ignoring optional procs captured by $DD_OPT_PROC_REGEX)
        datadog_supervisor_processes=$(echo "$supervisor_processes" |
                                       grep -Ev $DD_OPT_PROC_REGEX |
                                       grep $NAME |
                                       grep -c RUNNING)

        # Number of non-failed OPTIONAL supervisord programs
        datadog_supervisor_opt_processes=$(echo "$supervisor_processes" |
                                       grep -E $DD_OPT_PROC_REGEX |
                                       grep $NAME |
                                       grep -cv FATAL)

        # Number of expected running supervisord programs (ignoring optional procs)
        supervisor_config_programs=$(grep -Ev $DD_OPT_PROC_REGEX $SUPERVISOR_FILE |
                                     grep -c '\[program:')

        # Number of expected optional supervisord programs
        supervisor_config_opt_programs=$(grep -E $DD_OPT_PROC_REGEX $SUPERVISOR_FILE |
                                     grep -c '\[program:')

        if [ "$datadog_supervisor_processes" -ne "$supervisor_config_programs" ]; then
            ESSENTIAL=1
        fi

        if [ "$datadog_supervisor_opt_processes" -ne "$supervisor_config_opt_programs" ]  ; then
            OPTIONAL=1
        fi

        if [ "$QUERY" = "essential" ] && [ "$ESSENTIAL" -eq 0 ]; then
            FAIL=0
        elif [ "$QUERY" = "optional" ] && [ "$OPTIONAL" -eq 0 ]; then
            FAIL=0
        elif [ "$QUERY" = "all" ] && [ "$ESSENTIAL" = "$OPTIONAL" ] && [ $ESSENTIAL -eq 0 ]; then
            FAIL=0
        fi

        if [ "$FAIL" -gt 0 ]; then
            echo "$supervisor_processes"
            echo "$DESC (supervisor) is NOT running all child processes"
            return 1
        else
            echo "$DESC (supervisor) is running all child processes"
            return 0
        fi


    else
        echo "$DESC (supervisor) is not running"
        return 1
    fi
}

# Action to take
case "$1" in
    start)
        if [ ! -f $AGENTCONF ]; then
            echo "$AGENTCONF not found. Exiting."
            exit 3
        fi

        check_status essential > /dev/null
        if [ $? -eq 0 ]; then
            echo "$DESC is already running"
            exit 0
        else
            $0 stop > /dev/null 2>&1
        fi

        su $AGENTUSER -c "$AGENTPATH configcheck" > /dev/null 2>&1
        if [ $? -ne 0 ]; then
            log_daemon_msg "Invalid check configuration. Please run sudo /etc/init.d/datadog-agent configtest for more details."
            log_daemon_msg "Resuming starting process."
        fi


        log_daemon_msg "Starting $DESC (using supervisord)" "$NAME"
        PATH=$SYSTEM_PATH start-stop-daemon --start --quiet --oknodo --exec $SUPERVISORD_PATH -- -c $SUPERVISOR_FILE --pidfile $SUPERVISOR_PIDFILE
        if [ $? -ne 0 ]; then
            log_end_msg 1
        fi

        # check if the agent is running once per second for 10 seconds
        retries=10
        while [ $retries -gt 1 ]; do
          if check_status essential > /dev/null; then
              # We've started up successfully. Exit cleanly
              log_end_msg 0
              exit 0
          else
              retries=$(($retries - 1))
              sleep 1
          fi
        done
        # After 10 tries the agent didn't start. Report an error
        log_end_msg 1
        check_status # report what went wrong
        $0 stop
        exit 1
        ;;
    stop)
        log_daemon_msg "Stopping $DESC (stopping supervisord)" "$NAME"
        start-stop-daemon --stop --retry 30 --quiet --oknodo --pidfile $SUPERVISOR_PIDFILE

        log_end_msg $?

        ;;

    info)
        shift # Shift 'info' out of args so we can pass any
              # addtional options to the real command
              # (right now only dd-agent supports additional flags)
        su $AGENTUSER -c "$AGENTPATH info $@"
        COLLECTOR_RETURN=$?
        su $AGENTUSER -c "$DOGSTATSDPATH info"
        DOGSTATSD_RETURN=$?
        su $AGENTUSER -c "$FORWARDERPATH info"
        FORWARDER_RETURN=$?
        su $AGENTUSER -c "$TRACEAGENTPATH -info"
        TRACEAGENT_RETURN=$? # not used for now, Trace/APM not a core feature yet
        exit $(($COLLECTOR_RETURN+$DOGSTATSD_RETURN+$FORWARDER_RETURN))
        ;;

    status)
        # Note: sh does not support arrays
        # Check for kernel version 3.18+ - overlayfs has known bug affecting unix domain sockets
        major=$(echo "$( uname -r )" | cut -d"." -f1)
        minor=$(echo "$( uname -r )" | cut -d"." -f2)
        # If major version 3, and minor version 18+, OR major version 4+
        if [ "$DOCKER_DD_AGENT" != "" ] && ( ( [ $major -eq 3 ] && [ $minor -ge 18 ] ) || [ $major -gt 3 ] ); then
            RED='\033[0;31m' # Red Text
            NC='\033[0m' # No Color
            echo "${RED}Warning: Known bug in Linux Kernel 3.18+ causes 'status' to fail.${NC}"
            echo "Calling 'info', instead..."
            service datadog-agent info
            if [ $? -ne 0 ]; then
                exit 1
            fi
        else
            check_status
        fi
        exit $?
        ;;

    reload)
        $KILL_PATH -HUP `cat $COLLECTOR_PIDFILE`
        exit $?
        ;;

    restart|force-reload)
        $0 stop
        $0 start
        ;;

    condrestart|try-reload)
        check_status essential > /dev/null
        if [ $? -eq 0 ]; then
            $0 stop
            $0 start
        fi
        ;;

    configcheck)
        su $AGENTUSER -c "$AGENTPATH configcheck"
        exit $?
        ;;

    configtest)
        su $AGENTUSER -c "$AGENTPATH configcheck"
        exit $?
        ;;

    jmx)
        shift
        su $AGENTUSER -c "$AGENTPATH jmx $@"
        exit $?
        ;;

    flare)
        shift
        $AGENTPATH flare $@
        exit $?
        ;;

    *)
        N=/etc/init.d/$NAME
        echo "Usage: $N {start|stop|restart|info|status|reload|configcheck|configtest|jmx|flare}"
        exit 1
        ;;
esac

exit $?