cache

#!/bin/bash
#
# Managing a Semaphore project's dependency cache files

VERSION=0.3

# Allocated disk space limit

LIMIT=${CACHE_SIZE:-9437184}

# Verbose output log
VERBOSE_LOG=1

# LFTP Errors
E_NOSFL='Access failed: No such file'

# Script errors
E_FLMSNG=0 # file missing
E_FLAE=0 # file already exists

# Misc
DATE_FORMAT='%H:%M %d/%m/%Y'

cache::verbose() {
  VERBOSE_LOG=0
}

cache::normalize_string() {
  local word
  local result
  word=$1

  result=${word//\//-}
  if [ "$result" != "$1" ]; then
    cache::log "Key ${1} is normalized to ${result}."
  fi

  echo "$result"
}

cache::current_timestamp_in_milis() {
  DIST=$(uname)

  case $DIST in
    Darwin)
      gdate +%s%3N
      ;;
    Linux)
      date +%s%3N
     ;;
    *)
      echo "Unsupported distro $DIST"
      exit 1
    ;;
  esac
}

cache::duration() {

  local lftp_cmd
  local start
  local end
  local duration

  lftp_cmd=$1; shift
  start=$(cache::current_timestamp_in_milis)
  $lftp_cmd "$@"
  end=$(cache::current_timestamp_in_milis)
  duration=$(( $end - $start ))

  echo $duration > /tmp/duration

  return 0
}

cache::speed() {
  local duration
  local size

  duration=$1
  size=$2

  net_speed=$(( $size/$duration ))
  if [[ $net_speed > 0 ]]; then
    echo "cache.prod.speed:${net_speed}|ms" >> /tmp/semaphore-stats.txt
  fi

  return 0
}

################################################################################
# Send the files identified by the key, to the cache repository using LFTP
# Globals:
#   SEMAPHORE_CACHE_KEY
#   SEMAPHORE_LOCAL_CACHE_PATH
# Arguments:
#   cache_key, path
# Returns:
#   None
################################################################################
cache::store() {
  local usage

  usage='Usage: \n\t cache store [key] [local-path]\nExample: \n\t cache store bundle-$SEMAPHORE_GIT_BRANCH-$(checksum Gemfile.lock) vendor/bundle\n'

  if [[ $# -eq 0 ]]; then
    cache::log "${usage}"
  elif [[ $# -ne 2 ]]; then
    cache::err "Incorrect number of arguments!"
  else
    SEMAPHORE_LOCAL_CACHE_PATHS=$2
    SEMAPHORE_CACHE_KEY=$(cache::normalize_string $1)

    cache::lftp_put
  fi
}

cache::allocate_space() {
  local used_space
  local free_space
  local file
  local archive_size

  archive_size=$1

  file="/tmp/keys_data"
  cache::save_command_output "cls --sort=date -l" $file
  echo -e "$(cat $file | awk '{print $5}')" > "/tmp/usage_data"
  echo -e "$(cat $file | awk '{print $5, $9}')" > "/tmp/size_key"

  used_space=$(cache::used_space "/tmp/usage_data")
  free_space=$(cache::free_space ${used_space})
  rm -f "/tmp/usage_data"

  if [ $free_space -lt $archive_size ]; then
    cache::log "Not enough space, deleting the oldest keys."
    while [ $free_space -lt $archive_size ]; do
      deleted_key_size=$(cache::delete_oldest_key)
      free_space=$(($free_space + $deleted_key_size))
    done
  fi

  rm -f /tmp/size_key
  rm -f /tmp/keys_data
}

cache::delete_oldest_key() {
 local key
 local file

 file=/tmp/size_key

 size=$(cat /tmp/size_key | tail -1 | awk '{ print $1 }')
 key=$(cat /tmp/size_key | tail -1 | awk '{ print $2 }')
 cache::lftp "glob rm -f ${key}"

 cache::log "Key ${key} is deleted."
 sed -i '$d' $file
 echo $size
}

cache::lftp_put() {
  local archive_size
  local limit
  local allocated_space
  local duration

  limit=$(numfmt --from-unit=1024 $LIMIT)

  for local_path in ${SEMAPHORE_LOCAL_CACHE_PATHS//,/ }; do
    if ! [[ -e $local_path ]]; then
      cache::log "'${local_path}' doesn't exist locally."

      continue
    fi

    if cache::file_is_present ${SEMAPHORE_CACHE_KEY}; then
      cache::log "Key '${SEMAPHORE_CACHE_KEY}' already exists."
    else
      tar czf /tmp/$SEMAPHORE_CACHE_KEY $local_path

      archive_size=$(ls -l /tmp/$SEMAPHORE_CACHE_KEY | awk '{print $5}')
      if [ "${archive_size}" -gt "${limit}" ]; then
        allocated_space=$(numfmt --to=iec ${limit})
        cache::log "Archive exceeds allocated ${allocated_space} for cache."
      else
        cache::allocate_space $archive_size

        cache::log "Uploading '${local_path}' with cache key '${SEMAPHORE_CACHE_KEY}'..."
        cache::duration cache::lftp "put -c /tmp/${SEMAPHORE_CACHE_KEY}"
        if [[ -f /tmp/duration ]]; then
          duration=$(cat /tmp/duration)
          cache::speed $duration $archive_size
          rm -rf /tmp/duration
        fi
        rm -rf /tmp/$SEMAPHORE_CACHE_KEY
        cache::log "Upload complete."
      fi
    fi

  done
}

################################################################################
# Download the files identified by the key to the path
# Globals:
#   SEMAPHORE_CACHE_KEY
# Arguments:
#   None
# Returns:
#   None
################################################################################
cache::restore() {
  local usage
  local semaphore_cache_keys

  usage='Usage: \n\tcache restore [key,key_1,key_2]\nExample: \n\tcache store bundle-$SEMAPHORE_GIT_BRANCH-$SEMAPHORE_GIT_SHA\nFallback example:\n\tcache restore bundle-$SEMAPHORE_GIT_BRANCH-$(checksum Gemfile.lock),bundle-$SEMAPHORE_GIT_BRANCH\n'

  if [[ $# -eq 0 ]]; then
    cache::log "${usage}"
  elif [[ $# -ne 1 ]]; then
    cache::err "Incorrect number of arguments!"
  else
    semaphore_cache_keys=$1
    cache::download "keys" "/tmp/cache_keys"

    for key in ${semaphore_cache_keys//,/ }; do
      cache_key=$(cache::normalize_string $key)
      match=$(grep -m 1 ${cache_key} /tmp/cache_keys)

      if [[ ! -z "${match}" ]]; then
        exact_match=$(grep -x ${cache_key} /tmp/cache_keys)

        if [[ "${exact_match}" == "${cache_key}" ]]; then
          cache::log "HIT: ${cache_key}, using key ${cache_key}"
          cache::lftp_get $cache_key
          break
        else
          cache::log "HIT: ${cache_key}, using key ${match}"
          cache::lftp_get $match
          break
        fi

      else
        cache::log "MISS: ${cache_key}"
      fi
    done

    rm -f /tmp/cache_keys
  fi
}

cache::clear() {
  cache::lftp "glob -a rm -r *"

  cache::log "Deleted all caches."
}

cache::list() {
  if cache::is_not_empty; then
    cache::download "list" "/tmp/list_data"
    output=$(cat /tmp/list_data)
    cache::log "${output}"
    rm -f /tmp/list_data
  else
    cache::log "Cache is empty."
  fi
}

cache::usage() {
  local used_space
  local free_space
  local hr_used_space
  local hr_free_space
  local file

  file="/tmp/usage_data"
  cache::download "usage" $file

  used_space=$(cache::used_space $file)
  free_space=$(cache::free_space ${used_space})

  hr_used_space=$(numfmt --to=iec ${used_space})
  hr_free_space=$(numfmt --to=iec $free_space)

  cache::log "FREE SPACE: ${hr_free_space}"
  cache::log "USED SPACE: ${hr_used_space}"

  rm -f $file
}

cache::delete() {
  local key

  usage='Usage: \n\tcache delete [key]\nExample: \n\tcache delete bundle-$SEMAPHORE_GIT_BRANCH\n'

  if [[ $# -eq 0 ]]; then
    cache::log "${usage}"
  elif [[ $# -ne 1 ]]; then
    cache::err "Incorrect number of arguments!"
  else
    key=$(cache::normalize_string $1)

    if cache::file_is_present ${key}; then
      cache::lftp "glob rm -f ${key}"

      if cache::file_is_present ${key}; then
        cache::log "Failed to delete the key.."
      else
        cache::log "Key ${key} is deleted."
      fi
    else
      cache::log "Key ${key} doesn't exist in the cache store."
    fi
  fi
}

cache::has_key() {
  local key

  usage='Usage: \n\tcache has_key [key]\nExample: \n\tcache has_key bundle-$SEMAPHORE_GIT_BRANCH\n'

  if [[ $# -eq 0 ]]; then
    cache::log "${usage}"
  elif [[ $# -ne 1 ]]; then
    cache::err "Incorrect number of arguments!"
  else
    key=$(cache::normalize_string $1)

    if cache::file_is_present ${key}
    then
      cache::log "Key ${key} exists in the cache store."

      return 0
    else
      cache::log "Key ${key} doesn't exist in the cache store."

      return 1
    fi
  fi
}

cache::save_command_output() {
  local command
  local file
  local output

  command=$1
  file=$2

  output=$(echo -e "${command}" | lftp sftp://$SEMAPHORE_CACHE_USERNAME:DUMMY@$SEMAPHORE_CACHE_URL -e 'set sftp:connect-program "ssh -a -x -i '"${SEMAPHORE_CACHE_PRIVATE_KEY_PATH}"'"' 2>&1)

  echo -e "$output" > $file
}

cache::download() {
  local command
  local file

  file=$2

  case $1 in
    "keys" )
      command="cls --sort=date"

      cache::save_command_output "$command" $file

      ;;
    "usage" )
      command="cls --sort=date -l"

      cache::save_command_output "${command}" $file
      echo -e "$(cat $file | awk '{print $5}')" > $file

      ;;
    "list" )
      command="cls --sort=date -l"

      cache::save_command_output "$command" "${file}_tmp"

      printf "%-70s \t %10s \t %-10s\n" "KEY" "SIZE" "STORED AT" > $file
      echo -e "$(cat ${file}_tmp)" | awk '{ printf("%-70s \t %10s \t %-4s %2s %s\n", $9, $5, $6, $7, $8); }' | numfmt --field 2 --to=iec >> $file

      rm -f ${file}_tmp

      ;;
  esac
}

cache::used_space() {
  local file
  local sum
  file=$1

  sum=0

  while read line; do
    if [[ ! -z "$line" ]]; then sum=$((sum + $line)); fi
  done < $file

  echo "$sum"
}

cache::free_space() {
  local limit
  local difference

  used_space=$1
  limit=$(numfmt --from-unit=1024 $LIMIT)
  difference=$(($limit - $used_space))
  echo "$difference"
}

################################################################################
# Wrapper around LFTP 'get'
# Globals:
#   SEMAPHORE_CACHE_KEY_MD5
#   E_FLMSNG
# Arguments:
#   None
# Returns:
#   None
################################################################################
cache::lftp_get() {
  local key
  local archive_size
  local duration

  key=$1

  if cache::file_is_present $key; then
    cd /tmp
    cache::duration cache::lftp "get1 -c $key"
    cd - &>/dev/null
    restored_path=$(tar -ztvf /tmp/$key | head -1 | awk '{print $6}')
    tar xzf /tmp/$key -C .
    archive_size=$(ls -l /tmp/$key | awk '{print $5}')
    if [[ -f /tmp/duration ]]; then
      duration=$(cat /tmp/duration)
      cache::speed $duration $archive_size
      rm -rf /tmp/duration
    fi
    rm -rf /tmp/$key
    cache::log "Restored: ${restored_path}"
  else
    cache::log "Key '$key' does not exist in the cache store."
  fi
}

################################################################################
# Wrapper around LFTP
# Globals:
#   E_FLMSNG
#   E_NOSFL
#   W_FLAEX
#   SEMAPHORE_CACHE_URL
#   SEMAPHORE_CACHE_USERNAME
# Arguments:
#   command
#   log - when set to "with_logs" it prints command output log
# Returns:
#   None
################################################################################
cache::lftp() {
  local lftp_command
  local print_log
  local command_log
  local command_status

  lftp_command=$1
  print_log=${2:-false}

  if [[ -z $lftp_command ]]; then
    cache::err "LFTP command can't be empty!"
  fi

  command_log=$(echo -e "${lftp_command}" | lftp sftp://$SEMAPHORE_CACHE_USERNAME:DUMMY@$SEMAPHORE_CACHE_URL -e 'set sftp:connect-program "ssh -a -x -i '"${SEMAPHORE_CACHE_PRIVATE_KEY_PATH}"'"' 2>&1)
  command_status=$?

  if [[ "${print_log}" == "with_logs" ]] && [ -n "$command_log" ]; then
    cache::log "$command_log"
  fi

  # is_not_empty
  if [[ "${lftp_command}" == 'cls -l' ]]; then
    [ "$command_log" ] && return 0 || return 1
  fi

  if [[ "${lftp_command}" == 'cls --sort=date -l' ]] && [ -z "$command_log" ]; then
    cache::log "Cache is empty."
  fi

  # has_key and file_is_present
  if [[ "${lftp_command}" =~ 'cls' ]]; then
    if [[ "${command_log}" =~ "${E_NOSFL}" ]]; then
      E_FLMSNG=1

      return
    else
      E_FLMSNG=0
    fi
  fi

  if [[ "${command_log}" =~ "${E_NOSFL}" ]] && [[ "${lftp_command}" =~ 'cls' ]]; then
    E_FLMSNG=1

    return
  fi

  # Ignore the error when creating a directory which is already present
  if [[ "${command_log}" =~ 'File already exists' ]] && [[ "${lftp_command}" =~ 'mkdir' ]]; then
    W_FLAEX=1

    return
  fi

 # ignoring these lines in the output log until the debugging option is introduced
 #  if [[ $command_status -ne 0 ]]; then
 #    cache::log "Log:\n---\n${command_log}\n---\n"
 #    cache::err "Error while executing LFTP command '$lftp_command'"
 #   fi
}

################################################################################
# Calculates the md5 sum for the cache key (string, file)
# Globals:
#   SEMAPHORE_CACHE_KEY
#   SEMAPHORE_CACHE_KEY_MD5
# Arguments:
#   command
# Returns:
#   None
################################################################################

cache::calculate_key_md5() {
  if [[ -f $SEMAPHORE_CACHE_KEY ]]; then
    cache::log "Cache key is a file. Calculating md5 hash..."
    SEMAPHORE_CACHE_KEY_MD5=$(md5sum ${SEMAPHORE_CACHE_KEY})
  else
    cache::log "Cache key is a string. Calculating md5 hash..."
    SEMAPHORE_CACHE_KEY_MD5=$(echo ${SEMAPHORE_CACHE_KEY} | md5sum)
  fi


  if [[ $? -ne 0 ]]; then
    cache::err "Failed to generate md5 sum for the provided key. (${SEMAPHORE_CACHE_KEY})"
  else
    SEMAPHORE_CACHE_KEY_MD5=$(echo $SEMAPHORE_CACHE_KEY_MD5 | cut -d' ' -f1)

    cache::log "md5 hash: ${SEMAPHORE_CACHE_KEY_MD5}"
  fi
}

cache::print_usage() {
  echo -e "cache ${VERSION} | Utility for managing dependency caches"
  echo -e "cache [store|restore|list|clear|delete|has_key|usage] opts"
}

################################################################################
# Cheks the presence of the SSH key which is used to access the cache storage
# Globals:
#   SEMAPHORE_CACHE_PRIVATE_KEY_PATH
# Arguments:
#   none
# Returns:
#   none
################################################################################
cache::check_ssh_key() {
  if ! [[ -e $SEMAPHORE_CACHE_PRIVATE_KEY_PATH ]]; then
    cache::err "Private SSH key missing! (${SEMAPHORE_CACHE_PRIVATE_KEY_PATH})"
  fi
}

################################################################################
# Checks if 'lftp' is available
# Globals:
#   none
# Arguments:
#   none
# Returns:
#   none
################################################################################
cache::check_lftp() {
  if [[ $VERBOSE_LOG == 0 ]]; then
    cache::log "Checking if LFPT is present"
  fi

  if which lftp &> /dev/null; then
    return 0;
  else
    cache::err "The 'lftp' executable is missing or not in the \$PATH"
  fi
}

################################################################################
# Checks if the required environment variables are set
# Globals:
#   SEMAPHORE_CACHE_URL
#   SEMAPHORE_CACHE_USERNAME
#   SEMAPHORE_CACHE_PRIVATE_KEY_PATH
# Arguments:
#   none
# Returns:
#   none
################################################################################
cache::check_env() {
  if [[ $VERBOSE_LOG == 0 ]]; then
    cache::log "Checking environment variables"
  fi

  if [[ -z $SEMAPHORE_CACHE_URL ]]; then
    cache::err "'SEMAPHORE_CACHE_URL' environment variable not set!"
  fi

  if [[ -z $SEMAPHORE_CACHE_USERNAME ]]; then
    cache::err "'SEMAPHORE_CACHE_USERNAME' environment variable not set!"
  fi

  if [[ -z $SEMAPHORE_CACHE_PRIVATE_KEY_PATH ]]; then
    cache::err "'SEMAPHORE_CACHE_PRIVATE_KEY_PATH' environment variable not set!"
  fi
}

################################################################################
# Helper function to check if file is present on cache directory
# Globals:
#   E_FLMSNG
# Arguments:
#   key (file name)
# Returns:
#   true or false
################################################################################
cache::file_is_present() {
  local key

  key=$1
  if [[ $VERBOSE_LOG == 0 ]]; then
    cache::log "Checking if key $key is present in cache store."
  fi

  cache::lftp "cls ${key}"

  if [[ "$E_FLMSNG" == "1" ]]; then
    false
  elif [[ "$E_FLMSNG" == "0" ]]; then
    true
  fi
}


################################################################################
# Helper function to check if file is present on cache directory
# Globals:
#   E_FLMSNG
# Arguments:
#   key (file name)
# Returns:
#   0 or 1
################################################################################
cache::is_not_empty() {
  cache::lftp "cls -l"
  return $?
}

################################################################################
# Helper function to show error messages and to terminate execution on error
# Globals:
#   DATE_FORMAT
# Arguments:
#   message
# Returns:
#   1
################################################################################
cache::err() {
  echo -e "\n! [$(date +"${DATE_FORMAT}")]: $@\n" >&2
}


################################################################################
# Helper function to show log messages
# Globals:
#   DATE_FORMAT
# Arguments:
#   message
# Returns:
#   none
################################################################################
cache::log() {
  if [[ $VERBOSE_LOG == 0 ]]; then
    echo -e "[$(date +"${DATE_FORMAT}")]: $@" >&2
  else
    echo -e "$@" >&2
  fi
}

cache::main() {
  cache::check_lftp
  cache::check_ssh_key
  cache::check_env

  case "$1" in
    "restore" )
      shift

      cache::restore "$@"
      ;;
    "store" )
      shift

      cache::store "$@"
      ;;
    "clear" )
      shift

      cache::clear "$@"
      ;;
    "list" )
      shift

      cache::list "$@"
      ;;
    "usage" )
      shift

      cache::usage "$@"
      ;;
    "is_not_empty" )
      shift

      cache::is_not_empty "$@"
      ;;
    "has_key" )
      shift

      cache::has_key "$@"
      ;;
    "delete" )
      shift

      cache::delete "$@"
      ;;
    * )
      cache::print_usage
      ;;
  esac
}

cache::main "$@"