Merge pull request #64 from startupturbo/dmius-ebs-size-option

ebs-disk-size and before-init-sql-code options added

Author: dmius

nancy_run.sh

@@ -2,7 +2,7 @@
 
 DEBUG=0
 CURRENT_TS=$(date +%Y%m%d_%H%M%S%N_%Z)
-DOCKER_MACHINE="${DOCKER_MACHINE:-nancy-$CURRENT_TS}"
+DOCKER_MACHINE="nancy-$CURRENT_TS"
 DOCKER_MACHINE="${DOCKER_MACHINE//_/-}"
 DEBUG_TIMEOUT=0
 EBS_SIZE_MULTIPLIER=15
@@ -198,6 +198,9 @@ while true; do
     --after-db-init-code )
       #s3 url|filename|content
       AFTER_DB_INIT_CODE="$2"; shift 2 ;;
+    --before-db-init-code )
+      #s3 url|filename|content
+      BEFORE_DB_INIT_CODE="$2"; shift 2 ;;
     --workload-real )
       #s3 url
       WORKLOAD_REAL="$2"; shift 2 ;;
@@ -236,15 +239,16 @@ while true; do
       TMP_PATH="$2"; shift 2 ;;
     --debug-timeout )
       DEBUG_TIMEOUT="$2"; shift 2 ;;
-    -- )
-      >&2 echo "ERROR: Invalid option '$1'"
-      exit 1;
-      break ;;
+    --ebs-volume-size )
+        EBS_VOLUME_SIZE="$2"; shift 2 ;;
     * )
-      if [ "${1:0:2}" == "--" ]; then
+      option=$1
+      option="${option##*( )}"
+      option="${option%%*( )}"
+      if [ "${option:0:2}" == "--" ]; then
         >&2 echo "ERROR: Invalid option '$1'. Please double-check options."
         exit 1
-      elif [ "${1:0:2}" != "" ]; then
+      elif [ "$option" != "" ]; then
         >&2 echo "ERROR: \"nancy run\" does not support payload (except \"help\"). Use options, see \"nancy run help\")"
         exit 1
       fi
@@ -276,6 +280,8 @@ if [ $DEBUG -eq 1 ]; then
   echo "s3-cfg-path: $S3_CFG_PATH"
   echo "tmp-path: $TMP_PATH"
   echo "after-db-init-code: $AFTER_DB_INIT_CODE"
+  echo "before-db-init-code: $BEFORE_DB_INIT_CODE"
+  echo "ebs-volume-size: $EBS_VOLUME_SIZE"
 fi
 
 function checkPath() {
@@ -467,6 +473,17 @@ function checkParams() {
     fi
   fi
 
+  if [ ! -z ${BEFORE_DB_INIT_CODE+x} ]; then
+    checkPath BEFORE_DB_INIT_CODE
+    if [ "$?" -ne "0" ]; then
+      #>&2 echo "WARNING: Value given as after_db_init_code: '$AFTER_DB_INIT_CODE' not found as file will use as content"
+      echo "$BEFORE_DB_INIT_CODE" > $TMP_PATH/before_db_init_code_tmp.sql
+      BEFORE_DB_INIT_CODE="$TMP_PATH/before_db_init_code_tmp.sql"
+    else
+      [ "$DEBUG" -eq "1" ] && echo "DEBUG: Value given as before_db_init_code will use as filename"
+    fi
+  fi
+
   if [ ! -z ${TARGET_DDL_DO+x} ]; then
     checkPath TARGET_DDL_DO
     if [ "$?" -ne "0" ]; then
@@ -499,32 +516,50 @@ function checkParams() {
       [ "$DEBUG" -eq "1" ] && echo "DEBUG: Value given as target_config will use as filename"
     fi
   fi
+
+  if [ ! -z ${EBS_VOLUME_SIZE+x} ]; then
+    if [ "$RUN_ON" == "localhost" ] || [ ${AWS_EC2_TYPE:0:2} == 'i3' ]; then
+      >&2 echo "WARNING: ebs-volume-size is not required for aws i3 aws instances and local execution."
+    fi;
+    re='^[0-9]+$'
+    if ! [[ $EBS_VOLUME_SIZE =~ $re ]] ; then
+      >&2 echo "ERROR: ebs-volume-size must be numeric integer value."
+      exit 1;
+    fi
+  else
+    if [ ! ${AWS_EC2_TYPE:0:2} == 'i3' ]; then
+      >&2 echo "WARNING: ebs-volume-size is not given, will be calculate on base of dump size."
+    fi
+  fi
 }
 
 checkParams;
 
 # Determine dump file size
-if [ ! -z ${DB_DUMP_PATH+x} ]; then
+if ([ "$RUN_ON" == "aws" ] && [ ! ${AWS_EC2_TYPE:0:2} == "i3" ] && \
+   [ -z ${EBS_VOLUME_SIZE+x} ] && [ ! -z ${DB_DUMP_PATH+x} ]); then
+    echo "Calculate EBS volume size."
     dumpFileSize=0
     if [[ $DB_DUMP_PATH =~ "s3://" ]]; then
       dumpFileSize=$(s3cmd info $DB_DUMP_PATH | grep "File size:" )
       dumpFileSize=${dumpFileSize/File size:/}
       dumpFileSize=${dumpFileSize/\t/}
       dumpFileSize=${dumpFileSize// /}
-      #echo "S3 FILESIZE: $dumpFileSize"
+      [ $DEBUG -eq 1 ] && echo "S3 FILESIZE: $dumpFileSize"
     else
       dumpFileSize=$(stat -c%s "$DB_DUMP_PATH")
     fi
-    [ $DEBUG -eq 1 ] && echo "Dump filesize: $dumpFileSize bytes"
+    let dumpFileSize=dumpFileSize*$EBS_SIZE_MULTIPLIER
     KB=1024
     let minSize=300*$KB*$KB*$KB
     ebsSize=$minSize # 300 GB
     if [ "$dumpFileSize" -gt "$minSize" ]; then
         let ebsSize=$dumpFileSize
-        let ebsSize=$ebsSize*$EBS_SIZE_MULTIPLIER
         ebsSize=$(numfmt --to-unit=G $ebsSize)
-        EBS_SIZE=$ebsSize
-        [ $DEBUG -eq 1 ] && echo "EBS Size: $EBS_SIZE Gb"
+        EBS_VOLUME_SIZE=$ebsSize
+        [ $DEBUG -eq 1 ] && echo "EBS volume size: $EBS_VOLUME_SIZE Gb"
+    else
+      echo "EBS volume is not require."
     fi
 fi
 
@@ -594,6 +629,7 @@ function cleanupAndExit {
   echo "Remove temp files..." # if exists
   docker $dockerConfig exec -i ${containerHash} sh -c "sudo rm -rf $MACHINE_HOME"
   rm -f "$TMP_PATH/after_db_init_code_tmp.sql"
+  rm -f "$TMP_PATH/before_db_init_code_tmp.sql"
   rm -f "$TMP_PATH/workload_custom_sql_tmp.sql"
   rm -f "$TMP_PATH/target_ddl_do_tmp.sql"
   rm -f "$TMP_PATH/target_ddl_undo_tmp.sql"
@@ -716,10 +752,10 @@ elif [[ "$RUN_ON" = "aws" ]]; then
   else
     echo "Attempt use external disk"
     # Create new volume and attach them for non i3 instances if needed
-    if [ ! -z ${EBS_SIZE+x} ]; then
+    if [ ! -z ${EBS_VOLUME_SIZE+x} ]; then
       echo "Create and attach EBS volume"
-      [ $DEBUG -eq 1 ] && echo "Create volume with size: $EBS_SIZE Gb"
-      VOLUME_ID=$(aws ec2 create-volume --size $EBS_SIZE --region us-east-1 --availability-zone us-east-1a --volume-type gp2 | jq -r .VolumeId)
+      [ $DEBUG -eq 1 ] && echo "Create volume with size: $EBS_VOLUME_SIZE Gb"
+      VOLUME_ID=$(aws ec2 create-volume --size $EBS_VOLUME_SIZE --region us-east-1 --availability-zone us-east-1a --volume-type gp2 | jq -r .VolumeId)
       INSTANCE_ID=$(docker-machine ssh $DOCKER_MACHINE curl -s http://169.254.169.254/latest/meta-data/instance-id)
       sleep 10 # wait to volume will ready
       attachResult=$(aws ec2 attach-volume --device /dev/xvdf --volume-id $VOLUME_ID --instance-id $INSTANCE_ID --region us-east-1)
@@ -792,6 +828,13 @@ function copyFile() {
 # Dump
 sleep 2 # wait for postgres up&running
 
+echo "Apply sql code before db init"
+if ([ ! -z ${BEFORE_DB_INIT_CODE+x} ] && [ "$BEFORE_DB_INIT_CODE" != "" ])
+then
+  BEFORE_DB_INIT_CODE_FILENAME=$(basename $BEFORE_DB_INIT_CODE)
+  copyFile $BEFORE_DB_INIT_CODE
+  docker_exec bash -c "psql --set ON_ERROR_STOP=on -U postgres test -b -f $MACHINE_HOME/$BEFORE_DB_INIT_CODE_FILENAME"
+fi
 echo "Restore database dump"
 case "$DB_DUMP_EXT" in
   sql)

tests/nancy_run_before_init_code.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+output=$(${BASH_SOURCE%/*}/../nancy run \
+  --before-db-init-code "select abs from beforeinittable;" \
+  --workload-custom-sql "file://$srcDir/custom.sql" \
+  --db-dump-path "file://$srcDir/test.dump.bz2" \
+  --tmp-path $srcDir/tmp \
+  2>&1)
+
+if [[ $output =~ "ERROR:  relation \"beforeinittable\" does not exist" ]]; then
+  echo -e "\e[36mOK\e[39m"
+else
+  >&2 echo -e "\e[31mFAILED\e[39m"
+  >&2 echo -e "Output: $output"
+  exit 1
+fi

tests/nancy_run_ebs_disk_size.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+output=$(${BASH_SOURCE%/*}/../nancy run \
+  --debug \
+  --ebs-volume-size 37 \
+  2>&1)
+
+if [[ $output =~ "ebs-volume-size: 37" ]]; then
+  echo -e "\e[36mOK\e[39m"
+else
+  >&2 echo -e "\e[31mFAILED\e[39m"
+  >&2 echo -e "Output: $output"
+  exit 1
+fi

tests/nancy_run_ebs_disk_size_warning.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+output=$(${BASH_SOURCE%/*}/../nancy run \
+  --ebs-volume-size sa \
+  --workload-custom-sql "file://$srcDir/custom.sql" \
+  --db-dump-path "file://$srcDir/test.dump.bz2" \
+  --tmp-path $srcDir/tmp \
+  2>&1)
+
+if [[ $output =~ "WARNING: ebs-volume-size is not required for aws i3 aws instances and local execution." ]]; then
+  echo -e "\e[36mOK\e[39m"
+else
+  >&2 echo -e "\e[31mFAILED\e[39m"
+  >&2 echo -e "Output: $output"
+  exit 1
+fi