Merge pull request #93 from postgres-ai/dmius-region-param

Region param added to nancy run

Author: dmius

nancy_run.sh

@@ -16,7 +16,7 @@ KEEP_ALIVE=0
 VERBOSE_OUTPUT_REDIRECT=" > /dev/null"
 EBS_SIZE_MULTIPLIER=15
 POSTGRES_VERSION_DEFAULT=10
-AWS_BLOCK_DURATION=0 # by default no time limit
+AWS_BLOCK_DURATION=0
 
 #######################################
 # Print an error/warning/notice message to STDERR
@@ -72,7 +72,7 @@ function msg() {
 #             1 if the input is empty,
 #             -1 otherwise.
 #######################################
-function checkPath() {
+function check_path() {
   if [[ -z $1 ]]; then
     return 1
   fi
@@ -113,20 +113,22 @@ function checkPath() {
 #   (text) [5] AWS keypair to use
 #   (text) [6] Path to Private Key file to use for instance
 #              Matching public key with .pub extension should exist
-#   (text) [7] The AWS zone to launch the instance in (one of a,b,c,d,e) 
+#   (text) [7] The AWS region to launch the instance (for example: us-east-1, eu-central-1)
+#   (text) [8] The AWS zone to launch the instance in (one of a,b,c,d,e)
 # Returns:
 #   None
 #######################################
 function create_ec2_docker_machine() {
-  msg "Attempt to create a docker machine in zone $7 with price $3..."
+  msg "Attempt to create a docker machine in region $7 with price $3..."
   docker-machine create --driver=amazonec2 \
     --amazonec2-request-spot-instance \
     --amazonec2-instance-type=$2 \
     --amazonec2-spot-price=$3 \
     --amazonec2-block-duration-minutes=$4 \
     --amazonec2-keypair-name="$5" \
     --amazonec2-ssh-keypath="$6" \
-    --amazonec2-zone $7 \
+    --amazonec2-region="$7" \
+    --amazonec2-zone="$8" \
     $1 2> >(grep -v "failed waiting for successful resource state" >&2) &
 }
 
@@ -163,7 +165,7 @@ function destroy_docker_machine() {
 #   None
 #######################################
 function wait_ec2_docker_machine_ready() {
-  machine=$1
+  local machine=$1
   local check_price=$2
   while true; do
     sleep 5;
@@ -172,7 +174,7 @@ function wait_ec2_docker_machine_ready() {
     ((stop_now==1)) && return 0
     if $check_price ; then
       status=$( \
-        aws ec2 describe-spot-instance-requests \
+        aws --region=$AWS_REGION ec2 describe-spot-instance-requests \
         --filters="Name=launch.instance-type,Values=$AWS_EC2_TYPE" \
         | jq  '.SpotInstanceRequests | sort_by(.CreateTime) | .[] | .Status.Code' \
         | tail -n 1
@@ -205,7 +207,7 @@ function cleanup_and_exit {
     if [ ! -z ${VOLUME_ID+x} ]; then
         msg "Wait and delete volume $VOLUME_ID"
         sleep 60 # wait for the machine to be removed
-        delvolout=$(aws ec2 delete-volume --volume-id $VOLUME_ID)
+        delvolout=$(aws --region=$AWS_REGION ec2 delete-volume --volume-id $VOLUME_ID)
         msg "Volume $VOLUME_ID deleted"
     fi
   else
@@ -499,6 +501,8 @@ while [ $# -gt 0 ]; do
       AWS_SSH_KEY_PATH="$2"; shift 2 ;;
     --aws-ebs-volume-size )
         AWS_EBS_VOLUME_SIZE="$2"; shift 2 ;;
+    --aws-region )
+        AWS_REGION="$2"; shift 2 ;;
     --aws-block-duration )
         AWS_BLOCK_DURATION=$2; shift 2 ;;
 
@@ -598,18 +602,23 @@ if [[ "$RUN_ON" == "aws" ]]; then
     err "ERROR: AWS keypair name and ssh key file must be specified to run on AWS EC2."
     exit 1
   else
-    checkPath AWS_SSH_KEY_PATH
+    check_path AWS_SSH_KEY_PATH
   fi
   if [[ -z ${AWS_EC2_TYPE+x} ]]; then
     err "ERROR: AWS EC2 Instance type not given."
     exit 1
   fi
+  if [[ -z ${AWS_REGION+x} ]]; then
+    err "NOTICE: AWS EC2 region not given. Will used us-east-1."
+    AWS_REGION='us-east-1'
+  fi
   if [[ -z ${AWS_BLOCK_DURATION+x} ]]; then
-    err "NOTICE: Container live time duration is not given."
+    err "NOTICE: Container live time duration is not given. Will used 60 minutes."
+    AWS_BLOCK_DURATION=60
   else
     case $AWS_BLOCK_DURATION in
       0|60|120|240|300|360)
-        dbg "Container live time duration is $AWS_BLOCK_DURATION. "
+        dbg "Container live time duration is $AWS_BLOCK_DURATION."
       ;;
       *)
         err "Container live time duration (--aws-block-duration) has wrong value: $AWS_BLOCK_DURATION. Available values of AWS spot instance duration in minutes is 60, 120, 180, 240, 300, or 360)."
@@ -619,19 +628,23 @@ if [[ "$RUN_ON" == "aws" ]]; then
   fi
 elif [[ "$RUN_ON" == "localhost" ]]; then
   if [[ ! -z ${AWS_KEYPAIR_NAME+x} ]] || [[ ! -z ${AWS_SSH_KEY_PATH+x} ]] ; then
-    err "ERROR: options '--aws-keypair-name' and '--aws-ssh-key-path' must be used with '--run on aws'."
+    err "ERROR: options '--aws-keypair-name' and '--aws-ssh-key-path' must be used with '--run-on aws'."
     exit 1
   fi
   if [[ ! -z ${AWS_EC2_TYPE+x} ]]; then
-    err "ERROR: option '--aws-ec2-type' must be used with '--run on aws'."
+    err "ERROR: option '--aws-ec2-type' must be used with '--run-on aws'."
     exit 1
   fi
   if [[ ! -z ${AWS_EBS_VOLUME_SIZE+x} ]]; then
-    err "ERROR: option '--aws-ebs-volume-size' must be used with '--run on aws'."
+    err "ERROR: option '--aws-ebs-volume-size' must be used with '--run-on aws'."
+    exit 1
+  fi
+  if [[ ! -z ${AWS_REGION+x} ]]; then
+    err "ERROR: option '--aws-region' must be used with '--run-on aws'."
     exit 1
   fi
   if [[ "$AWS_BLOCK_DURATION" != "0" ]]; then
-    err "ERROR: option '--aws-block-duration' must be used with '--run on aws'."
+    err "ERROR: option '--aws-block-duration' must be used with '--run-on aws'."
     exit 1
   fi
 else
@@ -685,7 +698,7 @@ if [[ ! -z ${DB_PREPARED_SNAPSHOT+x} ]]  &&  [[ ! -z ${DB_DUMP+x} ]]; then
 fi
 
 if [[ ! -z ${DB_DUMP+x} ]]; then
-  checkPath DB_DUMP
+  check_path DB_DUMP
   if [[ "$?" -ne "0" ]]; then
     echo "$DB_DUMP" > $TMP_PATH/db_dump_tmp.sql
     DB_DUMP="$TMP_PATH/db_dump_tmp.sql"
@@ -698,7 +711,7 @@ if [[ -z ${PG_CONFIG+x} ]]; then
   err "NOTICE: No PostgreSQL config is provided. Will use default."
   # TODO(NikolayS) use "auto-tuning" – shared_buffers=1/4 RAM, etc
 else
-  checkPath PG_CONFIG
+  check_path PG_CONFIG
   if [[ "$?" -ne "0" ]]; then # TODO(NikolayS) support file:// and s3://
     #err "WARNING: Value given as pg_config: '$PG_CONFIG' not found as file will use as content"
     echo "$PG_CONFIG" > $TMP_PATH/pg_config_tmp.sql
@@ -724,18 +737,18 @@ if [[ -z ${ARTIFACTS_FILENAME+x} ]]; then
   ARTIFACTS_FILENAME=$DOCKER_MACHINE
 fi
 
-if [[ ! -z ${WORKLOAD_REAL+x} ]] && ! checkPath WORKLOAD_REAL; then
+if [[ ! -z ${WORKLOAD_REAL+x} ]] && ! check_path WORKLOAD_REAL; then
   err "ERROR: workload file '$WORKLOAD_REAL' not found."
   exit 1
 fi
 
-if [[ ! -z ${WORKLOAD_BASIS+x} ]] && ! checkPath WORKLOAD_BASIS; then
+if [[ ! -z ${WORKLOAD_BASIS+x} ]] && ! check_path WORKLOAD_BASIS; then
   err "ERROR: workload file '$WORKLOAD_BASIS' not found."
   exit 1
 fi
 
 if [[ ! -z ${WORKLOAD_CUSTOM_SQL+x} ]]; then
-  checkPath WORKLOAD_CUSTOM_SQL
+  check_path WORKLOAD_CUSTOM_SQL
   if [[ "$?" -ne "0" ]]; then
     #err "WARNING: Value given as workload-custom-sql: '$WORKLOAD_CUSTOM_SQL' not found as file will use as content"
     echo "$WORKLOAD_CUSTOM_SQL" > $TMP_PATH/workload_custom_sql_tmp.sql
@@ -744,7 +757,7 @@ if [[ ! -z ${WORKLOAD_CUSTOM_SQL+x} ]]; then
 fi
 
 if [[ ! -z ${COMMANDS_AFTER_CONTAINER_INIT+x} ]]; then
-  checkPath COMMANDS_AFTER_CONTAINER_INIT
+  check_path COMMANDS_AFTER_CONTAINER_INIT
   if [[ "$?" -ne "0" ]]; then
     #err "WARNING: Value given as after_db_init_code: '$COMMANDS_AFTER_CONTAINER_INIT' not found as file will use as content"
     echo "$COMMANDS_AFTER_CONTAINER_INIT" > $TMP_PATH/after_docker_init_code_tmp.sh
@@ -753,15 +766,15 @@ if [[ ! -z ${COMMANDS_AFTER_CONTAINER_INIT+x} ]]; then
 fi
 
 if [[ ! -z ${SQL_AFTER_DB_RESTORE+x} ]]; then
-  checkPath SQL_AFTER_DB_RESTORE
+  check_path SQL_AFTER_DB_RESTORE
   if [[ "$?" -ne "0" ]]; then
     echo "$SQL_AFTER_DB_RESTORE" > $TMP_PATH/after_db_init_code_tmp.sql
     SQL_AFTER_DB_RESTORE="$TMP_PATH/after_db_init_code_tmp.sql"
   fi
 fi
 
 if [[ ! -z ${SQL_BEFORE_DB_RESTORE+x} ]]; then
-  checkPath SQL_BEFORE_DB_RESTORE
+  check_path SQL_BEFORE_DB_RESTORE
   if [[ "$?" -ne "0" ]]; then
     #err "WARNING: Value given as before_db_init_code: '$SQL_BEFORE_DB_RESTORE' not found as file will use as content"
     echo "$SQL_BEFORE_DB_RESTORE" > $TMP_PATH/before_db_init_code_tmp.sql
@@ -770,23 +783,23 @@ if [[ ! -z ${SQL_BEFORE_DB_RESTORE+x} ]]; then
 fi
 
 if [[ ! -z ${DELTA_SQL_DO+x} ]]; then
-  checkPath DELTA_SQL_DO
+  check_path DELTA_SQL_DO
   if [[ "$?" -ne "0" ]]; then
     echo "$DELTA_SQL_DO" > $TMP_PATH/target_ddl_do_tmp.sql
     DELTA_SQL_DO="$TMP_PATH/target_ddl_do_tmp.sql"
   fi
 fi
 
 if [[ ! -z ${DELTA_SQL_UNDO+x} ]]; then
-  checkPath DELTA_SQL_UNDO
+  check_path DELTA_SQL_UNDO
   if [[ "$?" -ne "0" ]]; then
     echo "$DELTA_SQL_UNDO" > $TMP_PATH/target_ddl_undo_tmp.sql
     DELTA_SQL_UNDO="$TMP_PATH/target_ddl_undo_tmp.sql"
   fi
 fi
 
 if [[ ! -z ${DELTA_CONFIG+x} ]]; then
-  checkPath DELTA_CONFIG
+  check_path DELTA_CONFIG
   if [[ "$?" -ne "0" ]]; then
     echo "$DELTA_CONFIG" > $TMP_PATH/target_config_tmp.conf
     DELTA_CONFIG="$TMP_PATH/target_config_tmp.conf"
@@ -795,8 +808,9 @@ fi
 
 if [[ "$RUN_ON" == "aws" ]]; then
   if [[ ! -z ${AWS_EBS_VOLUME_SIZE+x} ]]; then
-    if ! [[ $AWS_EBS_VOLUME_SIZE =~ '^[0-9]+$' ]] ; then
-      err "ERROR: --ebs-volume-size must be integer."
+    re='^[0-9]+$'
+    if ! [[ $AWS_EBS_VOLUME_SIZE =~ $re ]] ; then
+      err "ERROR: --aws-ebs-volume-size must be integer."
       exit 1
     fi
   else
@@ -948,7 +962,7 @@ elif [[ "$RUN_ON" == "aws" ]]; then
   ## Get max price from history and apply multiplier
   # TODO detect region and/or allow to choose via options
   prices=$(
-    aws --region=us-east-1 ec2 \
+    aws --region=$AWS_REGION ec2 \
     describe-spot-price-history --instance-types $AWS_EC2_TYPE --no-paginate \
     --start-time=$(date +%s) --product-descriptions="Linux/UNIX (Amazon VPC)" \
     --query 'SpotPriceHistory[*].{az:AvailabilityZone, price:SpotPrice}'
@@ -959,33 +973,34 @@ elif [[ "$RUN_ON" == "aws" ]]; then
   region="${region/\"/}"
   minprice="${minprice/\"/}"
   minprice="${minprice/\"/}"
-  zone=${region: -1}
-  msg "Min price from history: $minprice in $region (zone: $zone)"
+  AWS_ZONE=${region: -1}
+  AWS_REGION=${region:: -1}
+  msg "Min price from history: $minprice in $region (zone: $AWS_ZONE)"
   multiplier="1.01"
   price=$(echo "$minprice * $multiplier" | bc -l)
   msg "Increased price: $price"
   EC2_PRICE=$price
-  if [ -z $zone ]; then
-    region='a' #default zone
+  if [[ -z $AWS_ZONE ]]; then
+    AWS_ZONE='a' #default zone
   fi
 
-  createDockerMachine $DOCKER_MACHINE $AWS_EC2_TYPE $EC2_PRICE \
-    $AWS_BLOCK_DURATION $AWS_KEYPAIR_NAME $AWS_SSH_KEY_PATH $zone;
-  status=$(waitEC2Ready "docker-machine create" "$DOCKER_MACHINE" 1)
+  create_ec2_docker_machine $DOCKER_MACHINE $AWS_EC2_TYPE $EC2_PRICE \
+    $AWS_BLOCK_DURATION $AWS_KEYPAIR_NAME $AWS_SSH_KEY_PATH $AWS_REGION $AWS_ZONE;
+  status=$(wait_ec2_docker_machine_ready "$DOCKER_MACHINE" true)
   if [[ "$status" == "price-too-low" ]]; then
     msg "Price $price is too low for $AWS_EC2_TYPE instance. Getting the up-to-date value from the error message..."
 
     #destroy_docker_machine $DOCKER_MACHINE
     # "docker-machine rm" doesn't work for "price-too-low" spot requests,
     # so we need to clean up them via aws cli interface directly
-    aws ec2 describe-spot-instance-requests \
+    aws --region=$AWS_REGION ec2 describe-spot-instance-requests \
       --filters 'Name=status-code,Values=price-too-low' \
     | grep SpotInstanceRequestId | awk '{gsub(/[,"]/, "", $2); print $2}' \
-    | xargs aws ec2 cancel-spot-instance-requests \
+    | xargs aws --region=$AWS_REGION ec2 cancel-spot-instance-requests \
       --spot-instance-request-ids || true
 
     corrrectPriceForLastFailedRequest=$( \
-      aws ec2 describe-spot-instance-requests \
+      aws --region=$AWS_REGION ec2 describe-spot-instance-requests \
         --filters="Name=launch.instance-type,Values=$AWS_EC2_TYPE" \
       | jq  '.SpotInstanceRequests[] | select(.Status.Code == "price-too-low") | .Status.Message' \
       | grep -Eo '[0-9]+[.][0-9]+' | tail -n 1 &
@@ -997,10 +1012,9 @@ elif [[ "$RUN_ON" == "aws" ]]; then
       DOCKER_MACHINE="nancy-$CURRENT_TS"
       DOCKER_MACHINE="${DOCKER_MACHINE//_/-}"
       #try start docker machine name with new price
-      msg "Attempt to create a new docker machine: $DOCKER_MACHINE with price: $EC2_PRICE."
-      createDockerMachine $DOCKER_MACHINE $AWS_EC2_TYPE $EC2_PRICE \
-        $AWS_BLOCK_DURATION $AWS_KEYPAIR_NAME $AWS_SSH_KEY_PATH $zone;
-      waitEC2Ready "docker-machine create" "$DOCKER_MACHINE" 0;
+      create_ec2_docker_machine $DOCKER_MACHINE $AWS_EC2_TYPE $EC2_PRICE \
+        $AWS_BLOCK_DURATION $AWS_KEYPAIR_NAME $AWS_SSH_KEY_PATH $AWS_REGION $AWS_ZONE
+      wait_ec2_docker_machine_ready "$DOCKER_MACHINE" false;
     else
       err "ERROR: Cannot determine actual price for the instance $AWS_EC2_TYPE."
       exit 1;
@@ -1026,28 +1040,23 @@ elif [[ "$RUN_ON" == "aws" ]]; then
     docker-machine ssh $DOCKER_MACHINE sudo add-apt-repository -y ppa:sbates
     docker-machine ssh $DOCKER_MACHINE "sudo apt-get update || true"
     docker-machine ssh $DOCKER_MACHINE sudo apt-get install -y nvme-cli
-
-    docker-machine ssh $DOCKER_MACHINE "echo \"# partition table of /dev/nvme0n1\" > /tmp/nvme.part"
-    docker-machine ssh $DOCKER_MACHINE "echo \"unit: sectors \" >> /tmp/nvme.part"
-    docker-machine ssh $DOCKER_MACHINE "echo \"/dev/nvme0n1p1 : start=2048, size=1855466702, Id=83 \" >> /tmp/nvme.part"
-    docker-machine ssh $DOCKER_MACHINE "echo \"/dev/nvme0n1p2 : start=0, size=0, Id=0 \" >> /tmp/nvme.part"
-    docker-machine ssh $DOCKER_MACHINE "echo \"/dev/nvme0n1p3 : start=0, size=0, Id=0 \" >> /tmp/nvme.part"
-    docker-machine ssh $DOCKER_MACHINE "echo \"/dev/nvme0n1p4 : start=0, size=0, Id=0 \" >> /tmp/nvme.part"
-
-    docker-machine ssh $DOCKER_MACHINE "sudo sfdisk /dev/nvme0n1 < /tmp/nvme.part"
-    docker-machine ssh $DOCKER_MACHINE "sudo mkfs -t ext4 /dev/nvme0n1p1"
+    docker-machine ssh $DOCKER_MACHINE "sudo parted -a optimal -s /dev/nvme0n1 mklabel gpt"
+    docker-machine ssh $DOCKER_MACHINE "sudo parted -a optimal -s /dev/nvme0n1 mkpart primary 0% 100%"
+    docker-machine ssh $DOCKER_MACHINE "sudo mkfs.ext4 /dev/nvme0n1p1"
     docker-machine ssh $DOCKER_MACHINE "sudo mount /dev/nvme0n1p1 /home/storage"
+    docker-machine ssh $DOCKER_MACHINE "sudo df -h /dev/nvme0n1p1"
   else
     msg "Use EBS volume"
     # Create new volume and attach them for non i3 instances if needed
     if [ ! -z ${AWS_EBS_VOLUME_SIZE+x} ]; then
       msg "Create and attach a new EBS volume (size: $AWS_EBS_VOLUME_SIZE GB)"
-      VOLUME_ID=$(aws ec2 create-volume --size $AWS_EBS_VOLUME_SIZE --region us-east-1 --availability-zone us-east-1a --volume-type gp2 | jq -r .VolumeId)
+      VOLUME_ID=$(aws --region=$AWS_REGION ec2 create-volume --size $AWS_EBS_VOLUME_SIZE --availability-zone $AWS_REGION$AWS_ZONE --volume-type gp2 | jq -r .VolumeId)
       INSTANCE_ID=$(docker-machine ssh $DOCKER_MACHINE curl -s http://169.254.169.254/latest/meta-data/instance-id)
       sleep 10 # wait to volume will ready
-      attachResult=$(aws ec2 attach-volume --device /dev/xvdf --volume-id $VOLUME_ID --instance-id $INSTANCE_ID --region us-east-1)
+      attachResult=$(aws --region=$AWS_REGION ec2 attach-volume --device /dev/xvdf --volume-id $VOLUME_ID --instance-id $INSTANCE_ID)
       docker-machine ssh $DOCKER_MACHINE sudo mkfs.ext4 /dev/xvdf
       docker-machine ssh $DOCKER_MACHINE sudo mount /dev/xvdf /home/storage
+      docker-machine ssh $DOCKER_MACHINE "sudo df -h /dev/xvdf"
     fi
   fi
 

tests/nancy_run_invalid_aws_option.sh

@@ -9,7 +9,7 @@ output=$(
     2>&1
 )
 
-if [[ $output =~ "must be used with '--run on aws'" ]]; then
+if [[ $output =~ "must be used with '--run-on aws'" ]]; then
   echo -e "\e[36mOK\e[39m"
 else
   >&2 echo -e "\e[31mFAILED\e[39m"

tests/nancy_run_options_both_dump_snapshot.sh

@@ -5,6 +5,7 @@ read -r -d '' params <<PARAMS
   --aws-ssh-key-path "/home/someuser/.ssh/awskey.pem" \
   --aws-ec2-type "r4.large" \
   --s3cfg-path "/home/someuser/.s3cfg" \
+  --aws-region "us-east-1" \
   --workload-real "s3://somebucket/db.sql.30min.pgreplay" \
   --tmp-path tmp \
   --db-dump "s3://somebucket/dump.sql.bz2" \

tests/nancy_run_options_ddl_do+_undo-.sh

@@ -4,6 +4,7 @@ read -r -d '' params <<PARAMS
   --run-on aws --aws-keypair-name awskey --pg-version 9.6 \
   --aws-ssh-key-path "/home/someuser/.ssh/awskey.pem" \
   --aws-ec2-type "r4.large" \
+  --aws-region "us-east-1" \
   --s3cfg-path "/home/someuser/.s3cfg" \
   --workload-real "s3://somebucket/db.sql.30min.pgreplay" \
   --tmp-path tmp \

tests/nancy_run_options_ddl_do-_undo+.sh

@@ -4,6 +4,7 @@ read -r -d '' params <<PARAMS
   --run-on aws --aws-keypair-name awskey --pg-version 9.6 \
   --aws-ssh-key-path "/home/someuser/.ssh/awskey.pem" \
   --aws-ec2-type "r4.large" \
+  --aws-region "us-east-1" \
   --s3cfg-path "/home/someuser/.s3cfg" \
   --workload-real "s3://somebucket/db.sql.30min.pgreplay" \
   --tmp-path tmp \

tests/nancy_run_options_multi_workloads.sh

@@ -4,6 +4,7 @@ read -r -d '' params <<PARAMS
   --run-on aws --aws-keypair-name awskey --pg-version 9.6 \
   --aws-ssh-key-path "/home/someuser/.ssh/awskey.pem" \
   --aws-ec2-type "r4.large" \
+  --aws-region "us-east-1" \
   --s3cfg-path "/home/someuser/.s3cfg" \
   --workload-real "s3://somebucket/db.sql.30min.pgreplay" \
   --workload-custom-sql "select\tnow();" \

tests/nancy_run_options_no_dump_snapshot.sh

@@ -4,6 +4,7 @@ read -r -d '' params <<PARAMS
   --run-on aws --aws-keypair-name awskey --pg-version 9.6 \
   --aws-ssh-key-path "/home/someuser/.ssh/awskey.pem" \
   --aws-ec2-type "r4.large" \
+  --aws-region "us-east-1" \
   --s3cfg-path "/home/someuser/.s3cfg" \
   --workload-real "s3://somebucket/db.sql.30min.pgreplay" \
   --tmp-path tmp