Skip to content
This repository was archived by the owner on Aug 16, 2021. It is now read-only.

Commit 3c27d20

Browse files
dmiusdmius
authored
Merge branch 'master' into dmius-duration-param
2 parents 93b7657 + e1fd19b commit 3c27d20

File tree

1 file changed

+176
-13
lines changed

1 file changed

+176
-13
lines changed

‎nancy_run.sh

Lines changed: 176 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
# Globals (some of them can be modified below)
1010
KB=1024
11-
DEBUG=0
11+
DEBUG=false
1212
CURRENT_TS=$(date +%Y%m%d_%H%M%S%N_%Z)
1313
DOCKER_MACHINE="nancy-$CURRENT_TS"
1414
DOCKER_MACHINE="${DOCKER_MACHINE//_/-}"
@@ -41,7 +41,9 @@ function err() {
4141
# None
4242
#######################################
4343
function dbg() {
44-
[[ "$DEBUG" -eq "1" ]] && msg "DEBUG: $@"
44+
if $DEBUG ; then
45+
msg "DEBUG: $@"
46+
fi
4547
}
4648

4749
#######################################
@@ -57,8 +59,166 @@ function msg() {
5759
echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $@"
5860
}
5961

60-
# Process CLI parameters
61-
while true; do
62+
#######################################
63+
# Check path to file/directory.
64+
# Globals:
65+
# None
66+
# Arguments:
67+
# (text) name of the variable holding the
68+
# file path (starts with 'file://' or 's3://') or any string
69+
# Returns:
70+
# (integer) for input starting with 's3://' always returns 0
71+
# for 'file://': 0 if file exists locally, error if it doesn't
72+
# 1 if the input is empty,
73+
# -1 otherwise.
74+
#######################################
75+
function checkPath() {
76+
if [[ -z $1 ]]; then
77+
return 1
78+
fi
79+
eval path=\$$1
80+
if [[ $path =~ "s3://" ]]; then
81+
dbg "$1 looks like a S3 file path. Warning: Its presence will not be checked!"
82+
return 0 # we do not actually check S3 paths at the moment
83+
elif [[ $path =~ "file://" ]]; then
84+
dbg "$1 looks like a local file path."
85+
path=${path/file:\/\//}
86+
if [[ -f $path ]]; then
87+
dbg "$path found."
88+
eval "$1=\"$path\"" # update original variable
89+
return 0 # file found
90+
else
91+
err "File '$path' is not found locally."
92+
exit 1
93+
fi
94+
else
95+
dbg "Value of $1 is not a file path. Use its value as a content."
96+
return -1 #
97+
fi
98+
}
99+
100+
### Docker tools ###
101+
102+
#######################################
103+
# Create Docker machine using an AWS EC2 spot instance
104+
# See also: https://docs.docker.com/machine/reference/create/
105+
# Globals:
106+
# None
107+
# Arguments:
108+
# (text) [1] Machine name
109+
# (text) [2] EC2 Instance type
110+
# (text) [3] Spot instance bid price (in dollars)
111+
# (int) [4] AWS spot instance duration in minutes (60, 120, 180, 240, 300,
112+
# or 360)
113+
# (text) [5] AWS keypair to use
114+
# (text) [6] Path to Private Key file to use for instance
115+
# Matching public key with .pub extension should exist
116+
# (text) [7] The AWS zone to launch the instance in (one of a,b,c,d,e)
117+
# Returns:
118+
# None
119+
#######################################
120+
function create_ec2_docker_machine() {
121+
msg "Attempt to create a docker machine in zone $7 with price $3..."
122+
docker-machine create --driver=amazonec2 \
123+
--amazonec2-request-spot-instance \
124+
--amazonec2-instance-type=$2 \
125+
--amazonec2-spot-price=$3 \
126+
--amazonec2-block-duration-minutes=$4 \
127+
--amazonec2-keypair-name="$5" \
128+
--amazonec2-ssh-keypath="$6" \
129+
--amazonec2-zone $7 \
130+
$1 2> >(grep -v "failed waiting for successful resource state" >&2) &
131+
}
132+
133+
#######################################
134+
# Order to destroy Docker machine (any platform)
135+
# See also: https://docs.docker.com/machine/reference/rm/
136+
# Globals:
137+
# None
138+
# Arguments:
139+
# (text) Machine name
140+
# Returns:
141+
# None
142+
#######################################
143+
function destroy_docker_machine() {
144+
# If spot request wasn't fulfilled, there is no associated instance,
145+
# so "docker-machine rm" will show an error, which is safe to ignore.
146+
# We better filter it out to avoid any confusions.
147+
# What is used here is called "process substitution",
148+
# see https://www.gnu.org/software/bash/manual/bash.html#Process-Substitution
149+
# The same trick is used in create_ec2_docker_machine() to filter out errors
150+
# when we have "price-too-low" attempts, such errors come in few minutes
151+
# after an attempt and are generally unexpected by user.
152+
cmdout=$(docker-machine rm --force $1 2> >(grep -v "unknown instance" >&2) )
153+
msg "Termination requested for machine, current status: $cmdout"
154+
}
155+
156+
#######################################
157+
# Wait until EC2 instance with Docker maching is up and running
158+
# Globals:
159+
# None
160+
# Arguments:
161+
# (text) Machine name
162+
# Returns:
163+
# None
164+
#######################################
165+
function wait_ec2_docker_machine_ready() {
166+
machine=$1
167+
local check_price=$2
168+
while true; do
169+
sleep 5;
170+
local stop_now=1
171+
ps ax | grep "docker-machine create" | grep "$machine" >/dev/null && stop_now=0
172+
((stop_now==1)) && return 0
173+
if $check_price ; then
174+
status=$( \
175+
aws ec2 describe-spot-instance-requests \
176+
--filters="Name=launch.instance-type,Values=$AWS_EC2_TYPE" \
177+
| jq '.SpotInstanceRequests | sort_by(.CreateTime) | .[] | .Status.Code' \
178+
| tail -n 1
179+
)
180+
if [[ "$status" == "\"price-too-low\"" ]]; then
181+
echo "price-too-low"; # this value is result of function (not message for user), to be checked later
182+
return 0
183+
fi
184+
fi
185+
done
186+
}
187+
188+
function cleanup_and_exit {
189+
if [ "$KEEP_ALIVE" -gt "0" ]; then
190+
msg "Debug timeout is $KEEP_ALIVE seconds – started."
191+
msg " To connect to the docker machine use:"
192+
msg " docker \`docker-machine config $DOCKER_MACHINE\` exec -it pg_nancy_${CURRENT_TS} bash"
193+
sleep $KEEP_ALIVE
194+
fi
195+
msg "Remove temp files..." # if exists
196+
if [[ ! -z "${dockerConfig+x}" ]]; then
197+
docker $dockerConfig exec -i ${containerHash} bash -c "sudo rm -rf $MACHINE_HOME"
198+
fi
199+
rm -rf "$TMP_PATH"
200+
if [[ "$RUN_ON" == "localhost" ]]; then
201+
msg "Remove docker container"
202+
docker container rm -f $containerHash
203+
elif [[ "$RUN_ON" == "aws" ]]; then
204+
destroy_docker_machine $DOCKER_MACHINE
205+
if [ ! -z ${VOLUME_ID+x} ]; then
206+
msg "Wait and delete volume $VOLUME_ID"
207+
sleep 60 # wait for the machine to be removed
208+
delvolout=$(aws ec2 delete-volume --volume-id $VOLUME_ID)
209+
msg "Volume $VOLUME_ID deleted"
210+
fi
211+
else
212+
err "ASSERT: must not reach this point"
213+
exit 1
214+
fi
215+
}
216+
217+
#######################################
218+
# # # # # MAIN # # # # #
219+
#######################################
220+
# Process CLI options
221+
while [ $# -gt 0 ]; do
62222
case "$1" in
63223
help )
64224
echo -e "\033[1mCOMMAND\033[22m
@@ -278,7 +438,7 @@ while true; do
278438
" | less -RFX
279439
exit ;;
280440
-d | --debug )
281-
DEBUG=1;
441+
DEBUG=true;
282442
VERBOSE_OUTPUT_REDIRECT='';
283443
shift ;;
284444
--keep-alive )
@@ -362,7 +522,7 @@ done
362522

363523
RUN_ON=${RUN_ON:-localhost}
364524

365-
if [[ $DEBUG -eq 1 ]]; then
525+
if $DEBUG ; then
366526
echo "DEBUG: ${DEBUG}"
367527
echo "KEEP_ALIVE: ${KEEP_ALIVE}"
368528
echo "RUN_ON: ${RUN_ON}"
@@ -680,8 +840,11 @@ if [[ "$RUN_ON" == "aws" ]] && [[ ! ${AWS_EC2_TYPE:0:2} == "i3" ]] \
680840
fi
681841
fi
682842

683-
set -ueo pipefail
684-
[[ $DEBUG -eq 1 ]] && set -uox pipefail # to debug
843+
if $DEBUG ; then
844+
set -xueo pipefail
845+
else
846+
set -ueo pipefail
847+
fi
685848
shopt -s expand_aliases
686849

687850
## Docker tools
@@ -792,7 +955,7 @@ elif [[ "$RUN_ON" == "aws" ]]; then
792955
--query 'SpotPriceHistory[*].{az:AvailabilityZone, price:SpotPrice}'
793956
)
794957
minprice=$(echo $prices | jq 'min_by(.price) | .price')
795-
region=$(echo $prices | jq 'min_by(.price) | .az')
958+
region=$(echo $prices | jq 'min_by(.price) | .az') #TODO(NikolayS) double-check zones&regions
796959
region="${region/\"/}"
797960
region="${region/\"/}"
798961
minprice="${minprice/\"/}"
@@ -813,14 +976,14 @@ elif [[ "$RUN_ON" == "aws" ]]; then
813976
if [[ "$status" == "price-too-low" ]]; then
814977
msg "Price $price is too low for $AWS_EC2_TYPE instance. Getting the up-to-date value from the error message..."
815978

816-
#destroyDockerMachine $DOCKER_MACHINE
979+
#destroy_docker_machine $DOCKER_MACHINE
817980
# "docker-machine rm" doesn't work for "price-too-low" spot requests,
818981
# so we need to clean up them via aws cli interface directly
819982
aws ec2 describe-spot-instance-requests \
820983
--filters 'Name=status-code,Values=price-too-low' \
821984
| grep SpotInstanceRequestId | awk '{gsub(/[,"]/, "", $2); print $2}' \
822-
| xargs --no-run-if-empty aws ec2 cancel-spot-instance-requests \
823-
--spot-instance-request-ids
985+
| xargs aws ec2 cancel-spot-instance-requests \
986+
--spot-instance-request-ids || true
824987

825988
corrrectPriceForLastFailedRequest=$( \
826989
aws ec2 describe-spot-instance-requests \
@@ -840,7 +1003,7 @@ elif [[ "$RUN_ON" == "aws" ]]; then
8401003
$AWS_BLOCK_DURATION $AWS_KEYPAIR_NAME $AWS_SSH_KEY_PATH $zone;
8411004
waitEC2Ready "docker-machine create" "$DOCKER_MACHINE" 0;
8421005
else
843-
err "$(date "+%Y-%m-%d %H:%M:%S") ERROR: Cannot determine actual price for the instance $AWS_EC2_TYPE."
1006+
err "ERROR: Cannot determine actual price for the instance $AWS_EC2_TYPE."
8441007
exit 1;
8451008
fi
8461009
fi

0 commit comments

Comments
 (0)