Skip to content
This repository was archived by the owner on Aug 16, 2021. It is now read-only.

Commit 348907f

Browse files
dmiusdmius
authored
Merge pull request #19 from startupturbo/dmius-file-params
SQL and conf params as filename
2 parents aec583b + 553249c commit 348907f

File tree

1 file changed

+122
-43
lines changed

1 file changed

+122
-43
lines changed

‎nancy_run.sh

Lines changed: 122 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -15,28 +15,37 @@ while true; do
1515
--pg-version )
1616
PG_VERSION="$2"; shift 2 ;;
1717
--pg-config )
18-
PG_CONFIG="$2"; shift 2 ;;
18+
#Still unsupported
19+
PG_CONFIG="$2"; shift 2;;
1920
--db-prepared-snapshot )
21+
#Still unsupported
2022
DB_PREPARED_SNAPSHOT="$2"; shift 2 ;;
2123
--db-dump-path )
2224
DB_DUMP_PATH="$2"; shift 2 ;;
2325
--after-db-init-code )
26+
#s3 url|filename +
2427
AFTER_DB_INIT_CODE="$2"; shift 2 ;;
2528
--workload-full-path )
29+
#s3 url
2630
WORKLOAD_FULL_PATH="$2"; shift 2 ;;
2731
--workload-basis-path )
32+
#Still unsuported
2833
WORKLOAD_BASIS_PATH="$2"; shift 2 ;;
2934
--workload-custom-sql )
35+
#s3 url|filename +
3036
WORKLOAD_CUSTOM_SQL="$2"; shift 2 ;;
3137
--workload-replay-speed )
3238
WORKLOAD_REPLAY_SPEED="$2"; shift 2 ;;
3339
--target-ddl-do )
40+
#s3 url|filename +
3441
TARGET_DDL_DO="$2"; shift 2 ;;
3542
--target-ddl-undo )
43+
#s3 url|filename +
3644
TARGET_DDL_UNDO="$2"; shift 2 ;;
3745
--clean-run-only )
3846
CLEAN_RUN_ONLY=1; shift 1 ;;
3947
--target-config )
48+
#s3 url|filename +
4049
TARGET_CONFIG="$2"; shift 2 ;;
4150
--artifacts-destination )
4251
ARTIFACTS_DESTINATION="$2"; shift 2 ;;
@@ -83,6 +92,46 @@ then
8392
echo "after-db-init-code: $AFTER_DB_INIT_CODE"
8493
fi
8594

95+
function checkPath() {
96+
if [ ! -v $1 ]
97+
then
98+
return 1
99+
fi
100+
eval path=\$$1
101+
if [[ $path =~ "s3://" ]]
102+
then
103+
return 0; ## do not check
104+
fi
105+
if [[ $path =~ "file:///" ]]
106+
then
107+
path=${path/file:\/\//}
108+
if [ -f $path ]
109+
then
110+
eval "$1=\"$path\"" # update original variable
111+
return 0 # file found
112+
else
113+
return 2 # file not found
114+
fi
115+
fi
116+
if [[ $path =~ "file://" ]]
117+
then
118+
curdir=$(pwd)
119+
path=$curdir/${path/file:\/\//}
120+
if [ -f $path ]
121+
then
122+
eval "$1=\"$path\"" # update original variable
123+
return 0 # file found
124+
else
125+
return 3 # file not found
126+
fi
127+
fi
128+
if [ -f $path ]
129+
then
130+
return 0;
131+
fi
132+
return -1 # incorrect path
133+
}
134+
86135
## Check params
87136
function checkParams() {
88137
if ([ ! -v AWS_KEY_PAIR ] || [ ! -v AWS_KEY_PATH ])
@@ -171,6 +220,22 @@ function checkParams() {
171220
>&2 echo "WARNING: Artifacts destination not given. Will use $DOCKER_MACHINE"
172221
ARTIFACTS_FILENAME=$DOCKER_MACHINE
173222
fi
223+
224+
[ -v WORKLOAD_FULL_PATH ] && ! checkPath WORKLOAD_FULL_PATH && >&2 echo "WARNING: file $AFTER_DB_INIT_CODE not found"
225+
226+
[ -v WORKLOAD_BASIS_PATH ] && ! checkPath WORKLOAD_BASIS_PATH && >&2 echo "WARNING: file $WORKLOAD_BASIS_PATH not found"
227+
228+
[ -v WORKLOAD_CUSTOM_SQL ] && ! checkPath WORKLOAD_CUSTOM_SQL && >&2 echo "WARNING: file $WORKLOAD_CUSTOM_SQL not found"
229+
230+
[ -v DB_DUMP_PATH ] && ! checkPath DB_DUMP_PATH && >&2 echo "WARNING: file $DB_DUMP_PATH not found"
231+
232+
[ -v AFTER_DB_INIT_CODE ] && ! checkPath AFTER_DB_INIT_CODE && >&2 echo "WARNING: file $AFTER_DB_INIT_CODE not found"
233+
234+
[ -v TARGET_DDL_DO ] && ! checkPath TARGET_DDL_DO && >&2 echo "WARNING: file $TARGET_DDL_DO not found"
235+
236+
[ -v TARGET_DDL_UNDO ] && ! checkPath TARGET_DDL_UNDO && >&2 echo "WARNING: file $TARGET_DDL_UNDO not found"
237+
238+
[ -v TARGET_CONFIG ] && ! checkPath TARGET_CONFIG && >&2 echo "WARNING: file $TARGET_CONFIG not found"
174239
}
175240

176241
checkParams;
@@ -189,7 +254,8 @@ function waitDockerReady() {
189254
((STOP==1)) && return 0
190255
if [ $checkPrice -eq 1 ]
191256
then
192-
status=$(aws ec2 describe-spot-instance-requests --filters="Name=launch.instance-type,Values=$AWS_EC2_TYPE" | jq '.SpotInstanceRequests[] | .Status.Code' | tail -n 1 )
257+
#status=$(aws ec2 describe-spot-instance-requests --filters="Name=launch.instance-type,Values=$AWS_EC2_TYPE" | jq '.SpotInstanceRequests[] | .Status.Code' | tail -n 1 )
258+
status=$(aws ec2 describe-spot-instance-requests --filters="Name=launch.instance-type,Values=$AWS_EC2_TYPE" | jq '.SpotInstanceRequests | sort_by(.CreateTime) | .[] | .Status.Code' | tail -n 1)
193259
if [ "$status" == "\"price-too-low\"" ]
194260
then
195261
echo "price-too-low"; # this value is result of function (not message for user), will check later
@@ -264,50 +330,50 @@ function cleanup {
264330
}
265331
trap cleanup EXIT
266332

267-
## Prepare conf, queries and dump files
268-
if ([ "$TARGET_CONFIG" != "" ] && [ "$TARGET_CONFIG" != "null" ])
333+
shopt -s expand_aliases
334+
alias sshdo='docker $dockerConfig exec -i pg_nancy '
335+
336+
## Copy data to docker machine
337+
docker-machine scp $S3_CFG_PATH $DOCKER_MACHINE:/home/ubuntu
338+
sshdo cp /machine_home/.s3cfg /root/.s3cfg
339+
sshdo s3cmd sync $DB_DUMP_PATH ./
340+
if ([ -v TARGET_CONFIG ] && [ "$TARGET_CONFIG" != "" ])
269341
then
270-
echo "TARGET_CONFIG is not empty: $TARGET_CONFIG"
271-
echo "$TARGET_CONFIG" > $TMP_PATH/conf_$DOCKER_MACHINE.tmp
342+
if [[ $TARGET_CONFIG =~ "s3://" ]]; then
343+
sshdo s3cmd sync $TARGET_CONFIG /machine_home/
344+
else
345+
docker-machine scp $TARGET_CONFIG $DOCKER_MACHINE:/home/ubuntu
346+
fi
272347
fi
273348

274349
if ([ -v TARGET_DDL_DO ] && [ "$TARGET_DDL_DO" != "" ])
275350
then
276-
echo "TARGET_DDL_DO is not empty: $TARGET_DDL_DO"
277-
echo "$TARGET_DDL_DO" > $TMP_PATH/ddl_do_$DOCKER_MACHINE.sql
351+
if [[ $TARGET_DDL_DO =~ "s3://" ]]; then
352+
sshdo s3cmd sync $TARGET_DDL_DO /machine_home/
353+
else
354+
docker-machine scp $TARGET_DDL_DO $DOCKER_MACHINE:/home/ubuntu
355+
fi
278356
fi
279357

280358
if ([ -v TARGET_DDL_UNDO ] && [ "$TARGET_DDL_UNDO" != "" ])
281359
then
282-
echo "TARGET_DDL_UNDO is not empty: $TARGET_DDL_UNDO"
283-
echo "$TARGET_DDL_UNDO" > $TMP_PATH/ddl_undo_$DOCKER_MACHINE.sql
360+
if [[ $TARGET_DDL_UNDO =~ "s3://" ]]; then
361+
sshdo s3cmd sync $TARGET_DDL_UNDO /machine_home/
362+
else
363+
docker-machine scp $TARGET_DDL_UNDO $DOCKER_MACHINE:/home/ubuntu
364+
fi
284365
fi
285366

286367
if ([ -v WORKLOAD_CUSTOM_SQL ] && [ "$WORKLOAD_CUSTOM_SQL" != "" ])
287368
then
288-
echo "WORKLOAD_CUSTOM_SQL is not empty: $WORKLOAD_CUSTOM_SQL"
289-
echo "$WORKLOAD_CUSTOM_SQL" > $TMP_PATH/queries_custom_$DOCKER_MACHINE.sql
369+
WORKLOAD_CUSTOM_FILENAME=$(basename $WORKLOAD_CUSTOM_SQL)
370+
if [[ $WORKLOAD_CUSTOM_SQL =~ "s3://" ]]; then
371+
sshdo s3cmd sync $WORKLOAD_CUSTOM_SQL /machine_home/
372+
else
373+
docker-machine scp $WORKLOAD_CUSTOM_SQL $DOCKER_MACHINE:/home/ubuntu
374+
fi
290375
fi
291376

292-
shopt -s expand_aliases
293-
alias sshdo='docker $dockerConfig exec -i pg_nancy '
294-
295-
## Copy data to docker machine
296-
docker-machine scp $S3_CFG_PATH $DOCKER_MACHINE:/home/ubuntu
297-
sshdo cp /machine_home/.s3cfg /root/.s3cfg
298-
sshdo s3cmd sync $DB_DUMP_PATH ./
299-
if [ -f "$TMP_PATH/conf_$DOCKER_MACHINE.tmp" ]; then
300-
docker-machine scp $TMP_PATH/conf_$DOCKER_MACHINE.tmp $DOCKER_MACHINE:/home/ubuntu
301-
fi
302-
if [ -f "$TMP_PATH/ddl_do_$DOCKER_MACHINE.sql" ]; then
303-
docker-machine scp $TMP_PATH/ddl_do_$DOCKER_MACHINE.sql $DOCKER_MACHINE:/home/ubuntu
304-
fi
305-
if [ -f "$TMP_PATH/ddl_undo_$DOCKER_MACHINE.sql" ]; then
306-
docker-machine scp $TMP_PATH/ddl_undo_$DOCKER_MACHINE.sql $DOCKER_MACHINE:/home/ubuntu
307-
fi
308-
if [ -f "$TMP_PATH/queries_custom_$DOCKER_MACHINE.sql" ]; then
309-
docker-machine scp $TMP_PATH/queries_custom_$DOCKER_MACHINE.sql $DOCKER_MACHINE:/home/ubuntu
310-
fi
311377
if ([ "$WORKLOAD_FULL_PATH" != "" ] && [ "$WORKLOAD_FULL_PATH" != "null" ])
312378
then
313379
sshdo s3cmd sync $WORKLOAD_FULL_PATH ./
@@ -321,17 +387,25 @@ sshdo bash -c "bzcat ./$DB_DUMP_FILENAME | psql --set ON_ERROR_STOP=on -U postgr
321387
# After init database sql code apply
322388
if ([ -v AFTER_DB_INIT_CODE ] && [ "$AFTER_DB_INIT_CODE" != "" ])
323389
then
324-
sshdo psql -U postgres test -c "$AFTER_DB_INIT_CODE"
390+
AFTER_DB_INIT_CODE_FILENAME=$(basename $AFTER_DB_INIT_CODE)
391+
if [[ $AFTER_DB_INIT_CODE =~ "s3://" ]]; then
392+
sshdo s3cmd sync $AFTER_DB_INIT_CODE /machine_home/
393+
else
394+
docker-machine scp $AFTER_DB_INIT_CODE $DOCKER_MACHINE:/home/ubuntu
395+
fi
396+
sshdo bash -c "psql -U postgres test -E -f /machine_home/$AFTER_DB_INIT_CODE_FILENAME"
325397
fi
326398
# Apply DDL code
327-
echo "Apply DDL SQL code from /machine_home/ddl_do_$DOCKER_MACHINE.sql"
328-
if [ -f "$TMP_PATH/ddl_do_$DOCKER_MACHINE.sql" ]; then
329-
sshdo bash -c "psql -U postgres test -E -f /machine_home/ddl_do_$DOCKER_MACHINE.sql"
399+
echo "Apply DDL SQL code"
400+
if ([ -v TARGET_DDL_DO ] && [ "$TARGET_DDL_DO" != "" ]); then
401+
TARGET_DDL_DO_FILENAME=$(basename $TARGET_DDL_DO)
402+
sshdo bash -c "psql -U postgres test -E -f /machine_home/$TARGET_DDL_DO_FILENAME"
330403
fi
331404
# Apply postgres configuration
332405
echo "Apply postgres conf from /machine_home/conf_$DOCKER_MACHINE.tmp"
333-
if [ -f "$TMP_PATH/conf_$DOCKER_MACHINE.tmp" ]; then
334-
sshdo bash -c "cat /machine_home/conf_$DOCKER_MACHINE.tmp >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
406+
if ([ -v TARGET_CONFIG ] && [ "$TARGET_CONFIG" != "" ]); then
407+
TARGET_CONFIG_FILENAME=$(basename $TARGET_CONFIG)
408+
sshdo bash -c "cat /machine_home/$TARGET_CONFIG_FILENAME >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
335409
sshdo bash -c "sudo /etc/init.d/postgresql restart"
336410
fi
337411
# Clear statistics and log
@@ -340,16 +414,17 @@ sshdo vacuumdb -U postgres test -j $(cat /proc/cpuinfo | grep processor | wc -l)
340414
sshdo bash -c "echo '' > /var/log/postgresql/postgresql-$PG_VERSION-main.log"
341415
# Execute workload
342416
echo "Execute workload..."
343-
if ([ "$WORKLOAD_FULL_PATH" != "" ] && [ "$WORKLOAD_FULL_PATH" != "null" ])
417+
if ([ -v WORKLOAD_FULL_PATH ] && [ "$WORKLOAD_FULL_PATH" != "" ] && [ "$WORKLOAD_FULL_PATH" != "null" ])
344418
then
345419
echo "Execute pgreplay queries..."
346420
sshdo psql -U postgres test -c 'create role testuser superuser login;'
347421
WORKLOAD_FILE_NAME=$(basename $WORKLOAD_FULL_PATH)
348422
sshdo bash -c "pgreplay -r -j ./$WORKLOAD_FILE_NAME"
349423
else
350-
if [ -f "$TMP_PATH/queries_custom_$DOCKER_MACHINE.sql" ]; then
424+
if ([ -v WORKLOAD_CUSTOM_SQL ] && [ "$WORKLOAD_CUSTOM_SQL" != "" ]); then
425+
WORKLOAD_CUSTOM_FILENAME=$(basename $WORKLOAD_CUSTOM_SQL)
351426
echo "Execute custom sql queries..."
352-
sshdo bash -c "psql -U postgres test -E -f /machine_home/queries_custom_$DOCKER_MACHINE.sql"
427+
sshdo bash -c "psql -U postgres test -E -f /machine_home/$WORKLOAD_CUSTOM_FILENAME"
353428
fi
354429
fi
355430

@@ -363,14 +438,18 @@ if [[ $ARTIFACTS_DESTINATION =~ "s3://" ]]; then
363438
sshdo s3cmd put /$ARTIFACTS_FILENAME.json $ARTIFACTS_DESTINATION/
364439
else
365440
sshdo cp /$ARTIFACTS_FILENAME.json /machine_home/
366-
docker-machine scp $DOCKER_MACHINE:/home/ubuntu/$ARTIFACTS_FILENAME.json $ARTIFACTS_DESTINATION/
441+
docker-machine scp $DOCKER_MACHINE:/home/ubuntu/$ARTIFACTS_FILENAME.json $ARTIFACTS_DESTINATION/
367442
fi
368443

369444
echo "Apply DDL undo SQL code from /machine_home/ddl_undo_$DOCKER_MACHINE.sql"
370-
if [ -f "$TMP_PATH/ddl_undo_$DOCKER_MACHINE.sql" ]; then
371-
sshdo bash -c "psql -U postgres test -E -f /machine_home/ddl_undo_$DOCKER_MACHINE.sql"
445+
if ([ -v TARGET_DDL_UNDO ] && [ "$TARGET_DDL_UNDO" != "" ]); then
446+
TARGET_DDL_UNDO_FILENAME=$(basename $TARGET_DDL_UNDO)
447+
sshdo bash -c "psql -U postgres test -E -f /machine_home/$TARGET_DDL_UNDO_FILENAME"
372448
fi
373449

450+
echo "Run done!"
451+
echo "Result log: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME.json"
452+
374453
sleep $DEBUG_TIMEOUT
375454

376455
echo Bye!

0 commit comments

Comments
 (0)