calc CPU count on docker, options renamed, commands-after-docker-init added

Author: dmius

nancy_run.sh

@@ -95,7 +95,7 @@ while true; do
 
   Specify the path to database dump (created by pg_dump) to be used as an input.
 
-  \033[1m--after-db-init-code\033[22m (string)
+  \033[1m--sql-after-db-restore\033[22m (string)
 
   Specify additional commands to be executed after database is initiated (dump
   loaded or snapshot attached).
@@ -199,10 +199,12 @@ while true; do
       DB_PREPARED_SNAPSHOT="$2"; shift 2 ;;
     --db-dump )
       DB_DUMP_PATH="$2"; shift 2 ;;
-    --after-db-init-code )
+    --commands-after-docker-init )
+      AFTER_DOCKER_INIT_CODE="$2"; shift 2 ;;
+    --sql-after-db-restore )
       #s3 url|filename|content
       AFTER_DB_INIT_CODE="$2"; shift 2 ;;
-    --before-db-init-code )
+    --sql-before-db-restore )
       #s3 url|filename|content
       BEFORE_DB_INIT_CODE="$2"; shift 2 ;;
     --workload-real )
@@ -284,6 +286,7 @@ if [ $DEBUG -eq 1 ]; then
   echo "s3-cfg-path: $S3_CFG_PATH"
   echo "tmp-path: $TMP_PATH"
   echo "after-db-init-code: $AFTER_DB_INIT_CODE"
+  echo "after_docker_init_code: $AFTER_DOCKER_INIT_CODE"
   echo "before-db-init-code: $BEFORE_DB_INIT_CODE"
   echo "ebs-volume-size: $EBS_VOLUME_SIZE"
 fi
@@ -466,25 +469,36 @@ function checkParams() {
     fi
   fi
 
+  if [ ! -z ${AFTER_DOCKER_INIT_CODE+x} ]; then
+    checkPath AFTER_DOCKER_INIT_CODE
+    if [ "$?" -ne "0" ]; then
+      #>&2 echo "WARNING: Value given as after_db_init_code: '$AFTER_DOCKER_INIT_CODE' not found as file will use as content"
+      echo "$AFTER_DOCKER_INIT_CODE" > $TMP_PATH/after_docker_init_code_tmp.sh
+      AFTER_DOCKER_INIT_CODE="$TMP_PATH/after_docker_init_code_tmp.sh"
+    else
+      [ "$DEBUG" -eq "1" ] && echo "DEBUG: Value given as commands-after-docker-init will use as filename"
+    fi
+  fi
+
   if [ ! -z ${AFTER_DB_INIT_CODE+x} ]; then
     checkPath AFTER_DB_INIT_CODE
     if [ "$?" -ne "0" ]; then
       #>&2 echo "WARNING: Value given as after_db_init_code: '$AFTER_DB_INIT_CODE' not found as file will use as content"
       echo "$AFTER_DB_INIT_CODE" > $TMP_PATH/after_db_init_code_tmp.sql
       AFTER_DB_INIT_CODE="$TMP_PATH/after_db_init_code_tmp.sql"
     else
-      [ "$DEBUG" -eq "1" ] && echo "DEBUG: Value given as after_db_init_code will use as filename"
+      [ "$DEBUG" -eq "1" ] && echo "DEBUG: Value given as sql-after-db-restore will use as filename"
     fi
   fi
 
   if [ ! -z ${BEFORE_DB_INIT_CODE+x} ]; then
     checkPath BEFORE_DB_INIT_CODE
     if [ "$?" -ne "0" ]; then
-      #>&2 echo "WARNING: Value given as after_db_init_code: '$AFTER_DB_INIT_CODE' not found as file will use as content"
+      #>&2 echo "WARNING: Value given as before_db_init_code: '$BEFORE_DB_INIT_CODE' not found as file will use as content"
       echo "$BEFORE_DB_INIT_CODE" > $TMP_PATH/before_db_init_code_tmp.sql
       BEFORE_DB_INIT_CODE="$TMP_PATH/before_db_init_code_tmp.sql"
     else
-      [ "$DEBUG" -eq "1" ] && echo "DEBUG: Value given as before_db_init_code will use as filename"
+      [ "$DEBUG" -eq "1" ] && echo "DEBUG: Value given as sql-before-db-restore will use as filename"
     fi
   fi
 
@@ -641,6 +655,7 @@ function cleanupAndExit {
   echo "$(date "+%Y-%m-%d %H:%M:%S"): Remove temp files..." # if exists
   docker $dockerConfig exec -i ${containerHash} sh -c "sudo rm -rf $MACHINE_HOME"
   rm -f "$TMP_PATH/after_db_init_code_tmp.sql"
+  rm -f "$TMP_PATH/after_docker_init_code_tmp.sh"
   rm -f "$TMP_PATH/before_db_init_code_tmp.sql"
   rm -f "$TMP_PATH/workload_custom_sql_tmp.sql"
   rm -f "$TMP_PATH/target_ddl_do_tmp.sql"
@@ -839,6 +854,19 @@ function copyFile() {
 ## Apply machine features
 # Dump
 sleep 2 # wait for postgres up&running
+OP_START_TIME=$(date +%s);
+if ([ ! -z ${AFTER_DOCKER_INIT_CODE+x} ] && [ "$AFTER_DOCKER_INIT_CODE" != "" ])
+then
+  echo "$(date "+%Y-%m-%d %H:%M:%S"): Apply code after docker init"
+  AFTER_DOCKER_INIT_CODE_FILENAME=$(basename $AFTER_DOCKER_INIT_CODE)
+  copyFile $AFTER_DOCKER_INIT_CODE
+  # --set ON_ERROR_STOP=on
+  docker_exec bash -c "chmod +x $MACHINE_HOME/$AFTER_DOCKER_INIT_CODE_FILENAME"
+  docker_exec sh $MACHINE_HOME/$AFTER_DOCKER_INIT_CODE_FILENAME
+  END_TIME=$(date +%s);
+  DURATION=$(echo $((END_TIME-OP_START_TIME)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
+  echo "$(date "+%Y-%m-%d %H:%M:%S"): After docker init code applied for $DURATION."
+fi
 
 OP_START_TIME=$(date +%s);
 if ([ ! -z ${BEFORE_DB_INIT_CODE+x} ] && [ "$BEFORE_DB_INIT_CODE" != "" ])
@@ -854,6 +882,9 @@ then
 fi
 OP_START_TIME=$(date +%s);
 echo "$(date "+%Y-%m-%d %H:%M:%S"): Restore database dump"
+#CPU_CNT=$(cat /proc/cpuinfo | grep processor | wc -l)
+CPU_CNT=$(docker_exec bash -c "cat /proc/cpuinfo | grep processor | wc -l") # for execute in docker
+let CPU_CNT=$CPU_CNT*2
 case "$DB_DUMP_EXT" in
   sql)
     docker_exec bash -c "cat $MACHINE_HOME/$DB_DUMP_FILENAME | psql --set ON_ERROR_STOP=on -U postgres test $OUTPUT_REDIRECT"
@@ -865,7 +896,7 @@ case "$DB_DUMP_EXT" in
     docker_exec bash -c "zcat $MACHINE_HOME/$DB_DUMP_FILENAME | psql --set ON_ERROR_STOP=on -U postgres test $OUTPUT_REDIRECT"
     ;;
   pgdmp)
-    docker_exec bash -c "pg_restore -j $(cat /proc/cpuinfo | grep processor | wc -l) --no-owner --no-privileges -U postgres -d test $MACHINE_HOME/$DB_DUMP_FILENAME"
+    docker_exec bash -c "pg_restore -j $CPU_CNT --no-owner --no-privileges -U postgres -d test $MACHINE_HOME/$DB_DUMP_FILENAME"
     ;;
 esac
 END_TIME=$(date +%s);
@@ -941,7 +972,7 @@ fi
 
 # Clear statistics and log
 echo "$(date "+%Y-%m-%d %H:%M:%S"): Execute vacuumdb..."
-docker_exec vacuumdb -U postgres test -j $(cat /proc/cpuinfo | grep processor | wc -l) --analyze
+docker_exec vacuumdb -U postgres test -j $CPU_CNT --analyze
 docker_exec bash -c "echo '' > /var/log/postgresql/postgresql-$PG_VERSION-main.log"
 # Execute workload
 OP_START_TIME=$(date +%s);
@@ -966,26 +997,30 @@ echo "$(date "+%Y-%m-%d %H:%M:%S"): Workload executed for $DURATION."
 OP_START_TIME=$(date +%s);
 echo "$(date "+%Y-%m-%d %H:%M:%S"): Prepare JSON log..."
 docker_exec bash -c "/root/pgbadger/pgbadger \
-  -j $(cat /proc/cpuinfo | grep processor | wc -l) \
+  -j $CPU_CNT \
   --prefix '%t [%p]: [%l-1] db=%d,user=%u (%a,%h)' /var/log/postgresql/* -f stderr \
   -o $MACHINE_HOME/$ARTIFACTS_FILENAME.json"
   #2> >(grep -v "install the Text::CSV_XS" >&2)
 
 docker_exec bash -c "gzip -c $logpath > $MACHINE_HOME/$ARTIFACTS_FILENAME.log.gz"
+docker_exec bash -c "gzip -c /etc/postgresql/$PG_VERSION/main/postgresql.conf > $MACHINE_HOME/$ARTIFACTS_FILENAME.conf.gz"
 echo "$(date "+%Y-%m-%d %H:%M:%S"): Save artifcats..."
 if [[ $ARTIFACTS_DESTINATION =~ "s3://" ]]; then
     docker_exec s3cmd put /$MACHINE_HOME/$ARTIFACTS_FILENAME.json $ARTIFACTS_DESTINATION/
     docker_exec s3cmd put /$MACHINE_HOME/$ARTIFACTS_FILENAME.log.gz $ARTIFACTS_DESTINATION/
+    docker_exec s3cmd put /$MACHINE_HOME/$ARTIFACTS_FILENAME.conf.gz $ARTIFACTS_DESTINATION/
 else
     if [ "$RUN_ON" = "localhost" ]; then
       docker cp $containerHash:$MACHINE_HOME/$ARTIFACTS_FILENAME.json $ARTIFACTS_DESTINATION/
       docker cp $containerHash:$MACHINE_HOME/$ARTIFACTS_FILENAME.log.gz $ARTIFACTS_DESTINATION/
+      docker cp $containerHash:$MACHINE_HOME/$ARTIFACTS_FILENAME.conf.gz $ARTIFACTS_DESTINATION/
       # TODO option: ln / cp
       #cp "$TMP_PATH/nancy_$containerHash/"$ARTIFACTS_FILENAME.json $ARTIFACTS_DESTINATION/
       #cp "$TMP_PATH/nancy_$containerHash/"$ARTIFACTS_FILENAME.log.gz $ARTIFACTS_DESTINATION/
     elif [ "$RUN_ON" = "aws" ]; then
       docker-machine scp $DOCKER_MACHINE:/home/storage/$ARTIFACTS_FILENAME.json $ARTIFACTS_DESTINATION/
       docker-machine scp $DOCKER_MACHINE:/home/storage/$ARTIFACTS_FILENAME.log.gz $ARTIFACTS_DESTINATION/
+      docker-machine scp $DOCKER_MACHINE:/home/storage/$ARTIFACTS_FILENAME.conf.gz $ARTIFACTS_DESTINATION/
     else
       >&2 echo "ASSERT: must not reach this point"
       exit 1