Skip to content
This repository was archived by the owner on Aug 16, 2021. It is now read-only.

Commit 584fbc0

Browse files
author
Dmitry
committed
Some remarks fixed, pg_stat_* artifcats
1 parent c7b1587 commit 584fbc0

File tree

1 file changed

+50
-13
lines changed

1 file changed

+50
-13
lines changed

nancy_run.sh

Lines changed: 50 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1175,7 +1175,6 @@ function apply_commands_after_container_init() {
11751175
msg "Apply code after docker init"
11761176
COMMANDS_AFTER_CONTAINER_INIT_FILENAME=$(basename $COMMANDS_AFTER_CONTAINER_INIT)
11771177
copy_file $COMMANDS_AFTER_CONTAINER_INIT
1178-
# --set ON_ERROR_STOP=on
11791178
docker_exec bash -c "chmod +x $MACHINE_HOME/$COMMANDS_AFTER_CONTAINER_INIT_FILENAME"
11801179
docker_exec sh $MACHINE_HOME/$COMMANDS_AFTER_CONTAINER_INIT_FILENAME
11811180
END_TIME=$(date +%s);
@@ -1374,7 +1373,7 @@ function prepare_start_workload() {
13741373
| grep / | sed -e 's/^[ \t]*//'"
13751374
)
13761375
docker_exec bash -c "mkdir $MACHINE_HOME/$ARTIFACTS_FILENAME"
1377-
docker_exec bash -c "gzip -c $logpath > $MACHINE_HOME/$ARTIFACTS_FILENAME/$ARTIFACTS_FILENAME.prepare.log.gz"
1376+
docker_exec bash -c "gzip -c $logpath > $MACHINE_HOME/$ARTIFACTS_FILENAME/postgresql.prepare.log.gz"
13781377

13791378
# Clear statistics and log
13801379
msg "Execute vacuumdb..."
@@ -1396,11 +1395,11 @@ function execute_workload() {
13961395
# Execute workload
13971396
OP_START_TIME=$(date +%s);
13981397
msg "Execute workload..."
1399-
if [ ! -z ${WORKLOAD_REAL+x} ] && [ "$WORKLOAD_REAL" != '' ]; then
1398+
if [[ ! -z ${WORKLOAD_REAL+x} ]] && [[ "$WORKLOAD_REAL" != '' ]]; then
14001399
msg "Execute pgreplay queries..."
14011400
docker_exec psql -U postgres $DB_NAME -c 'create role testuser superuser login;'
14021401
WORKLOAD_FILE_NAME=$(basename $WORKLOAD_REAL)
1403-
if [ ! -z ${WORKLOAD_REAL_REPLAY_SPEED+x} ] && [ "$WORKLOAD_REAL_REPLAY_SPEED" != '' ]; then
1402+
if [[ ! -z ${WORKLOAD_REAL_REPLAY_SPEED+x} ]] && [[ "$WORKLOAD_REAL_REPLAY_SPEED" != '' ]]; then
14041403
docker_exec bash -c "pgreplay -r -s $WORKLOAD_REAL_REPLAY_SPEED $MACHINE_HOME/$WORKLOAD_FILE_NAME"
14051404
else
14061405
docker_exec bash -c "pgreplay -r -j $MACHINE_HOME/$WORKLOAD_FILE_NAME"
@@ -1433,11 +1432,45 @@ function collect_results() {
14331432
docker_exec bash -c "/root/pgbadger/pgbadger \
14341433
-j $CPU_CNT \
14351434
--prefix '%t [%p]: [%l-1] db=%d,user=%u (%a,%h)' /var/log/postgresql/* -f stderr \
1436-
-o $MACHINE_HOME/$ARTIFACTS_FILENAME/$ARTIFACTS_FILENAME.json" \
1435+
-o $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json" \
1436+
2> >(grep -v "install the Text::CSV_XS" >&2)
1437+
msg "Prepare HTML log..."
1438+
docker_exec bash -c "/root/pgbadger/pgbadger \
1439+
-j $CPU_CNT \
1440+
--prefix '%t [%p]: [%l-1] db=%d,user=%u (%a,%h)' /var/log/postgresql/* -f stderr \
1441+
-o $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.html" \
14371442
2> >(grep -v "install the Text::CSV_XS" >&2)
14381443

1439-
docker_exec bash -c "gzip -c $logpath > $MACHINE_HOME/$ARTIFACTS_FILENAME/$ARTIFACTS_FILENAME.log.gz"
1440-
docker_exec bash -c "gzip -c /etc/postgresql/$PG_VERSION/main/postgresql.conf > $MACHINE_HOME/$ARTIFACTS_FILENAME/$ARTIFACTS_FILENAME.conf.gz"
1444+
#pg_stat_* artifacts
1445+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_activity) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_activity.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1446+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_archiver) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_archiver.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1447+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_bgwriter) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_bgwriter.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1448+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_database) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_database.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1449+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_database_conflicts) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_database_conflicts.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1450+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_all_tables) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_all_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1451+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_sys_tables) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_sys_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1452+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_user_tables) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_user_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1453+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_xact_all_tables) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_xact_all_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1454+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_xact_sys_tables) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_xact_sys_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1455+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_xact_user_tables) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_xact_user_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1456+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_all_indexes) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_all_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1457+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_sys_indexes) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_sys_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1458+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_user_indexes) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_user_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1459+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_all_tables) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_statio_all_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1460+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_sys_tables) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_statio_sys_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1461+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_user_tables) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_statio_user_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1462+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_all_indexes) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_statio_all_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1463+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_sys_indexes) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_statio_sys_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1464+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_user_indexes) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_statio_user_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1465+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_all_sequences) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_statio_all_sequences.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1466+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_sys_sequences) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_statio_sys_sequences.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1467+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_user_sequences) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_statio_user_sequences.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1468+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_user_functions) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_user_functions.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1469+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_xact_user_functions) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_xact_user_functions.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1470+
#docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_statements) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_statements.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1471+
1472+
docker_exec bash -c "gzip -c $logpath > $MACHINE_HOME/$ARTIFACTS_FILENAME/postgresql.workload.log.gz"
1473+
docker_exec bash -c "cp /etc/postgresql/$PG_VERSION/main/postgresql.conf $MACHINE_HOME/$ARTIFACTS_FILENAME/"
14411474
msg "Save artifacts..."
14421475
if [[ $ARTIFACTS_DESTINATION =~ "s3://" ]]; then
14431476
docker_exec s3cmd --recursive put /$MACHINE_HOME/$ARTIFACTS_FILENAME $ARTIFACTS_DESTINATION/
@@ -1486,12 +1519,16 @@ apply_ddl_undo_code;
14861519
END_TIME=$(date +%s);
14871520
DURATION=$(echo $((END_TIME-START_TIME)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
14881521
echo -e "$(date "+%Y-%m-%d %H:%M:%S"): Run done for $DURATION"
1489-
echo -e " Report: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME.json"
1490-
echo -e " Query log: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME.log.gz"
1522+
echo -e " JSON Report: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/pgbadger.json"
1523+
echo -e " HTML Report: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/pgbadger.html"
1524+
echo -e " Query log: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/postgresql.workload.log.gz"
1525+
echo -e " Prepare log: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/postgresql.prepare.log.gz"
1526+
echo -e " Postgresql configuration log: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/postgresql.conf"
1527+
14911528
echo -e " -------------------------------------------"
14921529
echo -e " Workload summary:"
1493-
echo -e " Summarized query duration:\t" $(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/$ARTIFACTS_FILENAME.json | jq '.overall_stat.queries_duration') " ms"
1494-
echo -e " Queries:\t\t\t" $( docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/$ARTIFACTS_FILENAME.json | jq '.overall_stat.queries_number')
1495-
echo -e " Query groups:\t\t" $(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/$ARTIFACTS_FILENAME.json | jq '.normalyzed_info| length')
1496-
echo -e " Errors:\t\t\t" $(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/$ARTIFACTS_FILENAME.json | jq '.overall_stat.errors_number')
1530+
echo -e " Summarized query duration:\t" $(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.queries_duration') " ms"
1531+
echo -e " Queries:\t\t\t" $( docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.queries_number')
1532+
echo -e " Query groups:\t\t" $(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.normalyzed_info| length')
1533+
echo -e " Errors:\t\t\t" $(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.errors_number')
14971534
echo -e "-------------------------------------------"

0 commit comments

Comments
 (0)