@@ -1175,7 +1175,6 @@ function apply_commands_after_container_init() {
1175
1175
msg " Apply code after docker init"
1176
1176
COMMANDS_AFTER_CONTAINER_INIT_FILENAME=$( basename $COMMANDS_AFTER_CONTAINER_INIT )
1177
1177
copy_file $COMMANDS_AFTER_CONTAINER_INIT
1178
- # --set ON_ERROR_STOP=on
1179
1178
docker_exec bash -c " chmod +x $MACHINE_HOME /$COMMANDS_AFTER_CONTAINER_INIT_FILENAME "
1180
1179
docker_exec sh $MACHINE_HOME /$COMMANDS_AFTER_CONTAINER_INIT_FILENAME
1181
1180
END_TIME=$( date +%s) ;
@@ -1374,7 +1373,7 @@ function prepare_start_workload() {
1374
1373
| grep / | sed -e 's/^[ \t]*//'"
1375
1374
)
1376
1375
docker_exec bash -c " mkdir $MACHINE_HOME /$ARTIFACTS_FILENAME "
1377
- docker_exec bash -c " gzip -c $logpath > $MACHINE_HOME /$ARTIFACTS_FILENAME /$ARTIFACTS_FILENAME .prepare.log.gz"
1376
+ docker_exec bash -c " gzip -c $logpath > $MACHINE_HOME /$ARTIFACTS_FILENAME /postgresql .prepare.log.gz"
1378
1377
1379
1378
# Clear statistics and log
1380
1379
msg " Execute vacuumdb..."
@@ -1396,11 +1395,11 @@ function execute_workload() {
1396
1395
# Execute workload
1397
1396
OP_START_TIME=$( date +%s) ;
1398
1397
msg " Execute workload..."
1399
- if [ ! -z ${WORKLOAD_REAL+x} ] && [ " $WORKLOAD_REAL " != ' ' ]; then
1398
+ if [[ ! -z ${WORKLOAD_REAL+x} ]] && [[ " $WORKLOAD_REAL " != ' ' ] ]; then
1400
1399
msg " Execute pgreplay queries..."
1401
1400
docker_exec psql -U postgres $DB_NAME -c ' create role testuser superuser login;'
1402
1401
WORKLOAD_FILE_NAME=$( basename $WORKLOAD_REAL )
1403
- if [ ! -z ${WORKLOAD_REAL_REPLAY_SPEED+x} ] && [ " $WORKLOAD_REAL_REPLAY_SPEED " != ' ' ]; then
1402
+ if [[ ! -z ${WORKLOAD_REAL_REPLAY_SPEED+x} ]] && [[ " $WORKLOAD_REAL_REPLAY_SPEED " != ' ' ] ]; then
1404
1403
docker_exec bash -c " pgreplay -r -s $WORKLOAD_REAL_REPLAY_SPEED $MACHINE_HOME /$WORKLOAD_FILE_NAME "
1405
1404
else
1406
1405
docker_exec bash -c " pgreplay -r -j $MACHINE_HOME /$WORKLOAD_FILE_NAME "
@@ -1433,11 +1432,45 @@ function collect_results() {
1433
1432
docker_exec bash -c " /root/pgbadger/pgbadger \
1434
1433
-j $CPU_CNT \
1435
1434
--prefix '%t [%p]: [%l-1] db=%d,user=%u (%a,%h)' /var/log/postgresql/* -f stderr \
1436
- -o $MACHINE_HOME /$ARTIFACTS_FILENAME /$ARTIFACTS_FILENAME .json" \
1435
+ -o $MACHINE_HOME /$ARTIFACTS_FILENAME /pgbadger.json" \
1436
+ 2> >( grep -v " install the Text::CSV_XS" >&2 )
1437
+ msg " Prepare HTML log..."
1438
+ docker_exec bash -c " /root/pgbadger/pgbadger \
1439
+ -j $CPU_CNT \
1440
+ --prefix '%t [%p]: [%l-1] db=%d,user=%u (%a,%h)' /var/log/postgresql/* -f stderr \
1441
+ -o $MACHINE_HOME /$ARTIFACTS_FILENAME /pgbadger.html" \
1437
1442
2> >( grep -v " install the Text::CSV_XS" >&2 )
1438
1443
1439
- docker_exec bash -c " gzip -c $logpath > $MACHINE_HOME /$ARTIFACTS_FILENAME /$ARTIFACTS_FILENAME .log.gz"
1440
- docker_exec bash -c " gzip -c /etc/postgresql/$PG_VERSION /main/postgresql.conf > $MACHINE_HOME /$ARTIFACTS_FILENAME /$ARTIFACTS_FILENAME .conf.gz"
1444
+ # pg_stat_* artifacts
1445
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_activity) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_activity.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1446
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_archiver) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_archiver.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1447
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_bgwriter) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_bgwriter.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1448
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_database) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_database.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1449
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_database_conflicts) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_database_conflicts.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1450
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_all_tables) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_all_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1451
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_sys_tables) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_sys_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1452
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_user_tables) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_user_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1453
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_xact_all_tables) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_xact_all_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1454
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_xact_sys_tables) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_xact_sys_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1455
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_xact_user_tables) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_xact_user_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1456
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_all_indexes) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_all_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1457
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_sys_indexes) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_sys_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1458
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_user_indexes) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_user_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1459
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_all_tables) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_statio_all_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1460
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_sys_tables) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_statio_sys_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1461
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_user_tables) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_statio_user_tables.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1462
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_all_indexes) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_statio_all_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1463
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_sys_indexes) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_statio_sys_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1464
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_user_indexes) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_statio_user_indexes.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1465
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_all_sequences) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_statio_all_sequences.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1466
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_sys_sequences) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_statio_sys_sequences.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1467
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_statio_user_sequences) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_statio_user_sequences.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1468
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_user_functions) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_user_functions.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1469
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_xact_user_functions) to /$MACHINE_HOME /$ARTIFACTS_FILENAME /pg_stat_xact_user_functions.csv with csv;' $VERBOSE_OUTPUT_REDIRECT "
1470
+ # docker_exec bash -c "psql -U postgres $DB_NAME -b -c '\copy (select * from pg_stat_statements) to /$MACHINE_HOME/$ARTIFACTS_FILENAME/pg_stat_statements.csv with csv;' $VERBOSE_OUTPUT_REDIRECT"
1471
+
1472
+ docker_exec bash -c " gzip -c $logpath > $MACHINE_HOME /$ARTIFACTS_FILENAME /postgresql.workload.log.gz"
1473
+ docker_exec bash -c " cp /etc/postgresql/$PG_VERSION /main/postgresql.conf $MACHINE_HOME /$ARTIFACTS_FILENAME /"
1441
1474
msg " Save artifacts..."
1442
1475
if [[ $ARTIFACTS_DESTINATION =~ " s3://" ]]; then
1443
1476
docker_exec s3cmd --recursive put /$MACHINE_HOME /$ARTIFACTS_FILENAME $ARTIFACTS_DESTINATION /
@@ -1486,12 +1519,16 @@ apply_ddl_undo_code;
1486
1519
END_TIME=$( date +%s) ;
1487
1520
DURATION=$( echo $(( END_TIME- START_TIME)) | awk ' {printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}' )
1488
1521
echo -e " $( date " +%Y-%m-%d %H:%M:%S" ) : Run done for $DURATION "
1489
- echo -e " Report: $ARTIFACTS_DESTINATION /$ARTIFACTS_FILENAME .json"
1490
- echo -e " Query log: $ARTIFACTS_DESTINATION /$ARTIFACTS_FILENAME .log.gz"
1522
+ echo -e " JSON Report: $ARTIFACTS_DESTINATION /$ARTIFACTS_FILENAME /pgbadger.json"
1523
+ echo -e " HTML Report: $ARTIFACTS_DESTINATION /$ARTIFACTS_FILENAME /pgbadger.html"
1524
+ echo -e " Query log: $ARTIFACTS_DESTINATION /$ARTIFACTS_FILENAME /postgresql.workload.log.gz"
1525
+ echo -e " Prepare log: $ARTIFACTS_DESTINATION /$ARTIFACTS_FILENAME /postgresql.prepare.log.gz"
1526
+ echo -e " Postgresql configuration log: $ARTIFACTS_DESTINATION /$ARTIFACTS_FILENAME /postgresql.conf"
1527
+
1491
1528
echo -e " -------------------------------------------"
1492
1529
echo -e " Workload summary:"
1493
- echo -e " Summarized query duration:\t" $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /$ARTIFACTS_FILENAME .json | jq ' .overall_stat.queries_duration' ) " ms"
1494
- echo -e " Queries:\t\t\t" $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /$ARTIFACTS_FILENAME .json | jq ' .overall_stat.queries_number' )
1495
- echo -e " Query groups:\t\t" $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /$ARTIFACTS_FILENAME .json | jq ' .normalyzed_info| length' )
1496
- echo -e " Errors:\t\t\t" $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /$ARTIFACTS_FILENAME .json | jq ' .overall_stat.errors_number' )
1530
+ echo -e " Summarized query duration:\t" $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /pgbadger .json | jq ' .overall_stat.queries_duration' ) " ms"
1531
+ echo -e " Queries:\t\t\t" $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /pgbadger .json | jq ' .overall_stat.queries_number' )
1532
+ echo -e " Query groups:\t\t" $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /pgbadger .json | jq ' .normalyzed_info| length' )
1533
+ echo -e " Errors:\t\t\t" $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /pgbadger .json | jq ' .overall_stat.errors_number' )
1497
1534
echo -e " -------------------------------------------"
0 commit comments