Improves handling of ssh error in sync-satellite logs

Fixes #1609

Replace tabs with spaces

Author: riya-17

server/bin/gold/test-28.txt

@@ -0,0 +1,163 @@
++++ Running pbench-sync-satellite satellite-one
+--- Finished pbench-sync-satellite (status=2)
++++ Running unit test audit
+Template:  pbench-unittests.v3.server-reports
+Index:  pbench-unittests.v3.server-reports.1970-01 1
+len(actions) = 1
+[
+    {
+        "_id": "5ca1ab1e70015f100dedfab1ed0ff1ce",
+        "_index": "pbench-unittests.v3.server-reports.1970-01",
+        "_op_type": "create",
+        "_source": {
+            "@generated-by": {
+                "commit_id": "unit-test",
+                "group_id": 43,
+                "hostname": "example.com",
+                "pid": 42,
+                "user_id": 44,
+                "version": ""
+            },
+            "@timestamp": "1970-01-01T00:00:42",
+            "chunk_id": 1,
+            "doctype": "status",
+            "name": "pbench-audit-server",
+            "text": "pbench-audit-server.run-1970-01-01T00:00:42-UTC(unit-test)\n",
+            "total_chunks": 1,
+            "total_size": 59
+        },
+        "_type": "pbench-server-reports"
+    }
+]
+--- Finished unit test audit (status=0)
++++ var/www/html tree state (/var/tmp/pbench-test-server/test-28/var-www-html)
+lrwxrwxrwx         63 incoming -> /var/tmp/pbench-test-server/test-28/pbench/public_html/incoming
+drwxrwxr-x          - pbench-results-host-info.versioned
+lrwxrwxrwx         38 pbench-results-host-info.versioned/pbench-results-host-info.URL002 -> pbench-results-host-info.URL002.active
+-rw-rw-r--        134 pbench-results-host-info.versioned/pbench-results-host-info.URL002.active
+-rw-rw-r--         95 pbench-results-host-info.versioned/pbench-results-host-info.URL002.maint
+lrwxrwxrwx         62 results -> /var/tmp/pbench-test-server/test-28/pbench/public_html/results
+lrwxrwxrwx         61 static -> /var/tmp/pbench-test-server/test-28/pbench/public_html/static
+lrwxrwxrwx         60 users -> /var/tmp/pbench-test-server/test-28/pbench/public_html/users
+--- var/www/html tree state
++++ results host info (/var/tmp/pbench-test-server/test-28/var-www-html/pbench-results-host-info.versioned)
+/var/tmp/pbench-test-server/test-28/var-www-html/pbench-results-host-info.versioned/pbench-results-host-info.URL002.active:[email protected]:/var/tmp/pbench-test-server/test-28/pbench-local/pbench-move-results-receive/fs-version-002
+/var/tmp/pbench-test-server/test-28/var-www-html/pbench-results-host-info.versioned/pbench-results-host-info.URL002.maint:MESSAGE===System Under Maintenance - please retry at a later time ([email protected])
+--- results host info
++++ var/www/html-satellite tree state (/var/tmp/pbench-test-server/test-28/var-www-html-satellite)
+lrwxrwxrwx         73 incoming -> /var/tmp/pbench-test-server/test-28/pbench-satellite/public_html/incoming
+drwxrwxr-x          - pbench-results-host-info.versioned
+lrwxrwxrwx         38 pbench-results-host-info.versioned/pbench-results-host-info.URL002 -> pbench-results-host-info.URL002.active
+-rw-rw-r--        138 pbench-results-host-info.versioned/pbench-results-host-info.URL002.active
+-rw-rw-r--         95 pbench-results-host-info.versioned/pbench-results-host-info.URL002.maint
+lrwxrwxrwx         72 results -> /var/tmp/pbench-test-server/test-28/pbench-satellite/public_html/results
+lrwxrwxrwx         71 static -> /var/tmp/pbench-test-server/test-28/pbench-satellite/public_html/static
+lrwxrwxrwx         70 users -> /var/tmp/pbench-test-server/test-28/pbench-satellite/public_html/users
+--- var/www/html-satellite tree state
++++ results host info (/var/tmp/pbench-test-server/test-28/var-www-html-satellite/pbench-results-host-info.versioned)
+/var/tmp/pbench-test-server/test-28/var-www-html-satellite/pbench-results-host-info.versioned/pbench-results-host-info.URL002.active:[email protected]:/var/tmp/pbench-test-server/test-28/pbench-satellite-local/pbench-move-results-receive/fs-version-002
+/var/tmp/pbench-test-server/test-28/var-www-html-satellite/pbench-results-host-info.versioned/pbench-results-host-info.URL002.maint:MESSAGE===System Under Maintenance - please retry at a later time ([email protected])
+--- results host info
++++ pbench tree state (/var/tmp/pbench-test-server/test-28/pbench)
+drwxrwxr-x          - archive
+drwxrwxr-x          - archive/fs-version-001
+drwxrwxr-x          - public_html
+drwxrwxr-x          - public_html/incoming
+drwxrwxr-x          - public_html/results
+drwxrwxr-x          - public_html/static
+drwxrwxr-x          - public_html/static/css
+drwxrwxr-x          - public_html/static/css/v0.2
+drwxrwxr-x          - public_html/static/css/v0.2/css
+-rw-rw-r--        308 public_html/static/css/v0.2/css/pbench_utils.css
+drwxrwxr-x          - public_html/static/css/v0.3
+drwxrwxr-x          - public_html/static/css/v0.3/css
+-rw-rw-r--      11798 public_html/static/css/v0.3/css/LICENSE.TXT
+-rw-rw-r--       3663 public_html/static/css/v0.3/css/jschart.css
+drwxrwxr-x          - public_html/static/js
+drwxrwxr-x          - public_html/static/js/v0.2
+drwxrwxr-x          - public_html/static/js/v0.2/js
+-rw-rw-r--       9415 public_html/static/js/v0.2/js/app.js
+-rw-rw-r--       5556 public_html/static/js/v0.2/js/pbench_utils.js
+drwxrwxr-x          - public_html/static/js/v0.3
+drwxrwxr-x          - public_html/static/js/v0.3/js
+-rw-rw-r--      11798 public_html/static/js/v0.3/js/LICENSE.TXT
+-rw-rw-r--     143934 public_html/static/js/v0.3/js/jschart.js
+drwxrwxr-x          - public_html/users
+--- pbench tree state
++++ pbench-local tree state (/var/tmp/pbench-test-server/test-28/pbench-local)
+drwxrwxr-x          - logs
+drwxrwxr-x          - logs/pbench-audit-server
+-rw-rw-r--          0 logs/pbench-audit-server/pbench-audit-server.error
+-rw-rw-r--        437 logs/pbench-audit-server/pbench-audit-server.log
+drwxrwxr-x          - logs/pbench-sync-satellite
+drwxrwxr-x          - logs/pbench-sync-satellite/ONE
+-rw-rw-r--          0 logs/pbench-sync-satellite/ONE/change_state.log
+-rw-rw-r--        105 logs/pbench-sync-satellite/pbench-sync-satellite.error
+-rw-rw-r--         61 logs/pbench-sync-satellite/pbench-sync-satellite.log
+drwxrwxr-x          - pbench-move-results-receive
+drwxrwxr-x          - pbench-move-results-receive/fs-version-002
+drwxrwxr-x          - quarantine
+drwxrwxr-x          - quarantine/duplicates-002
+drwxrwxr-x          - quarantine/errors-002
+drwxrwxr-x          - quarantine/md5-002
+drwxrwxr-x          - tmp
+--- pbench-local tree state
++++ pbench-satellite tree state (/var/tmp/pbench-test-server/test-28/pbench-satellite)
+drwxrwxr-x          - archive
+drwxrwxr-x          - archive/fs-version-001
+drwxrwxr-x          - public_html
+drwxrwxr-x          - public_html/incoming
+drwxrwxr-x          - public_html/results
+drwxrwxr-x          - public_html/static
+drwxrwxr-x          - public_html/static/css
+drwxrwxr-x          - public_html/static/css/v0.2
+drwxrwxr-x          - public_html/static/css/v0.2/css
+-rw-rw-r--        308 public_html/static/css/v0.2/css/pbench_utils.css
+drwxrwxr-x          - public_html/static/css/v0.3
+drwxrwxr-x          - public_html/static/css/v0.3/css
+-rw-rw-r--      11798 public_html/static/css/v0.3/css/LICENSE.TXT
+-rw-rw-r--       3663 public_html/static/css/v0.3/css/jschart.css
+drwxrwxr-x          - public_html/static/js
+drwxrwxr-x          - public_html/static/js/v0.2
+drwxrwxr-x          - public_html/static/js/v0.2/js
+-rw-rw-r--       9415 public_html/static/js/v0.2/js/app.js
+-rw-rw-r--       5556 public_html/static/js/v0.2/js/pbench_utils.js
+drwxrwxr-x          - public_html/static/js/v0.3
+drwxrwxr-x          - public_html/static/js/v0.3/js
+-rw-rw-r--      11798 public_html/static/js/v0.3/js/LICENSE.TXT
+-rw-rw-r--     143934 public_html/static/js/v0.3/js/jschart.js
+drwxrwxr-x          - public_html/users
+--- pbench-satellite tree state
++++ pbench-satellite-local tree state (/var/tmp/pbench-test-server/test-28/pbench-satellite-local)
+drwxrwxr-x          - logs
+drwxrwxr-x          - pbench-move-results-receive
+drwxrwxr-x          - pbench-move-results-receive/fs-version-002
+drwxrwxr-x          - quarantine
+drwxrwxr-x          - quarantine/duplicates-002
+drwxrwxr-x          - quarantine/errors-002
+drwxrwxr-x          - quarantine/md5-002
+drwxrwxr-x          - tmp
+--- pbench-satellite-local tree state
++++ pbench log file contents
+++++ pbench-local/logs
++++++ pbench-audit-server/pbench-audit-server.error
+----- pbench-audit-server/pbench-audit-server.error
++++++ pbench-audit-server/pbench-audit-server.log
+1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, retries: 0)
+1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0)
+----- pbench-audit-server/pbench-audit-server.log
++++++ pbench-sync-satellite/ONE/change_state.log
+----- pbench-sync-satellite/ONE/change_state.log
++++++ pbench-sync-satellite/pbench-sync-satellite.error
+pbench-sync-satellite: FAILED: ssh: connect to host pbench-satellite-error.example.com: No route to host
+----- pbench-sync-satellite/pbench-sync-satellite.error
++++++ pbench-sync-satellite/pbench-sync-satellite.log
+run-1970-01-01T00:00:42-UTC: start - 1970-01-01T00:00:42-UTC
+----- pbench-sync-satellite/pbench-sync-satellite.log
+---- pbench-local/logs
+++++ pbench-satellite-local/logs
+---- pbench-satellite-local/logs
+--- pbench log file contents
++++ test-execution.log file contents
+ssh pbench-satellite-error.example.com /var/tmp/pbench-test-server/test-28/opt/pbench-server-satellite/bin/pbench-sync-package-tarballs
+--- test-execution.log file contents

server/bin/pbench-base.sh

@@ -102,7 +102,12 @@ function log_finish {
 }
 
 function log_exit {
-    echo "${PROG}: ${1}" >&4
+    local _msg="${PROG}: ${1}"
+    if [[ -z "${3}" ]]; then
+        printf -- "%s\n" "${_msg}" >&4
+    else
+        printf -- "%s\n" "${_msg}" | tee -a "${3}" >&4
+    fi
     log_finish
     if [[ -z "${2}" ]]; then
         exit 1

server/bin/pbench-remote-sync-package-tarballs

@@ -22,18 +22,19 @@ if [[ -z "${dir}" || -z "${PROG}" ]]; then
 fi
 
 case $# in
-    2)
+    3)
         :
         ;;
     *)
-        echo "Usage: $PROG <satellite-config> <output-tar-ball>" >&2
+        echo "Usage: $PROG <satellite-config> <output-tar-ball> <ssh-error>" >&2
         exit 1
         ;;
 esac
 
 
 satellite_config=$1
 output_tar_ball=$2
+errors=$3
 shift 2
 
 remote_host=$(getconf.py satellite-host $satellite_config)
@@ -47,4 +48,4 @@ if [ -z "${remote_opt}" ]; then
     exit 1
 fi
 
-ssh ${remote_host} "${remote_opt}/bin/pbench-sync-package-tarballs" > ${output_tar_ball}
+ssh ${remote_host} "${remote_opt}/bin/pbench-sync-package-tarballs" > ${output_tar_ball}  2> ${errors}

server/bin/pbench-sync-satellite.sh

@@ -118,11 +118,13 @@ typeset -i nprocessed=0
 typeset -i nfailed_md5=0
 typeset -i nerrs=0
 
+syncerr=${tmp}/syncerrors
+
 # Fetch all the tarballs from remote host's archive
-pbench-remote-sync-package-tarballs ${satellite_config} ${tmp}/satellite.${remote_prefix}.tar
+pbench-remote-sync-package-tarballs ${satellite_config} ${tmp}/satellite.${remote_prefix}.tar ${syncerr}
 rc=$?
 if [[ $rc != 0 ]] ;then
-    log_exit "$PROG: pbench-remote-sync-package-tarballs: failed." 2
+    log_exit "FAILED: $(cat ${syncerr})" 2 "${mail_content}"
 fi
 
 if [ -s $tmp/satellite.$remote_prefix.tar ]; then

server/bin/state/test-28.config/pbench-server.cfg

@@ -0,0 +1,19 @@
+install-dir = %(unittest-dir)s/opt/pbench-server
+default-host = pbench-satellite-error.example.com
+
+[logging]
+logger_type = file
+logging_level = WARNING
+
+[satellite-one]
+# NOTE WELL: this satellite host name, pbench-satellite.example.com, is the
+# host name expected by the mock ssh command which triggers the behavior of
+# running the actual commands instead of just echoing them.
+satellite-host = pbench-satellite-error.example.com
+
+
+###########################################################################
+# The rest will come from the global state config file and the default config file.
+[config]
+path = %(unittest-dir)s/tmp, %(install-dir)s/lib/config
+files = state-pbench-server.cfg

server/bin/test-bin/ssh

@@ -9,5 +9,8 @@ if [ "${remote_host}" = "pbench-satellite.example.com" ] ;then
     # environment from leaking into the "remote" server environment.
     unset _PBENCH_SERVER_CONFIG
     $*
+elif [ "${remote_host}" == "pbench-satellite-error.example.com" ] ;then
+    echo "ssh: connect to host ${remote_host}: No route to host" >&2
+    exit 1
 fi
 exit 0

server/bin/unittests

@@ -755,6 +755,9 @@ declare -A cmds=(
 
     # Test culling unpacked tar balls
     [test-27]="_run pbench-cull-unpacked-tarballs"
+
+    # Test ssh-error
+    [test-28]="_run pbench-sync-satellite satellite-one"
 )
 all_tests_sorted=$(for x in ${!cmds[@]} ;do echo $x ;done | sed 's/\./-/' | sort -n -t '-' -k 3 | sort -n -t '-' -k 2 --stable | sed 's/\(.*-[0-9]\+\)-\([0-9]\+\)/\1.\2/')