Rework the checks and error messages displayed on network error

Create a new function isConnected in helpers.sh which calls getIPAddress, but also sets a new global
__NET_ERRMSG if not connected. This is used in setup.sh and packages.sh so we don't duplicate the same
message between files.

Split out a new function runCurl from the download function, to do lower level curl calls, but capture any
errors into __NET_ERRMSG for display in setup.sh or via the packaging functions. Use a connect timeout of 10
in the download function rather than 60 which is quite long, but keep the 60 second limit on no data transfer.

Adjust rp_remoteFileExists to use runCurl, so we can capture any error when checking a remote file,
and use this value for display in setup and in logs etc. Switch to using long versions of curl options for
readability, and add --show-error so we still output errors when being silent.

Also switch rp_getFileDate to use new runCurl function, so that any curl argument overrides are handled, and use
long format parameters also. No error handling is done here though.

Author: joolswills

scriptmodules/admin/setup.sh

@@ -179,8 +179,7 @@ function package_setup() {
         local has_binary=0
         local has_net=0
 
-        local ip="$(getIPAddress)"
-        [[ -n "$ip" ]] && has_net=1
+        isConnected && has_net=1
 
         # for modules with nonet flag that don't need to download data, we force has_net to 1, so we get install options
         hasFlag "${__mod_info[$id/flags]}" "nonet" && has_net=1
@@ -253,7 +252,7 @@ function package_setup() {
                 options+=(S "${option_msgs[S]}")
            fi
         else
-            status+="\nInstall options disabled (Unable to access internet)"
+            status+="\nInstall options disabled:\n$__NET_ERRMSG"
         fi
 
         if [[ "$is_installed" -eq 1 ]]; then
@@ -372,9 +371,8 @@ function section_gui_setup() {
         local pkgs=()
 
         status="Please choose a package from below"
-        local ip="$(getIPAddress)"
-        if [[ -z "$ip" ]]; then
-            status+="\nInstall options disabled (Unable to access internet)"
+        if ! isConnected; then
+            status+="\nInstall options disabled ($__NET_ERRMSG)"
             has_net=0
         fi
 

scriptmodules/helpers.sh

@@ -1003,6 +1003,48 @@ function applyPatch() {
     return 0
 }
 
+## @fn runCurl
+## @params ... commandline arguments to pass to curl
+## @brief Run curl with chosen parameters and handle curl errors
+## @details Runs curl with the provided parameters, whilst also capturing the output and extracting
+## any error message, which is stored in the global variable __NET_ERRMSG. Function returns the return
+## code provided by curl. The environment variable __curl_opts can be set to override default curl
+## parameters, eg - timeouts etc.
+## @retval curl return value
+function runCurl() {
+    local params=("$@")
+    # add any user supplied curl opts - timeouts can be overridden as curl uses the last parameters given
+    [[ -z "$__curl_opts" ]] && params+=($__curl_opts)
+
+    local cmd_err
+    local ret
+
+    # get the last non zero exit status (ignoring tee)
+    set -o pipefail
+
+    # set up additional file descriptor for stdin
+    exec 3>&1
+
+    # capture stderr - while passing both stdout and stderr to terminal
+    # curl like wget outputs the progress meter to stderr, so we will extract the error line later
+    cmd_err=$(curl "${params[@]}" 2>&1 1>&3 | tee /dev/stderr)
+    ret="$?"
+
+    # remove stdin copy
+    exec 3>&-
+
+    set +o pipefail
+
+    # if there was an error, extract it and put in __NET_ERRMSG
+    if [[ "$ret" -ne 0 ]]; then
+        # as we also capture the curl progress output, extract the last line which contains the error
+        __NET_ERRMSG="${cmd_err##*$'\n'}"
+    else
+        __NET_ERRMSG=""
+    fi
+    return "$ret"
+}
+
 ## @fn download()
 ## @param url url of file
 ## @param dest destination name (optional), use - for stdout
@@ -1030,43 +1072,21 @@ function download() {
         printMsgs "console" "Downloading $url to $dest ..."
         params+=(-o "$dest")
     fi
-    params+=(--connect-timeout 60 --speed-limit 1 --speed-time 60)
-    # add any user supplied curl opts - timeouts can be overridden as curl uses the last parameters given
-    [[ -z "$__curl_opts" ]] && params+=($__curl_opts)
+    params+=(--connect-timeout 10 --speed-limit 1 --speed-time 60)
     # add the url
     params+=("$url")
 
-    local cmd_err
     local ret
-
-    # get the last non zero exit status (ignoring tee)
-    set -o pipefail
-
-    # capture stderr - while passing both stdout and stderr to terminal
-    # curl like wget outputs the progress meter to stderr, so we will extract the error line later
-
-    # set up additional file descriptor for stdin
-    exec 3>&1
-
-    cmd_err=$(curl "${params[@]}" 2>&1 1>&3 | tee /dev/stderr)
+    runCurl "${params[@]}"
     ret="$?"
 
-    # remove stdin copy
-    exec 3>&-
-
-    set +o pipefail
-
     # if download failed, remove file, log error and return error code
     if [[ "$ret" -ne 0 ]]; then
         # remove dest if not set to stdout and exists
         [[ "$dest" != "-" && -f "$dest" ]] && rm "$dest"
-        # as we also capture the curl progress output, extract the last line which contains the error
-        cmd_err="${cmd_err##*$'\n'}"
-
-        md_ret_errors+=("URL $url failed to download.\n\n$cmd_err")
-        return "$ret"
+        md_ret_errors+=("URL $url failed to download.\n\n$__NET_ERRMSG")
     fi
-    return 0
+    return "$ret"
 }
 
 ## @fn downloadAndVerify()
@@ -1546,6 +1566,21 @@ function getIPAddress() {
     [[ -n "$ip_route" ]] && grep -oP "src \K[^\s]+" <<< "$ip_route"
 }
 
+## @fn isConnected()
+## @brief Simple check to see if there is a connection to the Internet.
+## @details Uses the getIPAddress function to check if we have a route to the Internet. Also sets
+## __NET_ERRMSG with an error message for use in packages / setup to display to the user if not.
+## @retval 0 on success
+## @retval 1 on failure
+function isConnected() {
+    local ip="$(getIPAddress)"
+    if [[ -z "$ip" ]]; then
+        __NET_ERRMSG="Not connected to the Internet"
+        return 1
+    fi
+    return 0
+}
+
 ## @fn adminRsync()
 ## @param src src folder on local system - eg "$__tmpdir/stats/"
 ## @param dest destination folder on remote system - eg "stats/"

scriptmodules/packages.sh

@@ -19,6 +19,8 @@ declare -A __sections=(
     [depends]="dependency"
 )
 
+__NET_ERRMSG=""
+
 function rp_listFunctions() {
     local id
     local desc
@@ -133,8 +135,7 @@ function rp_callModule() {
             local has_binary=0
             local has_net=0
 
-            local ip="$(getIPAddress)"
-            [[ -n "$ip" ]] && has_net=1
+            isConnected && has_net=1
 
             # for modules with nonet flag that don't need to download data, we force has_net to 1
             hasFlag "${__mod_info[$id/flags]}" "nonet" && has_net=1
@@ -148,7 +149,7 @@ function rp_callModule() {
 
             # fail if we don't seem to be connected
             if [[ "$has_net" -eq 0 ]]; then
-                __ERRMSGS+=("Can't install/update $md_id - unable to connect to the internet")
+                __ERRMSGS+=("Can't install/update $md_id - $__NET_ERRMSG")
                 return 1
             fi
 
@@ -424,7 +425,9 @@ function rp_getBinaryUrl() {
 function rp_remoteFileExists() {
     local url="$1"
     local ret
-    curl --max-time 10 -o /dev/null -sfI "$url"
+    # runCurl will cause stderr to be copied to output so we redirect both stdout/stderr to /dev/null.
+    # any errors will have been captured by runCurl
+    runCurl --max-time 10 --silent --show-error --fail --head "$url" &>/dev/null
     ret="$?"
     if [[ "$ret" -eq 0 ]]; then
         return 0
@@ -468,7 +471,7 @@ function rp_getFileDate() {
     [[ -z "$url" ]] && return 1
 
     # get last-modified date stripping any CR in the output
-    local file_date=$(curl -sfI --no-styled-output "$url" | tr -d "\r" | grep -ioP "last-modified: \K.+")
+    local file_date=$(runCurl --silent --fail --head --no-styled-output "$url" | tr -d "\r" | grep -ioP "last-modified: \K.+")
     # if there is a date set in last-modified header, then convert to iso-8601 format
     if [[ -n "$file_date" ]]; then
         file_date="$(date -Iseconds --date="$file_date")"