Merge #14115: lint: Make all linters work under the default macOS dev environment (build-osx.md)

341f7c7 macOS fix: Check for correct version of flake8 to avoid spurious warnings. The brew installed flake8 version is Python 2 based and does not work. (practicalswift)
908a559 macOS fix: Add excludes for checks added in the newer shellcheck version installed by brew (practicalswift)
ec4d57b macOS fix: Work around empty (sub)expression error when using BSD grep (practicalswift)
b57d7d9 macOS fix: Avoid mapfile due to ancient version of bash shipped with macOS (practicalswift)

Pull request description:

  The linters are thoroughly tested under Ubuntu which is what we use in Travis. When reading #14041 I understood that some developers were experiencing problems when running the linters on their local machines.

  Assuming these local machines were running macOS I installed a fresh macOS VM, followed the instructions in `build-osx.md` and ran the linters.

  This PR contains the changes needed to make `lint-all.sh` run as expected.

  Ideally the linters would continuously run also under a Travis macOS environment to make sure we catch these kind of issues before merge.

Tree-SHA512: b39c9a970d14d27db1fb592539923c0bc676b5217f415d02fda3f17bf54d46faa172376e8a3ecab07ca68a3acba9aebe00b2b1b2161b2a36b85fbb672e7efb5c

Author: MarcoFalke

test/lint/lint-format-strings.sh

@@ -33,7 +33,9 @@ if ! python3 -m doctest test/lint/lint-format-strings.py; then
 fi
 for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do
     IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}"
-    mapfile -t MATCHING_FILES < <(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue)")
+    for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue)"); do
+        MATCHING_FILES+=("${MATCHING_FILE}")
+    done
     if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then
         EXIT_CODE=1
     fi

test/lint/lint-locale-dependence.sh

@@ -197,11 +197,11 @@ REGEXP_IGNORE_KNOWN_VIOLATIONS=$(join_array "|" "${KNOWN_VIOLATIONS[@]}")
 
 # Invoke "git grep" only once in order to minimize run-time
 REGEXP_LOCALE_DEPENDENT_FUNCTIONS=$(join_array "|" "${LOCALE_DEPENDENT_FUNCTIONS[@]}")
-GIT_GREP_OUTPUT=$(git grep -E "[^a-zA-Z0-9_\`'\"<>](${REGEXP_LOCALE_DEPENDENT_FUNCTIONS}(|_r|_s))[^a-zA-Z0-9_\`'\"<>]" -- "*.cpp" "*.h")
+GIT_GREP_OUTPUT=$(git grep -E "[^a-zA-Z0-9_\`'\"<>](${REGEXP_LOCALE_DEPENDENT_FUNCTIONS}(_r|_s)?)[^a-zA-Z0-9_\`'\"<>]" -- "*.cpp" "*.h")
 
 EXIT_CODE=0
 for LOCALE_DEPENDENT_FUNCTION in "${LOCALE_DEPENDENT_FUNCTIONS[@]}"; do
-    MATCHES=$(grep -E "[^a-zA-Z0-9_\`'\"<>]${LOCALE_DEPENDENT_FUNCTION}(|_r|_s)[^a-zA-Z0-9_\`'\"<>]" <<< "${GIT_GREP_OUTPUT}" | \
+    MATCHES=$(grep -E "[^a-zA-Z0-9_\`'\"<>]${LOCALE_DEPENDENT_FUNCTION}(_r|_s)?[^a-zA-Z0-9_\`'\"<>]" <<< "${GIT_GREP_OUTPUT}" | \
         grep -vE "\.(c|cpp|h):\s*(//|\*|/\*|\").*${LOCALE_DEPENDENT_FUNCTION}" | \
         grep -vE 'fprintf\(.*(stdout|stderr)')
     if [[ ${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES} != "" ]]; then

test/lint/lint-python.sh

@@ -79,4 +79,12 @@ export LC_ALL=C
 # W605 invalid escape sequence "x"
 # W606 'async' and 'await' are reserved keywords starting with Python 3.7
 
-flake8 --ignore=B,C,E,F,I,N,W --select=E101,E112,E113,E115,E116,E125,E129,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,E901,E902,F401,F402,F403,F404,F405,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W191,W291,W292,W293,W504,W601,W602,W603,W604,W605,W606 .
+if ! command -v flake8 > /dev/null; then
+    echo "Skipping Python linting since flake8 is not installed. Install by running \"pip3 install flake8\""
+    exit 0
+elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then
+    echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8 by running \"pip3 install flake8\""
+    exit 0
+fi
+
+PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=E101,E112,E113,E115,E116,E125,E129,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,E901,E902,F401,F402,F403,F404,F405,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W191,W291,W292,W293,W504,W601,W602,W603,W604,W605,W606 .

test/lint/lint-shell-locale.sh

@@ -16,7 +16,7 @@ for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|univalue
     if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then
         continue
     fi
-    FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*|)$' "${SHELL_SCRIPT}" | head -1)
+    FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1)
     if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then
         echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}"
         EXIT_CODE=1

test/lint/lint-shell.sh

@@ -16,7 +16,14 @@ if [ "$TRAVIS" = "true" ]; then
   unset LC_ALL
 fi
 
+if ! command -v shellcheck > /dev/null; then
+    echo "Skipping shell linting since shellcheck is not installed."
+    exit 0
+fi
+
 # Disabled warnings:
+# SC1087: Use braces when expanding arrays, e.g. ${array[idx]} (or ${var}[.. to quiet).
+# SC1117: Backslash is literal in "\.". Prefer explicit escaping: "\\.".
 # SC2001: See if you can use ${variable//search/replace} instead.
 # SC2004: $/${} is unnecessary on arithmetic variables.
 # SC2005: Useless echo? Instead of 'echo $(cmd)', just use 'cmd'.
@@ -33,5 +40,8 @@ fi
 # SC2166: Prefer [ p ] && [ q ] as [ p -a q ] is not well defined.
 # SC2166: Prefer [ p ] || [ q ] as [ p -o q ] is not well defined.
 # SC2181: Check exit code directly with e.g. 'if mycmd;', not indirectly with $?.
-shellcheck -e SC2001,SC2004,SC2005,SC2006,SC2016,SC2028,SC2046,SC2048,SC2066,SC2086,SC2116,SC2148,SC2162,SC2166,SC2181 \
+# SC2206: Quote to prevent word splitting, or split robustly with mapfile or read -a.
+# SC2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting).
+# SC2230: which is non-standard. Use builtin 'command -v' instead.
+shellcheck -e SC1087,SC1117,SC2001,SC2004,SC2005,SC2006,SC2016,SC2028,SC2046,SC2048,SC2066,SC2086,SC2116,SC2148,SC2162,SC2166,SC2181,SC2206,SC2207,SC2230 \
     $(git ls-files -- "*.sh" | grep -vE 'src/(secp256k1|univalue)/')