Fix misc shellcheck complaints and run shellcheck on CI

fredrikekre · fredrikekre · commit 99d63612d105 · 2025-06-03T12:10:09.000+02:00
diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml
@@ -0,0 +1,27 @@
+---
+name: ShellCheck
+on:
+  push:
+    branches:
+      - 'main'
+      - 'release-*'
+  pull_request:
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    name: ShellCheck
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Run shellcheck on all .sh files
+        shell: bash
+        run: |
+          failures=0
+          while IFS= read -r -d '' file; do
+              if ! shellcheck "${file}"; then
+                  failures=$((failures + 1))
+              fi
+          done < <(find . -type f -name '*.sh' -print0)
+          if [[ "${failures}" -gt 0 ]]; then
+              echo "Shellcheck failed for ${failures} files."
+          fi
+          exit "${failures}"
diff --git a/.shellcheckrc b/.shellcheckrc
@@ -0,0 +1,2 @@
+# Allow opening any 'source'd file, even if not specified as input
+external-sources=true
diff --git a/debugging/bughunt/core_dump.sh b/debugging/bughunt/core_dump.sh
@@ -6,7 +6,7 @@ COREFILE="$(corefile)"
 
 # Read in all directories under `src/` and `base/` in the julia checkout;
 # we'll add these as source search paths in `gdb`
-readarray -t -d '' JULIA_SRC_DIRS < <(find /build/julia.git/src /build/julia.git/base -type d -print0)
+# readarray -t -d '' JULIA_SRC_DIRS < <(find /build/julia.git/src /build/julia.git/base -type d -print0)
 
 # Try `gdb` first, as it is less likely to freeze on us for some reason
 if [[ -n "$(which gdb)" ]]; then
diff --git a/debugging/bughunt/rr_trace.sh b/debugging/bughunt/rr_trace.sh
@@ -32,7 +32,7 @@ function fixup_trace_path() {
 }
 
 function any_failures_in_ps() {
-    [[ $("${RR}" ps "$(fixup_trace_path "${1}")" | awk "{sum+=\$3} END {print sum;}") > 0 ]]
+    [[ $("${RR}" ps "$(fixup_trace_path "${1}")" | awk "{sum+=\$3} END {print sum;}") -gt 0 ]]
 }
 
 function grep_ps() {
@@ -42,12 +42,18 @@ function grep_ps() {
 }
 
 
+# `ls` but using `find` to make shellcheck happy
+# (https://www.shellcheck.net/wiki/SC2012)
+lsfind() {
+    find "${1}" ! -name "$(basename "${1}")" -prune -exec basename {} \;
+}
+
 if [[ "$1" == "list" ]]; then
-    for trace in $(ls ${SCRIPT_DIR}/rr_traces | sort -V); do
+    for trace in $(lsfind "${SCRIPT_DIR}/rr_traces" | sort -V); do
         if any_failures_in_ps "${trace}"; then
             echo_red "$(basename "${trace}")"
         else
-            echo "$(basename "${trace}")"
+            basename "${trace}"
         fi
     done
     exit 0
@@ -60,12 +66,12 @@ if [[ "$1" == "grep" ]]; then
     fi
     shift 1
 
-    for trace in $(ls ${SCRIPT_DIR}/rr_traces | sort -V); do
+    for trace in $(lsfind "${SCRIPT_DIR}/rr_traces" | sort -V); do
         if grep_ps "${trace}" "$@"; then
             if any_failures_in_ps "${trace}"; then
                 echo_red "$(basename "${trace}")"
             else
-                echo "$(basename "${trace}")"
+                basename "${trace}"
             fi
         fi
     done
@@ -77,7 +83,7 @@ if [[ "$1" == "ps" ]]; then
         echo "Usage: ${SCRIPT} ps <trace>" >&2
         exit 1
     fi
-    TRACE_PATH="$(fixup_trace_path ${2})"
+    TRACE_PATH="$(fixup_trace_path "${2}")"
     shift 2
     exec "${RR}" ps "${TRACE_PATH}" "$@"
 fi
@@ -87,11 +93,11 @@ if [[ "$1" == "replay" ]]; then
         echo "Usage: ${SCRIPT} replay <trace>" >&2
         exit 1
     fi
-    TRACE_PATH="$(fixup_trace_path ${2})"
+    TRACE_PATH="$(fixup_trace_path "${2}")"
     shift 2
     exec "${RR}" replay -x /build/.gdbinit.src --serve-files "${TRACE_PATH}" "$@"
 fi
 
-echo "Unknown command: $@"
+echo "Unknown command: $*"
 print_usage
 exit 1
diff --git a/utilities/arches_env.sh b/utilities/arches_env.sh
@@ -36,7 +36,7 @@ for idx in "${!default_mapping_lines[@]}"; do
     defaults_map["${key}"]="${value}"
 done
 
-cat "${ARCHES_FILE}" | while read line; do
+while read -r line; do
     # Remove whitespace from the beginning and end of each line
     line="$(tr -s ' ' <<<"${line}")"
 
@@ -77,4 +77,4 @@ cat "${ARCHES_FILE}" | while read line; do
         echo -n "${name}=\"${value}\" "
     done
     echo
-done
+done <"${ARCHES_FILE}"
diff --git a/utilities/arches_pipeline_upload.sh b/utilities/arches_pipeline_upload.sh
@@ -21,7 +21,7 @@ if [[ ! -f "${YAML_FILE}" ]] ; then
 fi
 
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-"${BASH}" "${SCRIPT_DIR}/arches_env.sh" "${ARCHES_FILE}" | while read env_map; do
+"${BASH}" "${SCRIPT_DIR}/arches_env.sh" "${ARCHES_FILE}" | while read -r env_map; do
     # Export the environment mappings, then launch the yaml file
     eval "export ${env_map}"
     buildkite-agent pipeline upload "${YAML_FILE}"
diff --git a/utilities/build_envs.sh b/utilities/build_envs.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # First, extract information from our triplet
+# shellcheck source=SCRIPTDIR/extract_triplet.sh
 source .buildkite/utilities/extract_triplet.sh
 
 # Figure out what GNU Make is on this system
@@ -14,7 +15,8 @@ export MAKE
 # Apply fixups to our environment for when we're running on julia-buildkite pipeline
 if buildkite-agent meta-data exists BUILDKITE_JULIA_BRANCH; then
     # `BUILDKITE_BRANCH` should refer to `julia.git`, not `julia-buildkite.git`
-    export BUILDKITE_BRANCH=$(buildkite-agent meta-data get BUILDKITE_JULIA_BRANCH)
+    BUILDKITE_BRANCH=$(buildkite-agent meta-data get BUILDKITE_JULIA_BRANCH)
+    export BUILDKITE_BRANCH
 fi
 
 # Determine JULIA_CPU_TARGETS for different architectures
@@ -108,13 +110,16 @@ export JULIA_IMAGE_THREADS="$JULIA_CPU_THREADS"
 
 # Extract git information
 SHORT_COMMIT_LENGTH=10
-export LONG_COMMIT="$(git rev-parse HEAD)"
-export SHORT_COMMIT="$(echo ${LONG_COMMIT} | cut -c1-${SHORT_COMMIT_LENGTH})"
+LONG_COMMIT="$(git rev-parse HEAD)"
+SHORT_COMMIT="$(echo "${LONG_COMMIT}" | cut -c1-${SHORT_COMMIT_LENGTH})"
+export LONG_COMMIT SHORT_COMMIT
 
 # Extract information about the current julia version number
-export JULIA_VERSION="$(cat VERSION)"
-export MAJMIN="$(cut -d. -f1-2 <<<"${JULIA_VERSION}")"
-export MAJMINPAT="$(cut -d- -f1 <<<"${JULIA_VERSION}")"
+JULIA_VERSION="$(cat VERSION)"
+MAJMIN="$(cut -d. -f1-2 <<<"${JULIA_VERSION}")"
+MAJMINPAT="$(cut -d- -f1 <<<"${JULIA_VERSION}")"
+export JULIA_VERSION MAJMIN MAJMINPAT
+
 # If we're on a tag, then our "tar version" will be the julia version.
 # Otherwise, it's the short commit.
 if git describe --tags --exact-match >/dev/null 2>/dev/null; then
@@ -128,7 +133,8 @@ export TAR_VERSION
 # Build the filename that we'll upload as, and get the filename that will be built
 # These are not the same in situations such as `musl`, where the build system doesn't
 # differentiate but we need to give it a different name.
-export JULIA_BINARYDIST_FILENAME="$(${MAKE} print-JULIA_BINARYDIST_FILENAME | cut -c27- | tr -s ' ')"
+JULIA_BINARYDIST_FILENAME="$(${MAKE} print-JULIA_BINARYDIST_FILENAME | cut -c27- | tr -s ' ')"
+export JULIA_BINARYDIST_FILENAME
 
 export JULIA_INSTALL_DIR="julia-${TAR_VERSION}"
 JULIA_BINARY="${JULIA_INSTALL_DIR}/bin/julia${EXE}"
@@ -227,6 +233,6 @@ echo "Detected Julia version:            ${MAJMIN}  (${JULIA_VERSION})"
 echo "Detected build platform:           ${TRIPLET}  (${ARCH}, ${OS})"
 echo "Julia will be uploaded to:         s3://${UPLOAD_TARGETS[0]}.tar.gz"
 echo "With additional upload targets:"
-for UPLOAD_TARGET in ${UPLOAD_TARGETS[@]:1}; do
+for UPLOAD_TARGET in "${UPLOAD_TARGETS[@]:1}"; do
     echo " -> s3://${UPLOAD_TARGET}.tar.gz"
 done
diff --git a/utilities/build_julia.sh b/utilities/build_julia.sh
@@ -7,7 +7,9 @@
 set -euo pipefail
 
 # First, get things like `SHORT_COMMIT`, `JULIA_CPU_TARGET`, `UPLOAD_TARGETS`, etc...
+# shellcheck source=SCRIPTDIR/build_envs.sh
 source .buildkite/utilities/build_envs.sh
+# shellcheck source=SCRIPTDIR/word.sh
 source .buildkite/utilities/word.sh
 
 echo "--- Print software versions"
@@ -54,7 +56,8 @@ fi
 MFLAGS+=( "JULIA_CPU_TARGET=${JULIA_CPU_TARGET}" )
 
 # Finish off with any extra make flags from the `.arches` file
-MFLAGS+=( $(tr "," " " <<<"${MAKE_FLAGS}") )
+IFS=',' read -ra ARCHES_FLAGS <<<"${MAKE_FLAGS}"
+MFLAGS+=( "${ARCHES_FLAGS[@]}" )
 
 echo "Make Options:"
 for FLAG in "${MFLAGS[@]}"; do
diff --git a/utilities/deploy_docs.sh b/utilities/deploy_docs.sh
@@ -3,11 +3,12 @@
 set -euo pipefail
 
 # Get the `.buildkite/utilities/docs` folder path
-DOCS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+# DOCS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 echo "--- Download built docs"
 buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz"
 
 echo "--- Deploy docs"
-export DOCUMENTER_KEY="$(cat .buildkite/secrets/ssh_docs_deploy)"
+DOCUMENTER_KEY="$(cat .buildkite/secrets/ssh_docs_deploy)"
+export DOCUMENTER_KEY
 echo "Do something here!"
diff --git a/utilities/extract_triplet.sh b/utilities/extract_triplet.sh
@@ -2,7 +2,8 @@
 
 # Extract information from our triplet
 # Here is an example of a triplet: `x86_64-linux-gnu`
-export ARCH="$(cut -d- -f1 <<<"${TRIPLET}")"
+ARCH="$(cut -d- -f1 <<<"${TRIPLET}")"
+export ARCH
 case "${TRIPLET}" in
     # Linux
     *-gnu)
diff --git a/utilities/macos/codesign.sh b/utilities/macos/codesign.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
 set -euo pipefail
@@ -79,9 +79,9 @@ fi
 
 # Verify that we can load the given identity
 if [ "${CODESIGN_IDENTITY}" != "-" ]; then
-    if ! security find-identity -p codesigning ${KEYCHAIN_PATH:-} 2>&1 | grep "${CODESIGN_IDENTITY}" >/dev/null; then
+    if ! security find-identity -p codesigning "${KEYCHAIN_PATH:-}" 2>&1 | grep "${CODESIGN_IDENTITY}" >/dev/null; then
         echo "security find-identity found no matching identity for '${CODESIGN_IDENTITY}':"
-        security find-identity -p codesigning ${KEYCHAIN_PATH:-}
+        security find-identity -p codesigning "${KEYCHAIN_PATH:-}"
         exit 1
     fi
 fi
@@ -93,8 +93,8 @@ do_codesign() {
     # and replacing any previous signatures that may exist.
     codesign --sign "${CODESIGN_IDENTITY}" \
              --option=runtime \
-             --entitlements $(dirname "${0}")/Entitlements.plist \
-             ${KEYCHAIN_ARGS} \
+             --entitlements "$(dirname "${0}")/Entitlements.plist" \
+             "${KEYCHAIN_ARGS}" \
              --timestamp \
              --force \
              "${1}"
diff --git a/utilities/sign_tarball.sh b/utilities/sign_tarball.sh
@@ -3,15 +3,16 @@
 TARGET="${2}"
 
 # Create temporary gpg home so that we have an isolated keyring
-export GNUPGHOME="$(mktemp -d)"
+GNUPGHOME="$(mktemp -d)"
+export GNUPGHOME
 trap 'rm -rf "$GNUPGHOME"' EXIT INT TERM HUP
 
 # Import key as first argument.  The `touch` is a workaround for a `gpg-agent` bug.
 touch /dev/random /dev/urandom
 gpg --import <"${1}"
 
 # Sign the second argument as a file, creating a `.asc` file.
-gpg --armor --detach-sig --batch --yes "${2}"
+gpg --armor --detach-sig --batch --yes "${TARGET}"
 
 # Kill gpg-agent when we're done
 gpgconf --kill gpg-agent
diff --git a/utilities/test_julia.sh b/utilities/test_julia.sh
@@ -8,6 +8,7 @@
 set -euo pipefail
 
 # First, get things like `SHORT_COMMIT`, `JULIA_CPU_TARGET`, `UPLOAD_TARGETS`, etc...
+# shellcheck source=SCRIPTDIR/build_envs.sh
 source .buildkite/utilities/build_envs.sh
 
 echo "--- Print kernel version"
@@ -33,7 +34,7 @@ if [[ "${OS}" == "macos" ]]; then
     echo "--- [mac] Codesigning"
     .buildkite/utilities/macos/codesign.sh "${JULIA_INSTALL_DIR}"
     echo "--- [mac] Update checksums for stdlib cachefiles after codesigning"
-    JULIA_DEBUG=all ${JULIA_INSTALL_DIR}/bin/julia .buildkite/utilities/update_stdlib_pkgimage_checksums.jl
+    JULIA_DEBUG=all "${JULIA_INSTALL_DIR}/bin/julia" .buildkite/utilities/update_stdlib_pkgimage_checksums.jl
 fi
 
 
@@ -51,8 +52,9 @@ unset JULIA_PKG_SERVER
 # Make sure that temp files and temp directories are created in a location that is
 # backed by real storage, and not by a tmpfs, as some tests don't like that on Linux
 if [[ "${OS}" == "linux" ]]; then
-    export TMPDIR="$(pwd)/tmp"
-    mkdir -p ${TMPDIR}
+    TMPDIR="$(pwd)/tmp"
+    export TMPDIR
+    mkdir -p "${TMPDIR}"
 fi
 
 #Always set the max rss so that if tests add large global variables (which they do) we don't make the GC's life too hard
@@ -80,12 +82,12 @@ if [[ "${USE_RR-}" == "rr" ]] || [[ "${USE_RR-}" == "rr-net" ]]; then
 
     # rr: all tests EXCEPT the network-related tests
     # rr-net: ONLY the network-related tests
-    NETWORK_RELATED_TESTS=( Artifacts Downloads download LazyArtifacts LibGit2/online)
+    NETWORK_RELATED_TESTS=( Artifacts Downloads download LazyArtifacts LibGit2/online )
     if [[ "${USE_RR-}" == "rr" ]]; then
-        TESTS_TO_SKIP+=( ${NETWORK_RELATED_TESTS[@]} )
+        TESTS_TO_SKIP+=( "${NETWORK_RELATED_TESTS[@]}" )
     elif [[ "${USE_RR-}" == "rr-net" ]]; then
         # Overwrite TESTS_TO_RUN, to get rid of default `"all"`
-        TESTS_TO_RUN=( ${NETWORK_RELATED_TESTS[@]} )
+        TESTS_TO_RUN=( "${NETWORK_RELATED_TESTS[@]}" )
     fi
 elif [[ "${USE_RR-}" == "" ]]; then
     # Run inside of a timeout
@@ -122,9 +124,9 @@ fi
 # Build our `TESTS` string
 # `--ci` asserts that networking is available
 if [[ "${#TESTS_TO_SKIP[@]}" -gt 0 ]]; then
-    export TESTS="${TESTS_TO_RUN[@]} --ci --skip ${TESTS_TO_SKIP[@]}"
+    export TESTS="${TESTS_TO_RUN[*]} --ci --skip ${TESTS_TO_SKIP[*]}"
 else
-    export TESTS="${TESTS_TO_RUN[@]} --ci"
+    export TESTS="${TESTS_TO_RUN[*]} --ci"
 fi
 
 # Auto-set timeout to buildkite timeout minus 45m for most users
@@ -161,8 +163,8 @@ fi
 
 echo "--- Upload results.json report"
 # store the test job id so that the upload job can assign the results to the right job id
-buildkite-agent meta-data set BUILDKITE_TEST_JOB_ID_${BUILDKITE_STEP_KEY} "${BUILDKITE_JOB_ID}"
-echo "meta-data BUILDKITE_TEST_JOB_ID_${BUILDKITE_STEP_KEY} has been set to \"$(buildkite-agent meta-data get BUILDKITE_TEST_JOB_ID_${BUILDKITE_STEP_KEY})\""
+buildkite-agent meta-data set "BUILDKITE_TEST_JOB_ID_${BUILDKITE_STEP_KEY}" "${BUILDKITE_JOB_ID}"
+echo "meta-data BUILDKITE_TEST_JOB_ID_${BUILDKITE_STEP_KEY} has been set to \"$(buildkite-agent meta-data get "BUILDKITE_TEST_JOB_ID_${BUILDKITE_STEP_KEY}")\""
 if compgen -G "${JULIA_INSTALL_DIR}/share/julia/test/results*.json"; then
     (cd "${JULIA_INSTALL_DIR}/share/julia/test"; tar -czf results.tar.gz results*.json && buildkite-agent artifact upload "results.tar.gz")
 else
diff --git a/utilities/upload_julia.sh b/utilities/upload_julia.sh
diff --git a/utilities/windows/codesign.sh b/utilities/windows/codesign.sh

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# Allow opening any 'source'd file, even if not specified as input`
	`2`	`+external-sources=true`