Fix ignored failures in Homebrew tests

util/cron/test-homebrew-linux.bash

@@ -9,14 +9,15 @@
 # tarball created and sha of the tarball,
 # and run home-brew test-bot commands as homebrew CI does
 
+set -exo pipefail
+
 # Create a tarball from current repo.
 # The tarball is left in root of repo in tar/ directory.
 UTIL_CRON_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
 
 # common-tarball sets CHPL_HOME
 source $UTIL_CRON_DIR/common-tarball.bash
 
-
 # Tell gen_release to use existing repo instead of creating a new one with
 # git-archive.
 export CHPL_GEN_RELEASE_NO_CLONE=true

util/cron/test-homebrew.bash

@@ -6,7 +6,7 @@
 # replace the url and sha in the chapel formula with the url pointing to the tarball created and sha of the tarball.
 # run home-brew scripts to install chapel.
 
-set -ex
+set -exo pipefail
 
 UTIL_CRON_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
 
@@ -58,14 +58,18 @@ sed_command="sed -i.bak -e "
 $sed_command "s#url.*#url \"file\:///$location\"#" chapel.rb
 $sed_command "1s/sha256.*/sha256 \"$sha256\"/;t" -e "1,/sha256.*/s//sha256 \"$sha256\"/" chapel.rb
 
+${CHPL_HOME}/util/packaging/docker/test/brew_get_bogus_bottles.bash | sed -e '/<bottle-block-placeholder-injected-during-testing>/r /dev/stdin' -e '/<bottle-block-placeholder-injected-during-testing>/d' -i '' chapel.rb
+
 log_info "Chapel formula to be tested:"
 cat chapel.rb
 
 # Test if homebrew install using the chapel formula works.
 brew upgrade
 brew uninstall --force chapel
+
 # Remove the cached chapel tar file before running brew install --build-from-source chapel.rb
 rm -f $HOME/Library/Caches/Homebrew/downloads/*--chapel-${short_version}.tar.gz
+
 HOMEBREW_DEVELOPER=1 HOMEBREW_NO_INSTALL_FROM_API=1 brew install -v --build-from-source ./chapel.rb
 chpl --version
 

util/packaging/docker/test/Dockerfile

@@ -20,7 +20,6 @@ RUN brew update
 # Upgrade to the latest version of all installed formulae
 RUN brew upgrade
 
-
 # necessary so that updated chapel.rb formula can be committed, which is only way it
 # will be tested
 RUN git config --global user.name "Chapel Tester"
@@ -29,7 +28,10 @@ RUN git config --global user.email "[email protected]"
 # COPY chapel.rb and chapel*.tar.gz inside the container to run homebrew install
 COPY chapel.rb /home/linuxbrew/.linuxbrew/Homebrew/Library/Taps/homebrew/homebrew-core/Formula/c/chapel.rb
 COPY chapel*.tar.gz /home/linuxbrew/
-# must commit the change or brew doctor will complain
 WORKDIR /home/linuxbrew/.linuxbrew/Homebrew/Library/Taps/homebrew/homebrew-core/
+
+# must commit the change or brew doctor will complain
 RUN git add Formula/c/chapel.rb && git commit -m "update chapel.rb for nightly testing"
 WORKDIR /home/linuxbrew/
+
+COPY brew_get_bogus_bottles.bash brew_get_bogus_bottles.bash

util/packaging/docker/test/brew_get_bogus_bottles.bash

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Generate a "bottle do..." block with bogus SHA256 hashes, to work around
+# homebrew behavior of not erroring on failures for un-bottled platforms
+# (https://github.com/Homebrew/homebrew-test-bot/issues/791).
+
+# Have to unset this (just within this script) so `brew info` queries the API
+HOMEBREW_NO_INSTALL_FROM_API=
+
+# Run once before capturing output, due to initial run having output for
+# downloading metadata.
+brew info &> /dev/null
+
+platforms=$(brew info --json=v1 chapel | jq ".[0].bottle.stable.files | keys_unsorted[]" -r)
+
+max_length=$(echo "$platforms" | awk ' { if ( length > x ) { x = length; y = $0 } }END{ print x+1 }')
+
+echo "$platforms" |
+  while read -r platform; do
+    # hash platform name to get a real SHA which would not match the binary
+    namehash=$(echo $platform | sha256sum | cut -d" " -f1)
+    # print platforms with padding to align the sha256 hashes
+    printf "    sha256 %s \"%s\"\n" "$(printf "%-${max_length}s" "$platform:")" "$namehash"
+  done
+echo "  end"

util/packaging/docker/test/brew_install.bash

@@ -1,4 +1,13 @@
 #!/usr/bin/env bash
+
+set -ex
+
+# Hack to inject bogus bottle block into chapel.rb for testing purposes.
+cd /home/linuxbrew/.linuxbrew/Homebrew/Library/Taps/homebrew/homebrew-core
+/home/linuxbrew/brew_get_bogus_bottles.bash | sed -i -e '/<bottle-block-placeholder-injected-during-testing>/r /dev/stdin' -e '/<bottle-block-placeholder-injected-during-testing>/d' Formula/c/chapel.rb
+git add Formula/c/chapel.rb && git commit -m "update chapel.rb for nightly testing" && PAGER=cat git show
+cd /home/linuxbrew
+
 # Important to set HOMEBREW_NO_INSTALL_FROM_API to actually test changes
 export HOMEBREW_NO_INSTALL_FROM_API=1
 # Might not be needed, but also doesn't hurt and matches homebrew CI
@@ -14,35 +23,12 @@ export HOMEBREW_NO_AUTO_UPDATE=1
 # fi
 
 # tests commands
+
 brew test-bot --only-cleanup-before
-if [ $? -ne 0 ]; then
-      echo "brew test-bot --only-cleanup-before failed"
-      exit 1
-else
-      echo "brew test-bot --only-cleanup-before succeeded"
-fi
 
 brew test-bot --only-setup
-if [ $? -ne 0 ]; then
-      echo "brew test-bot --only-setup failed"
-      exit 1
-else
-      echo "brew test-bot --only-setup succeeded"
-fi
 
 brew test-bot --only-formulae-dependents --junit --testing-formulae=chapel --skipped-or-failed-formulae=chapel
-if [ $? -ne 0 ]; then
-      echo "brew test-bot --only-formulae-dependents --junit --testing-formulae=chapel --skipped-or-failed-formulae=chapel failed"
-      exit 1
-else
-      echo "brew test-bot --only-formulae-dependents --junit --testing-formulae=chapel --skipped-or-failed-formulae=chapel succeeded"
-fi
 
 # This is the bulk of the testing and the same command that Homebrew CI runs
 brew test-bot  --skip-online-checks --only-formulae --junit --only-json-tab --skip-dependents --testing-formulae="chapel" --added-formulae="" --deleted-formulae=""
-if [ $? -ne 0 ]; then
-      echo "brew test-bot --skip-online-checks --only-formulae --junit --only-json-tab --skip-dependents --testing-formulae="chapel" --added-formulae="" --deleted-formulae="" failed"
-      exit 1
-else
-      echo "brew test-bot --skip-online-checks --only-formulae --junit --only-json-tab --skip-dependents --testing-formulae="chapel" --added-formulae="" --deleted-formulae="" succeeded"
-fi

util/packaging/docker/test/homebrew_ci.bash

@@ -6,6 +6,8 @@ cd ${CHPL_HOME}/util/packaging/docker/test
 
 # Remove image with name homebrew_ci before creating a fresh image to avoid failures.
 docker image rm --force homebrew_ci
+
+# Build image
 docker build . --load --platform linux/amd64 -t homebrew_ci
 containerid= docker image ls | grep 'homebrew_ci' | awk '{print$3}'
 

util/packaging/homebrew/chapel-main.rb

@@ -10,9 +10,11 @@ class Chapel < Formula
 
   no_autobump! because: :bumped_by_upstream
 
-  # Don't include the bottle information in chapel-main.rb deliberately. The
-  # idea is that we don't want to accidentally use a published bottle in our testing,
-  # which would always report passing.
+  # Don't include real bottle information here, to avoid accidentally testing
+  # off of a published bottle.
+  bottle do
+  <bottle-block-placeholder-injected-during-testing>
+  # Intentionally not `end`ed so if insertion fails the test will break
 
   depends_on "cmake"
   depends_on "gmp"