Merge branch 'Travis-and-install-fixes'

 Fixes #101

Author: zbeekman

.travis.yml

@@ -13,18 +13,13 @@ env:
     - MPICH_GCC7_BOT_URL_HEAD="https://github.com/sourceryinstitute/OpenCoarrays/files/976779/"
     - MPICH_BOT_URL_TAIL="mpich-3.2_3.yosemite.bottle.1.tar.gz"
     - BUILD_TYPES="Release Debug RelWithDebInfo CodeCoverage"
-  matrix: # shellcheck installed manually below for macOS
-    - GCC=6 OSX_PACKAGES="gcc@6 cmake"
-    - GCC=7 OSX_PACKAGES="gcc cmake"
+  matrix:
+    - GCC=6 OSX_PACKAGES="gcc@6 shellcheck" BUILD_TYPE="InstallScript"
+    - GCC="6 7" OSX_PACKAGES="gcc gcc@6 cmake shellcheck" BUILD_TYPE="cmake"
 
 matrix:
   fast_finish: true
   include:
-    - os: osx
-      env:
-        - BUILD_TYPE="InstallScript"
-          OSX_PACKAGES="gcc@6"
-          GCC=6
     - &ubuntu
       os: linux
       sudo: false
@@ -44,8 +39,6 @@ matrix:
             - gcc-6
             - gfortran-6
             - g++-6
-            - binutils
-            - cmake-data
             - cmake
     -
       <<: *ubuntu
@@ -58,8 +51,6 @@ matrix:
             - gcc-7
             - gfortran-7
             - g++-7
-            - binutils
-            - cmake-data
             - cmake
     -
       <<: *ubuntu
@@ -87,37 +78,8 @@ matrix:
     - *linux_allowed_failure
 
 before_install:
-  - export FC=gfortran-${GCC}
-  - export CC=gcc-${GCC}
-  - export CXX=g++-${GCC}
-  - export MPICH_BOT_URL_HEAD=MPICH_GCC${GCC}_BOT_URL_HEAD
-  - |
-    if [[ (-n ${TRAVIS_TAG}) && (${TRAVIS_OS_NAME} == osx) ]] && ${TRAVIS_SECURE_ENV_VARS} ; then
-      brew update > /dev/null
-      brew ls --versions gpg2 >/dev/null || brew install gpg2
-      brew outdated gpg2 || brew upgrade gpg2
-      type -P openssl || brew install openssl
-    fi
-    if ! [[ "${TRAVIS_TAG}" ]] || ! ${TRAVIS_SECURE_ENV_VARS} ; then
-      unset encrypted_ef4535c39461_key || true
-      unset encrypted_ef4535c39461_iv || true
-      rm subkey-328B3A0E-secret.asc{,.enc} || true
-
-    fi
-    if [[ "${TRAVIS_TAG}" ]] ; then
-      curl https://izaakbeekman.com/izaak.pubkey.txt | gpg --import
-      git tag -v ${TRAVIS_TAG}
-    fi
-  - |
-    if [[ (-n ${TRAVIS}) && (${TRAVIS_OS_NAME} == osx) ]]; then
-      export PATH="${PATH}:${HOME}/Library/Python/2.7/bin"
-    else
-      [[ -d "${CACHE}/bin" ]] || mkdir -p "${CACHE}/bin"
-      export PATH="${CACHE}/bin:${PATH}"
-      ${FC} --version
-      ${CC} --version
-      ${CXX} --version
-    fi
+  - source developer-scripts/travis/before_install.all.sh
+  - developer-scripts/travis/before_install.${TRAVIS_OS_NAME}.sh
 
 install:
   - pwd
@@ -129,8 +91,7 @@ script:
 
 after_script:
   - find . -name '*.gcno' -print
-  - gcov-${GCC} --version
-  - bash <(curl -s https://codecov.io/bash) -x $(type -P gcov-${GCC})
+  - for version in ${GCC}; do bash <(curl -s https://codecov.io/bash) -x $(type -P gcov-${GCC}) ; done
   - |
     if [[ "${TRAVIS_TAG}" ]]; then
       if [[ "v${TRAVIS_TAG}" != "v$(sed -n '/[0-9]\{1,\}\(\.[0-9]\{1,\}\)\{1,\}/{s/^\([^.]*\)\([0-9]\{1,\}\(\.[0-9]\{1,\}\)\{1,\}\)\(.*\)/\2/p;q;}' .VERSION)" ]]; then

CMakeLists.txt

@@ -584,15 +584,16 @@ add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure)
 #---------------------------------------------------------------------------------------
 
 # Determine if we're using Open MPI
+cmake_host_system_information(RESULT N_CPU QUERY NUMBER_OF_LOGICAL_CORES)
+cmake_host_system_information(RESULT HOST_NAME QUERY HOSTNAME)
 execute_process(COMMAND ${MPIEXEC} --version
   OUTPUT_VARIABLE mpi_version_out)
 if (mpi_version_out MATCHES "[Oo]pen[ -][Mm][Pp][Ii]")
   message( STATUS "OpenMPI detected")
   set ( openmpi true )
+  # Write out a host file because OMPI's mpiexec is dumb
+  file(APPEND ${CMAKE_BINARY_DIR}/hostfile "${HOST_NAME} slots=${N_CPU}\n")
 endif ()
-
-include( ProcessorCount )
-ProcessorCount(N_CPU)
 function(add_caf_test name num_caf_img test_target)
   # Function to add MPI tests.
   if(TARGET ${test_target})
@@ -605,6 +606,10 @@ function(add_caf_test name num_caf_img test_target)
       message( FATAL_ERROR "Test ${name} requires ${min_test_imgs} but was only given ${num_caf_images}" )
     endif()
   endif()
+  # Add a host file for OMPI
+  if ( openmpi )
+    set(test_parameters --hostfile ${CMAKE_BINARY_DIR}/hostfile)
+  endif()
   if ( ((N_CPU LESS num_caf_img) OR (N_CPU EQUAL 0)) )
     message(STATUS "Test ${name} is oversubscribed: ${num_caf_img} CAF images requested with ${N_CPU} system processor available.")
     if ( openmpi )
@@ -613,7 +618,7 @@ function(add_caf_test name num_caf_img test_target)
       elseif ( N_CPU LESS 2 )
         set( num_caf_img 2 )
       endif()
-      set (test_parameters --oversubscribe)
+      set (test_parameters ${test_parameters} --oversubscribe)
       message( STATUS "Open-MPI back end detected, passing --oversubscribe for oversubscribed test, ${name}, with ${num_caf_img} ranks/images." )
     endif()
   endif()
@@ -762,8 +767,8 @@ if(opencoarrays_aware_compiler)
 
   # IMAGE FAIL tests
   if(NOT CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 7.0.0)
-    add_caf_test(image_status_test_1 4 image_status_test_1)
     if(CAF_ENABLE_FAILED_IMAGES)
+      add_caf_test(image_status_test_1 4 image_status_test_1)
       if ((NOT DEFINED ENV{TRAVIS}) OR (CAF_RUN_DEVELOPER_TESTS OR $ENV{OPENCOARRAYS_DEVELOPER}))
 	add_caf_test(image_fail_test_1 4 image_fail_test_1)
 	set_property(TEST image_fail_test_1 PROPERTY FAIL_REGULAR_EXPRESSION "Test failed.")

INSTALL.md

@@ -489,7 +489,7 @@ file.
 [MVAPICH]:http://mvapich.cse.ohio-state.edu
 [MacPorts]: https://www.macports.org
 [GCC]: http://gcc.gnu.org
-[TS18508 Additional Parallel Features in Fortran]: http://isotc.iso.org/livelink/livelink/nfetch/-8919044/8919782/8919787/17001078/ISO%2DIECJTC1%2DSC22%2DWG5_N2056_Draft_TS_18508_Additional_Paralle.pdf?nodeid=17181227&vernum=0
+[TS18508 Additional Parallel Features in Fortran]: https://isotc.iso.org/livelink/livelink/nfetch/-8919044/8919782/8919787/17001078/ISO%2DIECJTC1%2DSC22%2DWG5_N2056_Draft_TS_18508_Additional_Paralle.pdf?nodeid=17181227&vernum=0
 [GFortran Binaries]:  https://gcc.gnu.org/wiki/GFortranBinaries#FromSource
 [Installing GCC]: https://gcc.gnu.org/install/
 [Arch Linux]: https://www.archlinux.org
@@ -499,10 +499,10 @@ file.
 [commit comment 20539810]: https://github.com/sourceryinstitute/OpenCoarrays/commit/26e99919fe732576f7277a0e1b83f43cc7c9d749#commitcomment-20539810
 [Homebrew]: https://brew.sh
 [dnf]: https://github.com/rpm-software-management/dnf
-[port details]: http://www.freshports.org/lang/opencoarrays
+[port details]: https://www.freshports.org/lang/opencoarrays
 [port search]: https://www.freebsd.org/cgi/ports.cgi?query=opencoarrays
 [EasyBuild]: https://github.com/easybuilders/easybuild
-[Spack]: https://github.com/LLNL/spack
+[Spack]: https://github.com/spack/spack
 [add a new compiler to Spack]: http://spack.readthedocs.io/en/latest/tutorial_modules.html#add-a-new-compiler
 [loading the environment modules with Spack]: http://spack.readthedocs.io/en/latest/module_file_support.html#cmd-spack-module-loads
 [OpenCoarrays Portfile]: https://www.macports.org/ports.php?by=name&substr=opencoarrays

developer-scripts/travis/before_install.all.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+if ! [[ "${TRAVIS_TAG}" ]] || ! ${TRAVIS_SECURE_ENV_VARS} ; then
+    unset encrypted_ef4535c39461_key || true
+    unset encrypted_ef4535c39461_iv || true
+    rm subkey-328B3A0E-secret.asc{,.enc} || true
+else
+    echo "Encrypted vars not unset"
+fi
+if [[ "${OSTYPE}" == [Dd]arwin* ]]; then
+    export PATH="${PATH}:${HOME}/Library/Python/2.7/bin"
+else
+    export PATH="${CACHE}/bin:${PATH}"
+    export FC=gfortran-${GCC}
+    export CC=gcc-${GCC}
+    export CXX=g++-${GCC}
+fi

developer-scripts/travis/before_install.linux.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o verbose
+set -o pipefail
+set -o nounset
+set -o errtrace
+
+__file=developer-scripts/travis/install.linux.sh
+
+# Error tracing
+# requires `set -o errtrace`
+__caf_err_report() {
+    error_code=${?}
+    echo "Error (code=${error_code}) in ${__file} in function ${1} on line ${2}." >&2
+    false
+    return ${error_code}
+}
+# Always provide an error backtrace
+trap '__caf_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
+
+[[ -d "${CACHE}/bin" ]] || mkdir -p "${CACHE}/bin"
+${FC} --version
+${CC} --version
+${CXX} --version

developer-scripts/travis/before_install.osx.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o verbose
+set -o pipefail
+set -o nounset
+set -o errtrace
+
+__file=developer-scripts/travis/install.linux.sh
+
+# Error tracing
+# requires `set -o errtrace`
+__caf_err_report() {
+    error_code=${?}
+    echo "Error (code=${error_code}) in ${__file} in function ${1} on line ${2}." >&2
+    false
+    return ${error_code}
+}
+# Always provide an error backtrace
+trap '__caf_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
+
+if [[ -n "${TRAVIS_TAG}" ]] && ${TRAVIS_SECURE_ENV_VARS} ; then
+    brew update > /dev/null
+    brew ls --versions gpg2 >/dev/null || brew install gpg2
+    brew outdated gpg2 || brew upgrade gpg2
+    type -P openssl || brew install openssl
+    curl https://izaakbeekman.com/izaak.pubkey.txt | gpg --import
+    git tag -v "${TRAVIS_TAG}"
+fi
+gfortran --version || true
+gcc --version || true
+g++ --version || true

developer-scripts/travis/install.linux.sh

@@ -11,11 +11,10 @@ __file=developer-scripts/travis/install.linux.sh
 # Error tracing
 # requires `set -o errtrace`
 __caf_err_report() {
-  local error_code
-  error_code=${?}
-  echo "Error (code=${error_code}) in ${__file} in function ${1} on line ${2}." >&2
-  false
-  return ${error_code}
+    error_code=${?}
+    echo "Error (code=${error_code}) in ${__file} in function ${1} on line ${2}." >&2
+    false
+    return ${error_code}
 }
 # Always provide an error backtrace
 trap '__caf_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
@@ -28,6 +27,8 @@ if [[ "${BUILD_TYPE:-}" != InstallScript ]]; then # Ubuntu on Travis-CI, NOT tes
         # could use prerequisites/build instead...
         wget "${MPICH_URL_HEAD}/${MPICH_URL_TAIL}"
         tar -xzvf "${MPICH_URL_TAIL}"
+	export CC=gcc-${GCC}
+	export FC=gfortran-${GCC}
         (
 	    cd "${MPICH_URL_TAIL%.tar.gz}"
             ./configure --prefix="${CACHE}"

developer-scripts/travis/install.osx.sh

@@ -11,11 +11,10 @@ __file=developer-scripts/travis/install.osx.sh
 # Error tracing
 # requires `set -o errtrace`
 __caf_err_report() {
-  local error_code
-  error_code=${?}
-  echo "Error (code=${error_code}) in ${__file} in function ${1} on line ${2}." >&2
-  false
-  return ${error_code}
+    error_code=${?}
+    echo "Error (code=${error_code}) in ${__file} in function ${1} on line ${2}." >&2
+    false
+    return ${error_code}
 }
 # Always provide an error backtrace
 trap '__caf_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
@@ -25,20 +24,17 @@ echo "Performing Travis-CI installation phase on macOS..."
 # Update and install via Homebrew on macOS
 brew update > /dev/null
 
-brew ls --versions shellcheck >/dev/null || brew install --force-bottle shellcheck
-brew outdated shellcheck || brew upgrade --force-bottle shellcheck
 for pkg in ${OSX_PACKAGES}; do
-    brew ls --versions "${pkg}" >/dev/null || brew install "${pkg}" || brew link --overwrite "${pkg}"
-    brew outdated "${pkg}" || brew upgrade "${pkg}"
+    brew ls --versions "${pkg}" >/dev/null || brew install --force-bottle "${pkg}" || brew link --overwrite "${pkg}"
+    brew outdated "${pkg}" || brew upgrade --force-bottle "${pkg}"
 done
+
+# Uninstall mpich and openmpi so that we can install our own version
+brew uninstall --force --ignore-dependencies openmpi || true
+brew uninstall --force --ignore-dependencies mpich || true
+
 if [[ "${BUILD_TYPE:-}" == InstallScript ]]; then # uninstall some stuff if present
     brew uninstall --force --ignore-dependencies cmake || true
-    brew uninstall --force --ignore-dependencies mpich || true
-    brew uninstall --force --ignore-dependencies openmpi || true
-else
-    wget "${!MPICH_BOT_URL_HEAD}${MPICH_BOT_URL_TAIL}"
-    brew install --force-bottle "${MPICH_BOT_URL_TAIL}"
-    brew ls --versions mpich >/dev/null || brew install --force-bottle mpich
 fi
 
 {

developer-scripts/travis/test-script.InstallScript.sh

@@ -11,24 +11,38 @@ __file=developer-scripts/travis/test-script.InstallScript.sh
 # Error tracing
 # requires `set -o errtrace`
 __caf_err_report() {
-  local error_code
-  error_code=${?}
-  echo "Error (code=${error_code}) in ${__file} in function ${1} on line ${2}." >&2
-  false
-  return ${error_code}
+    error_code=${?}
+    echo "Error (code=${error_code}) in ${__file} in function ${1} on line ${2}." >&2
+    false
+    return ${error_code}
 }
 # Always provide an error backtrace
 trap '__caf_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
 
 echo "Performing Travis-CI script phase for the OpenCoarrays installation script..."
-
+export FC=gfortran-${GCC}
+export CC=gcc-${GCC}
+export CXX=g++-${GCC}
 ./install.sh --yes-to-all -i "${HOME}/opencoarrays" -j 4 -f "$(type -P "${FC}")" -c "$(type -P "${CC}")" -C "$(type -P "${CXX}")"
-CTEST_LOC=(prerequisites/installations/cmake/*/bin/ctest)
-INSTALLER_CTEST="${CTEST_LOC[${#CTEST_LOC[@]}-1]}"
-if [[ -x "${INSTALLER_CTEST}" ]] ;then
-    "${INSTALLER_CTEST}" --output-on-failure --schedule-random --repeat-until-fail "${NREPEAT:-5}"
+BUILD_LOC=(prerequisites/builds/opencoarrays/*/)
+BUILD_LOC_DIR="${BUILD_LOC[${#BUILD_LOC[@]}-1]}"
+if [[ -d "${BUILD_LOC_DIR}" ]]; then
+    echo "Found opencoarrays build directory created by the install script:"
+    echo "   ${BUILD_LOC_DIR}"
+    (
+	cd "${BUILD_LOC[${#BUILD_LOC[@]}-1]}"
+	CTEST_LOC=(../../../installations/cmake/*/bin/ctest)
+	INSTALLER_CTEST="${CTEST_LOC[${#CTEST_LOC[@]}-1]}"
+	if [[ -x "${INSTALLER_CTEST}" ]] ;then
+	    "${INSTALLER_CTEST}" --output-on-failure --schedule-random --repeat-until-fail "${NREPEAT:-5}"
+	else
+	    ctest --output-on-failure --schedule-random --repeat-until-fail "${NREPEAT:-5}"
+	fi
+    )
 else
-    ctest --output-on-failure --schedule-random --repeat-until-fail "${NREPEAT:-5}"
+    echo "Failed to find install.sh build directory. Contents of prerequisites/builds is:"
+    ls prerequisites/builds/*/
+    exit 5
 fi
 
 echo "Done."