Merge pull request #310 from sourceryinstitute/small-cleanups

Changes for resolving issues and release automation

 - Fixes #79
 - Fixes  #297 
 - Add regression test for #243 
 - Add regression for #172 (currently set to pass when test fails)
 - Migrates to use of a `.VERSION` file to make parsing easier for scripts and allow extra comments
 - Adds auto-upload of release assets upon tagging with git, but requires that the tag is PGP signed (`git tag -s <tag> [tree-ish]`)
 - This should compute the SHA 256 checksum and create a detached signature of the cryptographic SHA 256 checksum with an encrypted GPG subway I uploaded to the repo/travis. 🔮 🎩 🐇

Author: zbeekman

.VERSION

@@ -0,0 +1,5 @@
+# OpenCoarrays version file. Odd patch levels indicate inter-release
+# version, i.e., code is from SCM/git. This project uses semantic
+# versioning. For details see http://semver.org
+
+1.8.3

.gitattributes

@@ -21,11 +21,13 @@ prerequisites/build text
 *.tar binary
 *.gz binary
 *.tgz binary
+*.enc binary
 
 # Prevent dev-ops files from making it into the release archives
 .travis.yml export-ignore
 .pullapprove.yml export-ignore
 .gitattributes export-ignore
 .gitignore export-ignore
 developer-scripts export-ignore
-codecov.yml export-ignore
+codecov.yml export-ignore
+*.enc export-ignore

.pullapprove.yml

@@ -1,11 +1,37 @@
-approve_by_comment: true
-approve_regex: '^([Aa]pproved|:\+1:|:shipit:|:ship: it|LGTM|[Ll]ooks good to me|:zap:|:boom:|:clap:|:bow:)'
-reject_regex: '^([Nn]ope|[Nn]ope\.jpg|[Dd]enied|[Rr]ejected|:\-1:)'
-reset_on_push: true
-author_approval: default
-reviewers:
+version: 2
+extends: Default # see https://pullapprove.com/sourceryinstitute/  
+groups:
+  # These have all the group_defaults
+  Maintainers:
+    # When merging non-documentation changes into protected branches,
+    # require core team approval
     required: 1
-    members:
-        - afanfa
-        - rouson
-        - zbeekman
+    conditions:
+      branches:
+        - master
+        - devel
+        - caffeinate-opencoarrays
+      files:
+        exclude:
+          - "*.md"
+    users:
+      - afanfa
+      - rouson
+      - zbeekman
+        # https://github.com/orgs/sourceryinstitute/teams/team-opencoarrays
+  OpenCoarrays:
+    # Whenever someone has a PR, on any branch require at least 1 
+    # collaborator to explicitly review it
+    required: 1
+    users: all
+  ConfigManagers:
+    required: 1
+    conditions:
+      files:
+        - .travis.yml
+        - .pullapprove.yml
+        - codecov.yml
+        - .gitattributes
+        - "*.enc"
+    users:
+      - zbeekman

.travis.yml

@@ -67,6 +67,28 @@ matrix:
             - g++-6
 
 before_install:
+  - |
+    set -o errexit
+    if [[ "$TRAVIS_TAG" ]] && [[ "X$TRAVIS_OS_NAME" = "Xosx" ]] && $TRAVIS_SECURE_ENV_VARS ; then
+      brew update > /dev/null
+      [[ "$(brew ls --versions gpg2)" ]] || brew install gpg2
+      brew outdated gpg2 || brew upgrade gpg2
+      which openssl || brew install openssl
+    fi
+    if ! [[ "$TRAVIS_TAG" ]] || ! $TRAVIS_SECURE_ENV_VARS ; then
+      unset encrypted_ef4535c39461_key || true
+      unset encrypted_ef4535c39461_iv || true
+      rm subkey-328B3A0E-secret.asc{,.enc} || true
+
+    fi
+    if [[ "$TRAVIS_TAG" ]] ; then
+      curl https://izaakbeekman.com/izaak.pubkey.txt | gpg --import
+      git tag -v ${TRAVIS_TAG}
+    fi
+    set +o errexit
+  - 
+  - 
+  - 
   - |
     set -o errexit
     if [[ $TRAVIS ]] && [[ "X$TRAVIS_OS_NAME" = "Xosx" ]]; then
@@ -142,24 +164,78 @@ script:
       export PATH="$PATH:$HOME/bin"
       ./install.sh --yes-to-all -i "$HOME/opt/opencoarrays" -j 4 -f "$HOME/bin/gfortran" -c "$HOME/bin/gcc" -C "$HOME/bin/g++"
       cd prerequisites/builds/opencoarrays/*
-      ../../../installations/cmake/*/bin/ctest --output-on-failure
+      ../../../installations/cmake/*/bin/ctest --output-on-failure --schedule-random --repeat-until-fail 7
       cd -
     else
       mkdir cmake-build
       cd cmake-build
       cmake -DCMAKE_INSTALL_PREFIX:PATH="$HOME/OpenCoarrays" -DCMAKE_BUILD_TYPE="$BUILD_TYPE" ..
       make -j 4
-      ctest --output-on-failure
+      ctest --output-on-failure --schedule-random --repeat-until-fail 7
       make install
       cd ..
     fi
     set +o errexit
 
+after_script:
+  - |
+    if [ "$TRAVIS_TAG" ]; then
+      if [[ "v$TRAVIS_TAG" != "v$(sed -n 's/\([0-9]\{1,\}\(\.[0-9]\{1,\}\)\{1,\}\)/\1/p' .VERSION)" ]]; then
+        echo "ERROR: You are trying to tag a new release but have a version missmatch in \`.VERSION\`"
+        false # throw an error
+      fi
+    fi
+
 after_success:
   - find . -name '*.gcno' -print
   - gcov-6 --version
   - bash <(curl -s https://codecov.io/bash) -x $(which gcov-6)
 
+before_deploy:
+  - git archive -v --prefix "OpenCoarrays-${TRAVIS_TAG}/" -o "OpenCoarrays-${TRAVIS_TAG}.tar.gz" ${TRAVIS_TAG}
+  - |
+    echo '# To verify cryptographic checksums `shasum -c opencoarrays-'"${TRAVIS_TAG}"'-SHA256.txt` on Mac OS X,' > "opencoarrays-${TRAVIS_TAG}-SHA256.txt
+    echo '# `sha256sum -c  opencoarrays-'"${TRAVIS_TAG}"'-SHA256.txt` on Linux.' >> "opencoarrays-${TRAVIS_TAG}-SHA256.txt
+    if [[ "X$(uname -s)" == "XDarwin" ]]; then
+      shasum -a 256 "OpenCoarrays-${TRAVIS_TAG}.tar.gz" >> "opencoarrays-${TRAVIS_TAG}-SHA256.txt
+    else
+      sha256sum  "OpenCoarrays-${TRAVIS_TAG}.tar.gz" >> "opencoarrays-${TRAVIS_TAG}-SHA256.txt
+    fi
+  - |
+    set -o errexit
+    openssl aes-256-cbc -K $encrypted_ef4535c39461_key -iv $encrypted_ef4535c39461_iv -in subkey-328B3A0E-secret.asc.enc -out ./subkey-328B3A0E-secret.asc -d
+    gpg --allow-secret-key-import --import ./subkey-328B3A0E-secret.asc && rm subkey-328B3A0E-secret.asc
+    rm subkey-328B3A0E-secret.* || true
+    unset encrypted_ef4535c39461_key || true
+    unset encrypted_ef4535c39461_iv || true
+    gpg --armor --detach-sign -u 328B3A0E --comment 'Mac users can use GPGTools - https://gpgtools.org' \
+      --comment "Download Izaak Beekman's GPG public key from your" \
+      --comment 'trusted key server or from' \
+      --comment 'https://izaakbeekman.com/izaak.pubkey.txt' \
+      --comment 'Next add it to your GPG keyring, e.g.,' \
+      --comment '`curl https://izaakbeekman.com/izaak.pubkey.txt | gpg --import`' \
+      --comment "Make sure you have verified that the release archive's" \
+      --comment 'SHA256 checksum matches the provided' \
+      --comment "opencoarrays-${TRAVIS_TAG}-SHA256.txt and ensure that this file" \
+      --comment "and it's signature are in the same directory. Then" \
+      --comment 'verify with:' \
+      --comment '`gpg --verify opencoarrays-'"${TRAVIS_TAG}"'-SHA256.txt.asc`' \
+      "opencoarrays-${TRAVIS_TAG}-SHA256.txt"
+    set +o errexit
+
+deploy:
+  provider: releases
+  api_key:
+    secure: lBj4VgKhU74xfzPc0CDIji6cFrN/xEQIHLHjMcnDPWEF5JQe79aunMS5qiYu5ttve6khGI1WMFEe7clUO3TXiDUPyRMS4M8B2q3fuvUIsIbYgWcKq7ut+x6+dEQDj4DE3yWpPb8IDugWFp4vno1eQ2Rid1yr6Vn+rjnbE2i6z5SkXPnIv1ziQ9G+TF4ChUA8IOd1T78gJmML86P0rbt6nW8eVuJbza2y1ggHAY//q/b6rrbOFw5PqiOBqZ/BRrKJ5d2U2mvsoZtNn4OYyXw4eKEvNDIydwaIZROXfPRmN9zJ5G7+01QA5jlXpq17KfPBr7MXkn7nrQ5q9Z2ji4hRHk/AFzM/etfl1k4WCshYs13NlORpnU0vZtCMmmmsXWxj2kpsOHLWZQKdUAPs3olMPIG6aVnO5Abk1vrVSNy/ZeknbBTKCGScET1qRSaj6d43ynBwGkqHgaJdLBAwGe0Dpap6SsLGsAg56ZqHTngGlXB1KsbJKMkUsSC5RAgEuh9m55nsMaRlTYTrv+YADAQts7lV8yMCciMeJlrd3EYryyn4ZY1O82eOofhZZ5vQUktbXHtg6YnzguFwQyn+9yY/CQfi/vu2Aoq79NJEO/9UJ82ogn6gOoIebzb0xgIONjw4M68luR2U1CXtibzSYKWLGTeOj43khqZdbOpp6xncIv0=
+  file: 
+    - "OpenCoarrays-${TRAVIS_TAG}.tar.gz"
+    - "opencoarrays-${TRAVIS_TAG}-SHA256.txt"
+    - "opencoarrays-${TRAVIS_TAG}-SHA256.txt.asc"
+  skip_cleanup: true
+  on:
+    tags: true
+#    repo: sourceryinstitute/opencoarrays
+
 notifications:
   webhooks:
     urls:

CMakeLists.txt

@@ -6,8 +6,13 @@ set ( CMAKE_BUILD_TYPE "Release"
   CACHE STRING "Select which configuration to build." )
 set_property ( CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS ${CMAKE_CONFIGURATION_TYPES} )
 
-#Name project and specify source languages
-project(opencoarrays VERSION 1.8.3 LANGUAGES C Fortran)
+# Name project and specify source languages
+# Parse version from .VERSION file so that more info can be added and easier to get from scripts
+file( STRINGS ".VERSION" OpenCoarraysVersion
+  REGEX "[0-9]+\\.[0-9]+\\.[0-9]+"
+)
+project(opencoarrays VERSION "${OpenCoarraysVersion}" LANGUAGES C Fortran)
+message( STATUS "Building OpenCoarrays version: ${OpenCoarraysVersion}" )
 
 #Print an error message on an attempt to build inside the source directory tree:
 if ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
@@ -45,8 +50,48 @@ else()
   )
 endif()
 
-if (NOT (CMAKE_VERSION VERSION_LESS 3.3.1))
-  # Detect Fortran compiler version directly
+#-----------------------------------------------------------------
+# Set CMAKE_Fortran_COMPILER_VERSION if CMake doesn't do it for us
+#-----------------------------------------------------------------
+if (  NOT CMAKE_Fortran_COMPILER_VERSION )
+  if ( NOT (CMAKE_VERSION VERSION_LESS 3.3.1) )
+    message( AUTHOR_WARNING
+     "CMake ${CMAKE_VERSION} should know about Fortran compiler versions but is missing CMAKE_Fortran_COMPILER_VERSION variable."
+    )
+  endif()
+  # No CMAKE_Fortran_COMPILER_VERSION set, build our own
+  # Try extracting it directly from ISO_FORTRAN_ENV's compiler_version
+  # Write program for introspection
+  file( WRITE "${CMAKE_BINARY_DIR}/get_compiler_ver.f90"
+    "program main
+     use iso_fortran_env, only: compiler_version, output_unit
+     write(output_unit,'(a)') compiler_version()
+end program"
+  )
+  try_run( PROG_RAN COMPILE_SUCCESS
+    "${CMAKE_BINARY_DIR}" "${CMAKE_BINARY_DIR}/get_compiler_ver.f90"
+    RUN_OUTPUT_VARIABLE VER_STRING
+  )
+  if ( COMPILE_SUCCESS )
+    string( REGEX MATCH "[0-9]+\\.[0-9]+(\\.[0-9]+)?"
+      DETECTED_VER "${VER_STRING}"
+    )
+    message( STATUS "Detected Fortran compiler as ${VER_STRING}" )
+    message( STATUS "Extracted version number: ${DETECTED_VER}" )
+  endif()
+  if( ( NOT COMPILE_SUCCESS ) OR ( NOT DETECTED_VER ) )
+    message( WARNING "Could not reliably detect Fortran compiler version. We'll infer it from
+the C compiler if it matches the Fortran compiler ID." )
+  endif()
+  if( "${CMAKE_C_COMPILER_ID}" MATCHES "${CMAKE_Fortran_COMPILER_ID}" )
+    set( DETECTED_VER "${CMAKE_C_COMPILER_VERSION}" )
+  else()
+    message( FATAL_ERROR "Exhausted all possible means of detecting the Fortran compiler version, cannot proceed!" )
+  endif()
+  set( CMAKE_Fortran_COMPILER_VERSION "${DETECTED_VER}" )
+endif()
+  
+  # We have populated CMAKE_Fortran_COMPILER_VERSION if it was missing
   if(gfortran_compiler AND (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER 5.0.0))
     set(opencoarrays_aware_compiler true)
     add_definitions(-DPREFIX_NAME=_gfortran_caf_)
@@ -65,27 +110,9 @@ if (NOT (CMAKE_VERSION VERSION_LESS 3.3.1))
       CACHE STRING "Flags used by the compiler during release builds with debug info" FORCE)
     set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0")
   endif()
-else()
-  # Use the C compiler version as a proxy for the Fortran compiler version (won't work with NAG)
-  if(gfortran_compiler AND (CMAKE_C_COMPILER_VERSION VERSION_GREATER 5.0.0))
-    set(opencoarrays_aware_compiler true)
-    add_definitions(-DPREFIX_NAME=_gfortran_caf_)
-  else()
-    set(opencoarrays_aware_compiler false)
-    add_definitions(-DPREFIX_NAME=_caf_extensions_)
+  if ( gfortran_compiler AND ( NOT CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 7.0.0 ) )
+    add_definitions(-DGCC_GE_7) # Tell library to build against GFortran 7.x bindings b/c we might be using clang for C
   endif()
-  if(gfortran_compiler AND (CMAKE_C_COMPILER_VERSION VERSION_LESS 5.4))
-    # GCC patch to fix issue accepted for the 5.4 release
-    # See https://github.com/sourceryinstitute/opencoarrays/issues/28 and
-    # https://groups.google.com/forum/#!msg/opencoarrays/RZOwwYTqG80/46S9eL696dgJ
-    message( STATUS "Disabling optimization flags due to GCC < 5.4 bug")
-    set(CMAKE_Fortran_FLAGS_RELEASE -O0
-      CACHE STRING "Flags used by the compiler during release builds." FORCE)
-    set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-g -DNDEBUG -O0"
-      CACHE STRING "Flags used by the compiler during release builds with debug info" FORCE)
-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0")
-  endif()
-endif()
 
 if(gfortran_compiler)
   set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
@@ -119,10 +146,10 @@ get_filename_component( FTN_COMPILER_DIR "${CMAKE_Fortran_COMPILER}"
 get_filename_component( C_COMPILER_DIR "${CMAKE_C_COMPILER}"
   REALPATH )
 
-if (FTN_COMPILER_NAME MATCHES '^[mM][pP][iI]')
+if (FTN_COMPILER_NAME MATCHES "^[mM][pP][iI]")
   set (MPI_Fortran_COMPILER "${CMAKE_Fortran_COMPILER}")
 endif()
-if (C_COMPILER_NAME MATCHES '^[mM][pP][iI]')
+if (C_COMPILER_NAME MATCHES "^[mM][pP][iI]")
   set (MPI_C_COMPILER "${CMAKE_C_COMPILER}")
 endif()
 
@@ -271,6 +298,11 @@ set(CMAKE_Fortran_COMPILE_FLAGS ${CMAKE_Fortran_COMPILE_FLAGS} ${MPI_Fortran_COM
 set(CMAKE_Fortran_LINK_FLAGS ${CMAKE_Fortran_LINK_FLAGS} ${MPI_Fortran_LINK_FLAGS})
 include_directories(BEFORE ${MPI_C_INCLUDE_PATH} ${MPI_Fortran_INCLUDE_PATH})
 
+#---------------------------------------------------
+# Use standardized GNU install directory conventions
+#---------------------------------------------------
+include(GNUInstallDirs)
+
 #-------------------------------
 # Recurse into the src directory
 #-------------------------------
@@ -285,7 +317,7 @@ install(EXPORT OpenCoarraysTargets
   NAMESPACE
     OpenCoarrays::
   DESTINATION
-    lib/cmake/opencoarrays
+    "${CMAKE_INSTALL_LIBDIR}/cmake/opencoarrays"
 )
 include(CMakePackageConfigHelpers) # standard CMake module
 write_basic_package_version_file(
@@ -301,15 +333,13 @@ install(
     "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/OpenCoarraysConfig.cmake"
     "${CMAKE_CURRENT_BINARY_DIR}/OpenCoarraysConfigVersion.cmake"
   DESTINATION
-    lib/cmake/opencoarrays
+    "${CMAKE_INSTALL_LIBDIR}/cmake/opencoarrays"
 )
 
 add_library(OpenCoarrays INTERFACE)
 target_compile_options(OpenCoarrays INTERFACE -fcoarray=lib)
 target_link_libraries(OpenCoarrays INTERFACE caf_mpi)
 
-install(DIRECTORY  ${CMAKE_BINARY_DIR}/mod DESTINATION .)
-
 #------------------------------------------
 # Add portable unistall command to makefile
 #------------------------------------------
@@ -390,6 +420,7 @@ if(opencoarrays_aware_compiler)
   add_mpi_test(syncimages2 32 ${tests_root}/unit/sync/syncimages2)
   add_mpi_test(duplicate_syncimages 8 ${tests_root}/unit/sync/duplicate_syncimages)
   add_mpi_test(co_reduce 4 ${tests_root}/unit/collectives/co_reduce_test)
+  add_mpi_test(co_reduce_res_im 4 ${tests_root}/unit/collectives/co_reduce_res_im)
   add_mpi_test(syncimages_status 32 ${tests_root}/unit/sync/syncimages_status)
   add_mpi_test(sync_ring_abort_np3 3 ${tests_root}/unit/sync/sync_image_ring_abort_on_stopped_image)
   add_mpi_test(sync_ring_abort_np7 7 ${tests_root}/unit/sync/sync_image_ring_abort_on_stopped_image)
@@ -402,16 +433,17 @@ if(opencoarrays_aware_compiler)
   add_mpi_test(coarray_burgers_pde 2 ${tests_root}/integration/pde_solvers/coarrayBurgers/coarray_burgers_pde)
   add_mpi_test(co_heat 2 ${tests_root}/integration/pde_solvers/coarrayHeatSimplified/co_heat)
   add_mpi_test(asynchronous_hello_world 3 ${tests_root}/integration/events/asynchronous_hello_world)
-  if ( ("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "x86_64") AND ("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") )
-    if ( (NOT (DEFINED ENV{TRAVIS})) AND (NOT SKIP_ASSEMBLY_DEPS) )
-      add_mpi_test(coarray_navier_stokes 2 ${tests_root}/integration/pde_solvers/navier-stokes/coarray_navier_stokes)
-      set_property(TEST coarray_navier_stokes PROPERTY PASS_REGULAR_EXPRESSION "Test passed.")
-    endif()
-  endif()
 
   # Regression tests based on reported issues
+  if(gfortran_compiler AND (NOT CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 7.0.0))
+    # GFortran PR 78505 only fixed on trunk/gcc 7
+    add_mpi_test(source-alloc-no-sync 8 ${tests_root}/regression/reported/source-alloc-sync)
+  endif()
   add_mpi_test(convert-before-put 3 ${tests_root}/regression/reported/convert-before-put)
   add_mpi_test(event-post 3 ${tests_root}/regression/reported/event-post)
+  add_mpi_test(co_reduce-factorial 4 ${tests_root}/regression/reported/co_reduce-factorial)
+  # remove this before merging into master
+  set_property(TEST co_reduce-factorial PROPERTY WILL_FAIL TRUE)
 else()
   add_test(co_sum_extension ${tests_root}/unit/extensions/test-co_sum-extension.sh)
   set_property(TEST co_sum_extension PROPERTY PASS_REGULAR_EXPRESSION "Test passed.")