ARROW-17545: [C++][CI] Mandate C++17 instead of C++11 (#13991)

This PR switches our build system to require C++17 instead of C++11.

Because the conda packaging jobs are out of sync with the conda-forge files, the Windows conda packaging jobs are broken with this change. The related task (sync conda packaging files with conda-forge) is tracked in ARROW-17635.


Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>

Author: pitrou

.env

@@ -86,8 +86,8 @@ ARROW_R_DEV=TRUE
 R_PRUNE_DEPS=FALSE
 TZ=UTC
 
-# -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
-DEVTOOLSET_VERSION=-1
+# Any non-empty string will install devtoolset-${DEVTOOLSET_VERSION}
+DEVTOOLSET_VERSION=
 
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the

.github/workflows/r.yml

@@ -119,12 +119,13 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - { org: "rstudio", image: "r-base", tag: "4.0-centos7" }
-          - { org: "rhub", image: "debian-gcc-devel", tag: "latest" }
+          - { org: "rstudio", image: "r-base", tag: "4.0-centos7", devtoolset: "8" }
+          - { org: "rhub", image: "debian-gcc-devel", tag: "latest", devtoolset: "" }
     env:
       R_ORG: ${{ matrix.config.org }}
       R_IMAGE: ${{ matrix.config.image }}
       R_TAG: ${{ matrix.config.tag }}
+      DEVTOOLSET_VERSION: ${{ matrix.config.devtoolset }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v3
@@ -184,11 +185,6 @@ jobs:
         run: |
           ci/scripts/ccache_setup.sh
           echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV
-      # We must enable actions/cache before r-lib/actions/setup-r to ensure
-      # using system tar instead of tar provided by Rtools.
-      # We can use tar provided by Rtools when we drop support for Rtools 3.5.
-      # Because Rtools 4.0 or later has zstd. actions/cache requires zstd
-      # when tar is GNU tar.
       - name: Cache ccache
         uses: actions/cache@v2
         with:
@@ -268,8 +264,6 @@ jobs:
         with:
           r-version: ${{ matrix.config.rversion }}
           rtools-version: ${{ matrix.config.rtools }}
-          # RSPM keeps install times short for 3.6
-          use-public-rspm: true
           Ncpus: 2
       - uses: r-lib/actions/setup-r-dependencies@v2
         env:

c_glib/meson.build

@@ -21,7 +21,7 @@ project('arrow-glib', 'c', 'cpp',
         license: 'Apache-2.0',
         default_options: [
           'c_std=c99',
-          'cpp_std=c++11',
+          'cpp_std=c++17',
         ])
 
 version = '10.0.0-SNAPSHOT'

ci/conan/all/test_package/CMakeLists.txt

@@ -30,5 +30,5 @@ find_package(Arrow REQUIRED)
 
 add_executable(${PROJECT_NAME} test_package.cpp)
 target_link_libraries(${PROJECT_NAME} arrow::arrow)
-target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_11)
+target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17)
 target_compile_definitions(${PROJECT_NAME} PRIVATE WITH_JEMALLOC)

ci/docker/linux-r.dockerfile

@@ -27,7 +27,7 @@ ENV R_BIN=${r_bin}
 ARG r_dev=FALSE
 ENV ARROW_R_DEV=${r_dev}
 
-ARG devtoolset_version=-1
+ARG devtoolset_version=
 ENV DEVTOOLSET_VERSION=${devtoolset_version}
 
 ARG r_prune_deps=FALSE

ci/scripts/cpp_build.sh

@@ -142,7 +142,7 @@ cmake \
   -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE:-OFF} \
   -DCMAKE_C_FLAGS="${CFLAGS:-}" \
   -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \
-  -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-11}" \
+  -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-17}" \
   -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \
   -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
   -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \

ci/scripts/r_docker_configure.sh

@@ -57,7 +57,7 @@ if [ ${R_CUSTOM_CCACHE} = "true" ]; then
 CCACHE=ccache
 CC=\$(CCACHE) gcc\$(VER)
 CXX=\$(CCACHE) g++\$(VER)
-CXX11=\$(CCACHE) g++\$(VER)" >> ~/.R/Makevars
+CXX17=\$(CCACHE) g++\$(VER)" >> ~/.R/Makevars
 
   mkdir -p ~/.ccache/
   echo "max_size = 5.0G
@@ -69,7 +69,7 @@ fi
 
 # Special hacking to try to reproduce quirks on centos using non-default build
 # tooling.
-if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then
+if [[ -n "$DEVTOOLSET_VERSION" ]]; then
   $PACKAGE_MANAGER install -y centos-release-scl
   $PACKAGE_MANAGER install -y "devtoolset-$DEVTOOLSET_VERSION"
 fi

ci/scripts/r_test.sh

@@ -26,8 +26,21 @@ pushd ${source_dir}
 
 printenv
 
+if [[ -n "$DEVTOOLSET_VERSION" ]]; then
+  # enable the devtoolset version to use it
+  source /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable
+
+  # Build images which require the devtoolset don't have CXX17 variables
+  # set as the system compiler doesn't support C++17
+  mkdir -p ~/.R
+  echo "CC = $(which gcc) -fPIC" >> ~/.R/Makevars
+  echo "CXX17 = $(which g++) -fPIC" >> ~/.R/Makevars
+  echo "CXX17STD = -std=c++17" >> ~/.R/Makevars
+  echo "CXX17FLAGS = ${CXX11FLAGS}" >> ~/.R/Makevars
+fi
+
 # Run the nixlibs.R test suite, which is not included in the installed package
- ${R_BIN} -e 'setwd("tools"); testthat::test_dir(".")'
+${R_BIN} -e 'setwd("tools"); testthat::test_dir(".")'
 
 # Before release, we always copy the relevant parts of the cpp source into the
 # package. In some CI checks, we will use this version of the source:
@@ -77,11 +90,6 @@ export ARROW_DEBUG_MEMORY_POOL=trap
 export TEXMFCONFIG=/tmp/texmf-config
 export TEXMFVAR=/tmp/texmf-var
 
-if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then
-  # enable the devtoolset version to use it
-  source /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable
-fi
-
 # Make sure we aren't writing to the home dir (CRAN _hates_ this but there is no official check)
 BEFORE=$(ls -alh ~/)
 

cpp/CMakeLists.txt

@@ -561,10 +561,10 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}")
 # C++ specific flags.
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CXX_COMMON_FLAGS} ${ARROW_CXXFLAGS}")
 
-# Remove --std=c++11 to avoid errors from C compilers
-string(REPLACE "-std=c++11" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
+# Remove --std=c++17 to avoid errors from C compilers
+string(REPLACE "-std=c++17" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
 
-# Add C++-only flags, like -std=c++11
+# Add C++-only flags, like -std=c++17
 set(CMAKE_CXX_FLAGS "${CXX_ONLY_FLAGS} ${CMAKE_CXX_FLAGS}")
 
 # ASAN / TSAN / UBSAN

cpp/build-support/update-flatbuffers.sh

@@ -33,9 +33,6 @@ OUT_DIR="$SOURCE_DIR/generated"
 FILES=($(find $FORMAT_DIR -name '*.fbs'))
 FILES+=("$SOURCE_DIR/arrow/ipc/feather.fbs")
 
-# add compute ir files
-FILES+=($(find "$TOP/experimental/computeir" -name '*.fbs'))
-
 $FLATC --cpp --cpp-std c++11 \
   --scoped-enums \
   -o "$OUT_DIR" \