WIP: Dummy PR to check maint-18.1.0 status

c_glib/meson.build

@@ -35,7 +35,7 @@ project('arrow-glib', 'c', 'cpp',
         #   * 22.04: 0.61.2
         meson_version: '>=0.53.2')
 
-version = '18.0.0-SNAPSHOT'
+version = '18.0.0'
 if version.endswith('-SNAPSHOT')
   version_numbers = version.split('-')[0].split('.')
   version_tag = version.split('-')[1]

c_glib/vcpkg.json

@@ -1,6 +1,6 @@
 {
   "name": "arrow-glib",
-  "version-string": "18.0.0-SNAPSHOT",
+  "version-string": "18.0.0",
   "dependencies": [
     "glib",
     "gobject-introspection",

ci/appveyor-cpp-build.bat

@@ -139,7 +139,7 @@ set PARQUET_HOME=%CONDA_PREFIX%\Library
 
 @rem Download IANA Timezone Database to a non-standard location to
 @rem test the configurability of the timezone database path
-curl https://data.iana.org/time-zones/releases/tzdata2021e.tar.gz --output tzdata.tar.gz || exit /B
+curl https://data.iana.org/time-zones/releases/tzdata2024b.tar.gz --output tzdata.tar.gz || exit /B
 mkdir %USERPROFILE%\Downloads\test\tzdata
 tar --extract --file tzdata.tar.gz --directory %USERPROFILE%\Downloads\test\tzdata
 curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml ^

ci/docker/conda-python.dockerfile

@@ -28,7 +28,7 @@ COPY ci/conda_env_python.txt \
 RUN mamba install -q -y \
         --file arrow/ci/conda_env_python.txt \
         $([ "$python" == $(gdb --batch --eval-command 'python import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') ] && echo "gdb") \
-        "python=${python}.*=*_cpython" \
+        "python=${python}.*=*_cp*" \
         nomkl && \
     mamba clean --all
 

ci/docker/python-free-threaded-wheel-manylinux-test-imports.dockerfile

@@ -18,6 +18,8 @@
 ARG base
 FROM ${base}
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update -y -q && \
     apt install -y -q --no-install-recommends software-properties-common gpg-agent && \
     add-apt-repository -y ppa:deadsnakes/ppa && \

ci/docker/python-wheel-manylinux.dockerfile

@@ -114,5 +114,10 @@ RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-${PYTHON_ABI_TAG
 SHELL ["/bin/bash", "-i", "-c"]
 ENTRYPOINT ["/bin/bash", "-i", "-c"]
 
+# Remove once there are released Cython wheels for 3.13 free-threaded available
+RUN if [ "${python_abi_tag}" = "cp313t" ]; then \
+      pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary ; \
+    fi
+
 COPY python/requirements-wheel-build.txt /arrow/python/
 RUN pip install -r /arrow/python/requirements-wheel-build.txt

ci/scripts/PKGBUILD

@@ -18,7 +18,7 @@
 _realname=arrow
 pkgbase=mingw-w64-${_realname}
 pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}"
-pkgver=17.0.0.9000
+pkgver=18.0.0
 pkgrel=8000
 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)"
 arch=("any")

ci/scripts/download_tz_database.sh

@@ -20,11 +20,11 @@
 set -ex
 
 # Download database
-curl https://data.iana.org/time-zones/releases/tzdata2021e.tar.gz --output ~/Downloads/tzdata2021e.tar.gz
+curl https://data.iana.org/time-zones/releases/tzdata2024b.tar.gz --output ~/Downloads/tzdata.tar.gz
 
 # Extract
 mkdir -p ~/Downloads/tzdata
-tar --extract --file ~/Downloads/tzdata2021e.tar.gz --directory ~/Downloads/tzdata
+tar --extract --file ~/Downloads/tzdata.tar.gz --directory ~/Downloads/tzdata
 
 # Download Windows timezone mapping
 curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml --output ~/Downloads/tzdata/windowsZones.xml

ci/scripts/python_wheel_macos_build.sh

@@ -49,8 +49,15 @@ fi
 echo "=== (${PYTHON_VERSION}) Install Python build dependencies ==="
 export PIP_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])')
 
+# Remove once there are released Cython wheels for 3.13 free-threaded available
+FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")"
+if [[ $FREE_THREADED_BUILD == "True"  ]]; then
+  pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary
+fi
+# With Python 3.9, the `--upgrade` flag is required to force full replacement of setuptools' distutils patching
+pip install --upgrade --target $PIP_SITE_PACKAGES "setuptools>=58"
+
 pip install \
-  --upgrade \
   --only-binary=:all: \
   --target $PIP_SITE_PACKAGES \
   -r ${source_dir}/python/requirements-wheel-build.txt

ci/scripts/python_wheel_manylinux_build.sh

@@ -55,7 +55,6 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_GANDIVA:=OFF}
 : ${ARROW_GCS:=ON}
 : ${ARROW_HDFS:=ON}
-: ${ARROW_JEMALLOC:=ON}
 : ${ARROW_MIMALLOC:=ON}
 : ${ARROW_ORC:=ON}
 : ${ARROW_PARQUET:=ON}
@@ -81,6 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
     # 4k and 64k page arm64 systems. For more context see
     # https://github.com/apache/arrow/issues/10929
     export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
+    : ${ARROW_JEMALLOC:=OFF}
+else
+    : ${ARROW_JEMALLOC:=ON}
 fi
 
 mkdir /tmp/arrow-build

cpp/CMakeLists.txt

@@ -79,7 +79,7 @@ if(POLICY CMP0170)
   cmake_policy(SET CMP0170 NEW)
 endif()
 
-set(ARROW_VERSION "18.0.0-SNAPSHOT")
+set(ARROW_VERSION "18.0.0")
 
 string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}")
 

cpp/cmake_modules/DefineOptions.cmake

@@ -364,14 +364,18 @@ takes precedence over ccache if a storage backend is configured" ON)
 
   set(ARROW_JEMALLOC_DESCRIPTION "Build the Arrow jemalloc-based allocator")
   if(WIN32
-     OR "${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD"
+     OR CMAKE_SYSTEM_NAME STREQUAL "FreeBSD"
+     OR CMAKE_SYSTEM_PROCESSOR MATCHES "aarch|ARM|arm"
      OR NOT ARROW_ENABLE_THREADING)
     # jemalloc is not supported on Windows.
     #
     # jemalloc is the default malloc implementation on FreeBSD and can't
     # be built with --disable-libdl on FreeBSD. Because lazy-lock feature
     # is required on FreeBSD. Lazy-lock feature requires libdl.
     #
+    # jemalloc may have a problem on ARM.
+    # See also: https://github.com/apache/arrow/issues/44342
+    #
     # jemalloc requires thread.
     define_option(ARROW_JEMALLOC ${ARROW_JEMALLOC_DESCRIPTION} OFF)
   else()
@@ -636,6 +640,11 @@ Always OFF if building binaries" OFF)
                        "Compiler flags to append when pre-compiling Gandiva operations"
                        "")
 
+  #----------------------------------------------------------------------
+  set_option_category("Cross compiling")
+
+  define_option_string(ARROW_GRPC_CPP_PLUGIN "grpc_cpp_plugin path to be used" "")
+
   #----------------------------------------------------------------------
   set_option_category("Advanced developer")
 

cpp/cmake_modules/FindabslAlt.cmake

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if(abslAlt_FOUND)
+  return()
+endif()
+
+set(find_package_args)
+
+if(abslAlt_FIND_QUIETLY)
+  list(APPEND find_package_args QUIET)
+endif()
+if(abslAlt_FIND_REQUIRED)
+  list(APPEND find_package_args REQUIRED)
+endif()
+
+find_package(absl ${find_package_args})
+
+if(NOT DEFINED absl_VERSION)
+  # Abseil does not define a version when build 'live at head'.
+  # As this is their recommended path we need to define a large version to pass version checks.
+  # CMake removes the '_head' suffix for version comparison but it will show up in the logs
+  # and matches the abseil-cpp.pc version of 'head'
+  set(absl_VERSION 99999999_head)
+endif()
+
+set(abslAlt_VERSION ${absl_VERSION})
+
+find_package_handle_standard_args(
+  abslAlt
+  REQUIRED_VARS absl_FOUND
+  VERSION_VAR abslAlt_VERSION)

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -816,21 +816,9 @@ endif()
 if(DEFINED ENV{ARROW_THRIFT_URL})
   set(THRIFT_SOURCE_URL "$ENV{ARROW_THRIFT_URL}")
 else()
-  set_urls(THRIFT_SOURCE_URL
-           "https://www.apache.org/dyn/closer.cgi?action=download&filename=/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://downloads.apache.org/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://apache.claz.org/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://apache.cs.utah.edu/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://apache.mirrors.lucidnetworks.net/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://apache.osuosl.org/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://ftp.wayne.edu/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://mirror.olnevhost.net/pub/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://mirrors.gigenet.com/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://mirrors.koehn.com/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://mirrors.ocf.berkeley.edu/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://mirrors.sonic.net/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "https://us.mirrors.quenda.co/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
-           "${THIRDPARTY_MIRROR_URL}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz")
+  set(THRIFT_SOURCE_URL
+      "https://dlcdn.apache.org/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+  )
 endif()
 
 if(DEFINED ENV{ARROW_UCX_URL})
@@ -1775,6 +1763,7 @@ macro(build_thrift)
       set(THRIFT_LIB_SUFFIX "md")
       list(APPEND THRIFT_CMAKE_ARGS "-DWITH_MT=OFF")
     endif()
+    # NOTE(amoeba): When you bump Thrift to >=0.21.0, change bin to lib
     set(THRIFT_LIB
         "${THRIFT_PREFIX}/bin/${CMAKE_IMPORT_LIBRARY_PREFIX}thrift${THRIFT_LIB_SUFFIX}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
     )
@@ -4169,7 +4158,7 @@ if(ARROW_WITH_GOOGLE_CLOUD_CPP OR ARROW_WITH_GRPC)
                      ARROW_PC_PACKAGE_NAME
                      ${ARROW_ABSL_PC_PACKAGE_NAME}
                      HAVE_ALT
-                     FALSE
+                     TRUE
                      FORCE_ANY_NEWER_VERSION
                      TRUE
                      REQUIRED_VERSION
@@ -4223,6 +4212,14 @@ if(ARROW_WITH_GRPC)
       target_link_libraries(gRPC::grpc++ INTERFACE gRPC::grpc_asan_suppressed)
     endif()
   endif()
+
+  if(ARROW_GRPC_CPP_PLUGIN)
+    if(NOT TARGET gRPC::grpc_cpp_plugin)
+      add_executable(gRPC::grpc_cpp_plugin IMPORTED)
+    endif()
+    set_target_properties(gRPC::grpc_cpp_plugin PROPERTIES IMPORTED_LOCATION
+                                                           ${ARROW_GRPC_CPP_PLUGIN})
+  endif()
 endif()
 
 # ----------------------------------------------------------------------

cpp/src/arrow/CMakeLists.txt

@@ -460,7 +460,7 @@ endif()
 
 set(ARROW_VENDORED_SRCS
     vendored/base64.cpp
-    vendored/datetime/tz.cpp
+    vendored/datetime.cpp
     vendored/double-conversion/bignum-dtoa.cc
     vendored/double-conversion/bignum.cc
     vendored/double-conversion/cached-powers.cc
@@ -488,7 +488,7 @@ set(ARROW_VENDORED_SRCS
 if(APPLE)
   list(APPEND ARROW_VENDORED_SRCS vendored/datetime/ios.mm)
 endif()
-set_source_files_properties(vendored/datetime/tz.cpp
+set_source_files_properties(vendored/datetime.cpp
                             PROPERTIES SKIP_PRECOMPILE_HEADERS ON
                                        SKIP_UNITY_BUILD_INCLUSION ON)
 arrow_add_object_library(ARROW_VENDORED ${ARROW_VENDORED_SRCS})

cpp/src/arrow/chunk_resolver.cc

@@ -26,7 +26,7 @@
 #include "arrow/array.h"
 #include "arrow/record_batch.h"
 
-namespace arrow::internal {
+namespace arrow {
 
 namespace {
 template <typename T>
@@ -167,4 +167,4 @@ void ChunkResolver::ResolveManyImpl(int64_t n_indices, const uint64_t* logical_i
                     logical_index_vec, out_chunk_location_vec, chunk_hint);
 }
 
-}  // namespace arrow::internal
+}  // namespace arrow

cpp/src/arrow/chunk_resolver.h

@@ -27,12 +27,12 @@
 #include "arrow/type_fwd.h"
 #include "arrow/util/macros.h"
 
-namespace arrow::internal {
+namespace arrow {
 
-struct ChunkResolver;
+class ChunkResolver;
 
 template <typename IndexType>
-struct TypedChunkLocation {
+struct ARROW_EXPORT TypedChunkLocation {
   /// \brief Index of the chunk in the array of chunks
   ///
   /// The value is always in the range `[0, chunks.size()]`. `chunks.size()` is used
@@ -41,7 +41,7 @@ struct TypedChunkLocation {
 
   /// \brief Index of the value in the chunk
   ///
-  /// The value is UNDEFINED if chunk_index >= chunks.size()
+  /// The value is UNDEFINED if `chunk_index >= chunks.size()`
   IndexType index_in_chunk = 0;
 
   TypedChunkLocation() = default;
@@ -61,7 +61,7 @@ using ChunkLocation = TypedChunkLocation<int64_t>;
 
 /// \brief An utility that incrementally resolves logical indices into
 /// physical indices in a chunked array.
-struct ARROW_EXPORT ChunkResolver {
+class ARROW_EXPORT ChunkResolver {
  private:
   /// \brief Array containing `chunks.size() + 1` offsets.
   ///
@@ -76,14 +76,11 @@ struct ARROW_EXPORT ChunkResolver {
 
  public:
   explicit ChunkResolver(const ArrayVector& chunks) noexcept;
+
   explicit ChunkResolver(const std::vector<const Array*>& chunks) noexcept;
+
   explicit ChunkResolver(const RecordBatchVector& batches) noexcept;
 
-  /// \brief Construct a ChunkResolver from a vector of chunks.size() + 1 offsets.
-  ///
-  /// The first offset must be 0 and the last offset must be the logical length of the
-  /// chunked array. Each offset before the last represents the starting logical index of
-  /// the corresponding chunk.
   explicit ChunkResolver(std::vector<int64_t> offsets) noexcept
       : offsets_(std::move(offsets)), cached_chunk_(0) {
 #ifndef NDEBUG
@@ -115,11 +112,11 @@ struct ARROW_EXPORT ChunkResolver {
   /// The returned ChunkLocation contains the chunk index and the within-chunk index
   /// equivalent to the logical index.
   ///
-  /// \pre index >= 0
-  /// \post location.chunk_index in [0, chunks.size()]
+  /// \pre `index >= 0`
+  /// \post `location.chunk_index` in `[0, chunks.size()]`
   /// \param index The logical index to resolve
   /// \return ChunkLocation with a valid chunk_index if index is within
-  ///         bounds, or with chunk_index == chunks.size() if logical index is
+  ///         bounds, or with `chunk_index == chunks.size()` if logical index is
   ///         `>= chunked_array.length()`.
   inline ChunkLocation Resolve(int64_t index) const {
     const auto cached_chunk = cached_chunk_.load(std::memory_order_relaxed);
@@ -133,13 +130,13 @@ struct ARROW_EXPORT ChunkResolver {
   /// The returned ChunkLocation contains the chunk index and the within-chunk index
   /// equivalent to the logical index.
   ///
-  /// \pre index >= 0
-  /// \post location.chunk_index in [0, chunks.size()]
+  /// \pre `index >= 0`
+  /// \post `location.chunk_index` in `[0, chunks.size()]`
   /// \param index The logical index to resolve
   /// \param hint ChunkLocation{} or the last ChunkLocation returned by
   ///             this ChunkResolver.
   /// \return ChunkLocation with a valid chunk_index if index is within
-  ///         bounds, or with chunk_index == chunks.size() if logical index is
+  ///         bounds, or with `chunk_index == chunks.size()` if logical index is
   ///         `>= chunked_array.length()`.
   inline ChunkLocation ResolveWithHint(int64_t index, ChunkLocation hint) const {
     assert(hint.chunk_index < static_cast<uint32_t>(offsets_.size()));
@@ -281,4 +278,13 @@ struct ARROW_EXPORT ChunkResolver {
   }
 };
 
-}  // namespace arrow::internal
+// Explicitly instantiate template base struct, for DLL linking on Windows
+template struct TypedChunkLocation<int32_t>;
+template struct TypedChunkLocation<int16_t>;
+template struct TypedChunkLocation<int8_t>;
+template struct TypedChunkLocation<uint8_t>;
+template struct TypedChunkLocation<uint16_t>;
+template struct TypedChunkLocation<uint32_t>;
+template struct TypedChunkLocation<int64_t>;
+template struct TypedChunkLocation<uint64_t>;
+}  // namespace arrow

cpp/src/arrow/chunk_resolver_benchmark.cc

@@ -28,9 +28,6 @@
 
 namespace arrow {
 
-using internal::ChunkResolver;
-using internal::TypedChunkLocation;
-
 namespace {
 
 int64_t constexpr kChunkedArrayLength = std::numeric_limits<uint16_t>::max();