Merge remote-tracking branch 'apache/main' into fix/gh-41011

Author: zanmato1984

.github/workflows/dev.yml

@@ -105,7 +105,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit
-          pip install "cython>=3" setuptools pytest requests setuptools-scm
+          pip install "cython>=3.1" setuptools pytest requests setuptools-scm
       - name: Run Release Test
         shell: bash
         run: |

ci/conda_env_python.txt

@@ -20,7 +20,7 @@
 # Not a direct dependency of s3fs, but needed for our s3fs fixture
 boto3
 cffi
-cython>=3
+cython>=3.1
 cloudpickle
 fsspec
 hypothesis

ci/docker/linux-apt-python-313-freethreading.dockerfile

@@ -27,7 +27,7 @@ RUN apt-get update -y -q && \
     rm -rf /var/lib/apt/lists*
 
 COPY python/requirements-build.txt \
-     python/requirements-test.txt \
+     python/requirements-test-3.13t.txt \
      /arrow/python/
 
 ENV ARROW_PYTHON_VENV /arrow-dev
@@ -38,7 +38,7 @@ RUN ${ARROW_PYTHON_VENV}/bin/python -m pip install \
       --prefer-binary \
       --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \
       -r arrow/python/requirements-build.txt \
-      -r arrow/python/requirements-test.txt
+      -r arrow/python/requirements-test-3.13t.txt
 
 # We want to run the PyArrow test suite with the GIL disabled, but cffi
 # (more precisely, the `_cffi_backend` module) currently doesn't declare

ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile

@@ -37,16 +37,13 @@ ENV PYTHON_CMD="py -${python}t"
 SHELL ["cmd", "/S", "/C"]
 RUN %PYTHON_CMD% -m pip install -U pip setuptools
 
-COPY python/requirements-wheel-test.txt C:/arrow/python/
+COPY python/requirements-wheel-test-3.13t.txt C:/arrow/python/
 # Cython and Pandas wheels for 3.13 free-threaded are not released yet
 RUN %PYTHON_CMD% -m pip install \
     --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
     --pre \
     --prefer-binary \
-    -r C:/arrow/python/requirements-wheel-test.txt
-# cffi-based tests would crash when importing cffi.
-# hadolint ignore=DL3059
-RUN %PYTHON_CMD% -m pip uninstall -y cffi
+    -r C:/arrow/python/requirements-wheel-test-3.13t.txt
 
 ENV PYTHON="${python}t"
 ENV PYTHON_GIL=0

ci/docker/python-wheel-manylinux-test.dockerfile

@@ -28,7 +28,8 @@ RUN pip install -r /arrow/python/requirements-wheel-test.txt
 RUN apt-get update -y -q && \
     apt-get install -y -q \
         build-essential \
-        python3-dev && \
+        python3-dev \
+        tzdata-legacy && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
 

cpp/CMakeLists.txt

@@ -177,6 +177,7 @@ set(ARROW_DOC_DIR "share/doc/${PROJECT_NAME}")
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 
 set(ARROW_LLVM_VERSIONS
+    "21.1"
     "20.1"
     "19.1"
     "18.1"

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -1723,6 +1723,8 @@ if(ARROW_NEED_GFLAGS)
       set(GFLAGS_LIBRARIES gflags-shared)
     elseif(TARGET gflags_shared)
       set(GFLAGS_LIBRARIES gflags_shared)
+    elseif(TARGET gflags::gflags)
+      set(GFLAGS_LIBRARIES gflags::gflags)
     endif()
   endif()
 endif()

cpp/src/arrow/compare.h

@@ -83,6 +83,38 @@ class EqualOptions {
     return res;
   }
 
+  /// Whether the \ref arrow::Schema property is used in the comparison.
+  ///
+  /// This option only affects the Equals methods
+  /// and has no effect on ApproxEquals methods.
+  bool use_schema() const { return use_schema_; }
+
+  /// Return a new EqualOptions object with the "use_schema_" property changed.
+  ///
+  /// Setting this option is false making the value of \ref EqualOptions::use_metadata
+  /// is ignored.
+  EqualOptions use_schema(bool v) const {
+    auto res = EqualOptions(*this);
+    res.use_schema_ = v;
+    return res;
+  }
+
+  /// Whether the "metadata" in \ref arrow::Schema is used in the comparison.
+  ///
+  /// This option only affects the Equals methods
+  /// and has no effect on the ApproxEquals methods.
+  ///
+  /// Note: This option is only considered when \ref arrow::EqualOptions::use_schema is
+  /// set to true.
+  bool use_metadata() const { return use_metadata_; }
+
+  /// Return a new EqualOptions object with the "use_metadata" property changed.
+  EqualOptions use_metadata(bool v) const {
+    auto res = EqualOptions(*this);
+    res.use_metadata_ = v;
+    return res;
+  }
+
   /// The ostream to which a diff will be formatted if arrays disagree.
   /// If this is null (the default) no diff will be formatted.
   std::ostream* diff_sink() const { return diff_sink_; }
@@ -103,6 +135,8 @@ class EqualOptions {
   bool nans_equal_ = false;
   bool signed_zeros_equal_ = true;
   bool use_atol_ = false;
+  bool use_schema_ = true;
+  bool use_metadata_ = false;
 
   std::ostream* diff_sink_ = NULLPTR;
 };

cpp/src/arrow/compute/expression.cc

@@ -545,67 +545,61 @@ Result<Expression> BindNonRecursive(Expression::Call call, bool insert_implicit_
   std::vector<TypeHolder> types = GetTypes(call.arguments);
   ARROW_ASSIGN_OR_RAISE(call.function, GetFunction(call, exec_context));
 
-  auto FinishBind = [&] {
-    compute::KernelContext kernel_context(exec_context, call.kernel);
-    if (call.kernel->init) {
-      const FunctionOptions* options =
-          call.options ? call.options.get() : call.function->default_options();
-      ARROW_ASSIGN_OR_RAISE(
-          call.kernel_state,
-          call.kernel->init(&kernel_context, {call.kernel, types, options}));
-
-      kernel_context.SetState(call.kernel_state.get());
-    }
-
-    ARROW_ASSIGN_OR_RAISE(
-        call.type, call.kernel->signature->out_type().Resolve(&kernel_context, types));
-    return Status::OK();
-  };
-
   // First try and bind exactly
   Result<const Kernel*> maybe_exact_match = call.function->DispatchExact(types);
   if (maybe_exact_match.ok()) {
     call.kernel = *maybe_exact_match;
-    if (FinishBind().ok()) {
-      return Expression(std::move(call));
+  } else {
+    if (!insert_implicit_casts) {
+      return maybe_exact_match.status();
     }
-  }
 
-  if (!insert_implicit_casts) {
-    return maybe_exact_match.status();
-  }
+    // If exact binding fails, and we are allowed to cast, then prefer casting literals
+    // first.  Since DispatchBest generally prefers up-casting the best way to do this is
+    // first down-cast the literals as much as possible
+    types = GetTypesWithSmallestLiteralRepresentation(call.arguments);
+    ARROW_ASSIGN_OR_RAISE(call.kernel, call.function->DispatchBest(&types));
 
-  // If exact binding fails, and we are allowed to cast, then prefer casting literals
-  // first.  Since DispatchBest generally prefers up-casting the best way to do this is
-  // first down-cast the literals as much as possible
-  types = GetTypesWithSmallestLiteralRepresentation(call.arguments);
-  ARROW_ASSIGN_OR_RAISE(call.kernel, call.function->DispatchBest(&types));
+    for (size_t i = 0; i < types.size(); ++i) {
+      if (types[i] == call.arguments[i].type()) continue;
 
-  for (size_t i = 0; i < types.size(); ++i) {
-    if (types[i] == call.arguments[i].type()) continue;
+      if (const Datum* lit = call.arguments[i].literal()) {
+        ARROW_ASSIGN_OR_RAISE(Datum new_lit,
+                              compute::Cast(*lit, types[i].GetSharedPtr()));
+        call.arguments[i] = literal(std::move(new_lit));
+        continue;
+      }
 
-    if (const Datum* lit = call.arguments[i].literal()) {
-      ARROW_ASSIGN_OR_RAISE(Datum new_lit, compute::Cast(*lit, types[i].GetSharedPtr()));
-      call.arguments[i] = literal(std::move(new_lit));
-      continue;
-    }
+      // construct an implicit cast Expression with which to replace this argument
+      Expression::Call implicit_cast;
+      implicit_cast.function_name = "cast";
+      implicit_cast.arguments = {std::move(call.arguments[i])};
 
-    // construct an implicit cast Expression with which to replace this argument
-    Expression::Call implicit_cast;
-    implicit_cast.function_name = "cast";
-    implicit_cast.arguments = {std::move(call.arguments[i])};
+      // TODO(wesm): Use TypeHolder in options
+      implicit_cast.options = std::make_shared<compute::CastOptions>(
+          compute::CastOptions::Safe(types[i].GetSharedPtr()));
 
-    // TODO(wesm): Use TypeHolder in options
-    implicit_cast.options = std::make_shared<compute::CastOptions>(
-        compute::CastOptions::Safe(types[i].GetSharedPtr()));
+      ARROW_ASSIGN_OR_RAISE(
+          call.arguments[i],
+          BindNonRecursive(std::move(implicit_cast),
+                           /*insert_implicit_casts=*/false, exec_context));
+    }
+  }
 
+  compute::KernelContext kernel_context(exec_context, call.kernel);
+  if (call.kernel->init) {
+    const FunctionOptions* options =
+        call.options ? call.options.get() : call.function->default_options();
     ARROW_ASSIGN_OR_RAISE(
-        call.arguments[i],
-        BindNonRecursive(std::move(implicit_cast),
-                         /*insert_implicit_casts=*/false, exec_context));
+        call.kernel_state,
+        call.kernel->init(&kernel_context, {call.kernel, types, options}));
+
+    kernel_context.SetState(call.kernel_state.get());
   }
 
-  RETURN_NOT_OK(FinishBind());
+  ARROW_ASSIGN_OR_RAISE(
+      call.type, call.kernel->signature->out_type().Resolve(&kernel_context, types));
+
   return Expression(std::move(call));
 }