Upgrade arrow 20 (#1)

Changes for arrow 20 upgrade

Author: scott-routledge2

recipe/meta.yaml

@@ -33,7 +33,7 @@ source:
     folder: cpp/submodules/parquet-testing
 
 build:
-  number: 5
+  number: 6
   # for cuda support, building with one version is enough to be compatible with
   # all later versions, since arrow is only using libcuda, and not libcudart.
   skip: true  # [cuda_compiler_version not in ("None", cuda_compiler_version_min)]
@@ -70,7 +70,7 @@ requirements:
     - clangdev {{ llvm_version }}
     - llvmdev {{ llvm_version }}
     - aws-crt-cpp
-    - aws-sdk-cpp <=1.11.485
+    - aws-sdk-cpp
     # azure filesystem dependencies, currently broken on windows, see
     # https://github.com/apache/arrow/issues/41990
     - azure-core-cpp                    # [unix]
@@ -223,7 +223,7 @@ outputs:
         - {{ compiler("cuda") }}                 # [cuda_compiler_version != "None"]
       host:
         - aws-crt-cpp
-        - aws-sdk-cpp <=1.11.485
+        - aws-sdk-cpp
         - azure-core-cpp                    # [unix]
         - azure-identity-cpp                # [unix]
         - azure-storage-blobs-cpp           # [unix]

recipe/patches/0004-Bodo-Changes.patch

@@ -1,9 +1,9 @@
 diff --git a/cpp/src/arrow/dataset/file_parquet.cc b/cpp/src/arrow/dataset/file_parquet.cc
-index 1f8b6cc488..6ac6642fdb 100644
+index e6ac389..60561d2 100644
 --- a/cpp/src/arrow/dataset/file_parquet.cc
 +++ b/cpp/src/arrow/dataset/file_parquet.cc
 @@ -26,16 +26,23 @@
- 
+
  #include "arrow/compute/cast.h"
  #include "arrow/compute/exec.h"
 +#include "arrow/dataset/dataset.h"
@@ -26,10 +26,10 @@ index 1f8b6cc488..6ac6642fdb 100644
  #include "arrow/util/tracing_internal.h"
  #include "parquet/arrow/reader.h"
  #include "parquet/arrow/schema.h"
-@@ -555,6 +562,68 @@ Future<std::shared_ptr<parquet::arrow::FileReader>> ParquetFileFormat::GetReader
+@@ -558,6 +565,68 @@ Future<std::shared_ptr<parquet::arrow::FileReader>> ParquetFileFormat::GetReader
        });
  }
- 
+
 +struct CastingGenerator {
 +  CastingGenerator(RecordBatchGenerator source, std::shared_ptr<Schema> final_schema,
 +                   const std::unordered_set<std::string>& cols_to_skip,
@@ -95,7 +95,7 @@ index 1f8b6cc488..6ac6642fdb 100644
  struct SlicingGenerator {
    SlicingGenerator(RecordBatchGenerator source, int64_t batch_size)
        : state(std::make_shared<State>(source, batch_size)) {}
-@@ -617,6 +686,9 @@ Result<RecordBatchGenerator> ParquetFileFormat::ScanBatchesAsync(
+@@ -620,6 +689,9 @@ Result<RecordBatchGenerator> ParquetFileFormat::ScanBatchesAsync(
        [this, options, parquet_fragment, pre_filtered,
         row_groups](const std::shared_ptr<parquet::arrow::FileReader>& reader) mutable
        -> Result<RecordBatchGenerator> {
@@ -105,7 +105,7 @@ index 1f8b6cc488..6ac6642fdb 100644
      // Ensure that parquet_fragment has FileMetaData
      RETURN_NOT_OK(parquet_fragment->EnsureCompleteMetadata(reader.get()));
      if (!pre_filtered) {
-@@ -633,12 +705,24 @@ Result<RecordBatchGenerator> ParquetFileFormat::ScanBatchesAsync(
+@@ -636,12 +708,24 @@ Result<RecordBatchGenerator> ParquetFileFormat::ScanBatchesAsync(
              kParquetTypeName, options.get(), default_fragment_scan_options));
      int batch_readahead = options->batch_readahead;
      int64_t rows_to_readahead = batch_readahead * options->batch_size;
@@ -136,10 +136,10 @@ index 1f8b6cc488..6ac6642fdb 100644
        return sliced;
      }
 diff --git a/cpp/src/arrow/dataset/scanner.cc b/cpp/src/arrow/dataset/scanner.cc
-index a856a792a2..5c10dfc6ac 100644
+index 8fa45ac..7cd0b73 100644
 --- a/cpp/src/arrow/dataset/scanner.cc
 +++ b/cpp/src/arrow/dataset/scanner.cc
-@@ -355,8 +355,10 @@ class OneShotFragment : public Fragment {
+@@ -360,8 +360,10 @@ class OneShotFragment : public Fragment {
      ARROW_ASSIGN_OR_RAISE(
          auto background_gen,
          MakeBackgroundGenerator(std::move(batch_it_), options->io_context.executor()));
@@ -151,60 +151,68 @@ index a856a792a2..5c10dfc6ac 100644
 +    return MakeTransferredGenerator(std::move(background_gen), cpu_executor);
    }
    std::string type_name() const override { return "one-shot"; }
- 
-@@ -382,7 +384,7 @@ Result<TaggedRecordBatchIterator> AsyncScanner::ScanBatches() {
+
+@@ -387,7 +389,7 @@ Result<TaggedRecordBatchIterator> AsyncScanner::ScanBatches() {
        [this](::arrow::internal::Executor* executor) {
          return ScanBatchesAsync(executor);
        },
 -      scan_options_->use_threads);
 +      scan_options_->use_threads, this->async_cpu_executor());
  }
- 
+
  Result<EnumeratedRecordBatchIterator> AsyncScanner::ScanBatchesUnordered() {
-@@ -390,7 +392,7 @@ Result<EnumeratedRecordBatchIterator> AsyncScanner::ScanBatchesUnordered() {
+@@ -395,7 +397,7 @@ Result<EnumeratedRecordBatchIterator> AsyncScanner::ScanBatchesUnordered() {
        [this](::arrow::internal::Executor* executor) {
          return ScanBatchesUnorderedAsync(executor);
        },
 -      scan_options_->use_threads);
 +      scan_options_->use_threads, this->async_cpu_executor());
  }
- 
+
  Result<std::shared_ptr<Table>> AsyncScanner::ToTable() {
-@@ -400,7 +402,7 @@ Result<std::shared_ptr<Table>> AsyncScanner::ToTable() {
+@@ -405,7 +407,7 @@ Result<std::shared_ptr<Table>> AsyncScanner::ToTable() {
  }
- 
+
  Result<EnumeratedRecordBatchGenerator> AsyncScanner::ScanBatchesUnorderedAsync() {
 -  return ScanBatchesUnorderedAsync(::arrow::internal::GetCpuThreadPool(),
 +  return ScanBatchesUnorderedAsync(this->async_cpu_executor(),
                                     /*sequence_fragments=*/false);
  }
- 
-@@ -601,7 +603,7 @@ Result<std::shared_ptr<Table>> AsyncScanner::Head(int64_t num_rows) {
+
+@@ -606,7 +608,7 @@ Result<std::shared_ptr<Table>> AsyncScanner::Head(int64_t num_rows) {
  }
- 
+
  Result<TaggedRecordBatchGenerator> AsyncScanner::ScanBatchesAsync() {
 -  return ScanBatchesAsync(::arrow::internal::GetCpuThreadPool());
 +  return ScanBatchesAsync(this->async_cpu_executor());
  }
- 
+
  Result<TaggedRecordBatchGenerator> AsyncScanner::ScanBatchesAsync(
-@@ -778,7 +780,7 @@ Future<int64_t> AsyncScanner::CountRowsAsync(Executor* executor) {
+@@ -783,7 +785,7 @@ Future<int64_t> AsyncScanner::CountRowsAsync(Executor* executor) {
  }
- 
+
  Future<int64_t> AsyncScanner::CountRowsAsync() {
 -  return CountRowsAsync(::arrow::internal::GetCpuThreadPool());
 +  return CountRowsAsync(this->async_cpu_executor());
  }
- 
+
  Result<int64_t> AsyncScanner::CountRows() {
 diff --git a/cpp/src/arrow/dataset/scanner.h b/cpp/src/arrow/dataset/scanner.h
-index d2de267897..1c605c1bf2 100644
+index 5031057..c867ece 100644
 --- a/cpp/src/arrow/dataset/scanner.h
 +++ b/cpp/src/arrow/dataset/scanner.h
-@@ -107,6 +107,11 @@ struct ARROW_DS_EXPORT ScanOptions {
+@@ -35,6 +35,7 @@
+ #include "arrow/type_fwd.h"
+ #include "arrow/util/async_generator_fwd.h"
+ #include "arrow/util/iterator.h"
++#include "arrow/util/thread_pool.h"
+ #include "arrow/util/type_fwd.h"
+
+ namespace arrow {
+@@ -104,6 +105,11 @@ struct ARROW_DS_EXPORT ScanOptions {
    /// Note: The IOContext executor will be ignored if use_threads is set to false
    io::IOContext io_context;
- 
+
 +  /// ExecContext for any CPU tasks
 +  ///
 +  /// Note: The ExecContext executor will be ignored if use_threads is set to false
@@ -213,20 +221,20 @@ index d2de267897..1c605c1bf2 100644
    /// If true the scanner will scan in parallel
    ///
    /// Note: If true, this will use threads from both the cpu_executor and the
-@@ -442,6 +447,11 @@ class ARROW_DS_EXPORT Scanner {
+@@ -459,6 +465,11 @@ class ARROW_DS_EXPORT Scanner {
        TaggedRecordBatchIterator scan);
- 
+
    const std::shared_ptr<ScanOptions> scan_options_;
 +
 +  ::arrow::internal::Executor* async_cpu_executor() const {
 +    return scan_options_->exec_context.executor() ? scan_options_->exec_context.executor()
 +                                                  : ::arrow::internal::GetCpuThreadPool();
 +  }
  };
- 
+
  /// \brief ScannerBuilder is a factory class to construct a Scanner. It is used
 diff --git a/cpp/src/arrow/util/thread_pool.h b/cpp/src/arrow/util/thread_pool.h
-index 44b1e227b0..218edc60ca 100644
+index cd32781..a8935d7 100644
 --- a/cpp/src/arrow/util/thread_pool.h
 +++ b/cpp/src/arrow/util/thread_pool.h
 @@ -20,6 +20,7 @@
@@ -240,7 +248,7 @@ index 44b1e227b0..218edc60ca 100644
 @@ -591,6 +592,21 @@ typename Fut::SyncType RunSynchronously(FnOnce<Fut(Executor*)> get_future,
    }
  }
- 
+
 +template <typename T>
 +Iterator<T> IterateSynchronously(
 +    FnOnce<Result<std::function<Future<T>()>>(Executor*)> get_gen, bool use_threads,
@@ -274,5 +282,5 @@ index 44b1e227b0..218edc60ca 100644
 -  }
 +  return IterateSynchronously(std::move(get_gen), use_threads, GetCpuThreadPool());
  }
- 
+
  }  // namespace internal