diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index 8bf9ccf17..274cbd3c1 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -8,34 +8,21 @@ jobs: vmImage: ubuntu-latest strategy: matrix: - linux_64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11: - CONFIG: linux_64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11 - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64-cuda11.8:ubi8 - linux_64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13: - CONFIG: linux_64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13 - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 - linux_aarch64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11: - CONFIG: linux_aarch64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11 - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64-cuda11.8:ubi8 - linux_aarch64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13: - CONFIG: linux_aarch64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13 - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 - linux_ppc64le_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11: - CONFIG: linux_ppc64le_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11 - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64-cuda11.8:ubi8 - linux_ppc64le_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13: - CONFIG: linux_ppc64le_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13 + linux_64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14: + CONFIG: linux_64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14 UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 timeoutInMinutes: 360 - variables: {} + variables: + - group: AWS-427-S3-Access-Keys steps: + - task: DownloadSecureFile@1 + name: secret_file + displayName: "Download Secrets File" + inputs: + secureFile: "publish_binary_secrets" + - script: | sudo mkdir -p /opt/empty_dir || true for d in \ @@ -77,9 +64,17 @@ jobs: else export IS_PR_BUILD="False" fi + + sudo chmod a+r $(secret_file.secureFilePath) + export SECRETS_DIR=$HOME/secrets + mkdir -p $SECRETS_DIR + sudo cp $(secret_file.secureFilePath) $SECRETS_DIR/secret_file + .scripts/run_docker_build.sh displayName: Run docker build env: BINSTAR_TOKEN: $(BINSTAR_TOKEN) FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN) - STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) \ No newline at end of file + STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) diff --git a/.azure-pipelines/azure-pipelines-osx.yml b/.azure-pipelines/azure-pipelines-osx.yml index ac0bf384e..cde770028 100755 --- a/.azure-pipelines/azure-pipelines-osx.yml +++ b/.azure-pipelines/azure-pipelines-osx.yml @@ -37,4 +37,4 @@ jobs: env: BINSTAR_TOKEN: $(BINSTAR_TOKEN) FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN) - STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) \ No newline at end of file + STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index c6ed88960..4396042b1 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -8,11 +8,11 @@ jobs: vmImage: windows-2022 strategy: matrix: - win_64_cuda_compilerNonecuda_compiler_versionNone: - CONFIG: win_64_cuda_compilerNonecuda_compiler_versionNone + win_64_cuda_compiler_version11.8: + CONFIG: win_64_cuda_compiler_version11.8 UPLOAD_PACKAGES: 'True' - win_64_cuda_compilernvcccuda_compiler_version11.8: - CONFIG: win_64_cuda_compilernvcccuda_compiler_version11.8 + win_64_cuda_compiler_versionNone: + CONFIG: win_64_cuda_compiler_versionNone UPLOAD_PACKAGES: 'True' timeoutInMinutes: 360 variables: @@ -38,4 +38,4 @@ jobs: UPLOAD_TEMP: $(UPLOAD_TEMP) BINSTAR_TOKEN: $(BINSTAR_TOKEN) FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN) - STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) \ No newline at end of file + STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) diff --git a/.ci_support/linux_64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml b/.ci_support/linux_64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml index 8c3881ddd..f0eb8a338 100644 --- a/.ci_support/linux_64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml +++ b/.ci_support/linux_64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml @@ -1,13 +1,13 @@ aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 azure_core_cpp: -- 1.14.0 +- 1.16.0 azure_identity_cpp: -- 1.10.0 +- 1.12.0 azure_storage_blobs_cpp: -- 12.13.0 +- 12.14.0 azure_storage_files_datalake_cpp: - 12.12.0 bzip2: @@ -43,21 +43,21 @@ gflags: glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libopentelemetry_cpp: -- '1.20' +- '1.21' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -65,7 +65,7 @@ lz4_c: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -73,12 +73,12 @@ snappy: target_platform: - linux-64 thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler - cuda_compiler_version + - cuda_compiler - docker_image zlib: - '1' diff --git a/.ci_support/linux_64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml b/.ci_support/linux_64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14.yaml similarity index 85% rename from .ci_support/linux_64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml rename to .ci_support/linux_64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14.yaml index 0faf91694..2979cc495 100644 --- a/.ci_support/linux_64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml +++ b/.ci_support/linux_64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14.yaml @@ -1,13 +1,13 @@ aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 azure_core_cpp: -- 1.14.0 +- 1.16.0 azure_identity_cpp: -- 1.10.0 +- 1.12.0 azure_storage_blobs_cpp: -- 12.13.0 +- 12.14.0 azure_storage_files_datalake_cpp: - 12.12.0 bzip2: @@ -15,7 +15,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '13' +- '14' c_stdlib: - sysroot c_stdlib_version: @@ -27,7 +27,7 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- None +- cuda-nvcc cuda_compiler_version: - None cuda_compiler_version_min: @@ -35,7 +35,7 @@ cuda_compiler_version_min: cxx_compiler: - gxx cxx_compiler_version: -- '13' +- '14' docker_image: - quay.io/condaforge/linux-anvil-x86_64:alma9 gflags: @@ -43,21 +43,21 @@ gflags: glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libopentelemetry_cpp: -- '1.20' +- '1.21' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -65,7 +65,7 @@ lz4_c: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -73,12 +73,12 @@ snappy: target_platform: - linux-64 thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler - cuda_compiler_version + - cuda_compiler - docker_image zlib: - '1' diff --git a/.ci_support/linux_aarch64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml b/.ci_support/linux_aarch64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml index 30f814b1e..274b2f0bf 100644 --- a/.ci_support/linux_aarch64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml +++ b/.ci_support/linux_aarch64_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml @@ -1,13 +1,13 @@ aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 azure_core_cpp: -- 1.14.0 +- 1.16.0 azure_identity_cpp: -- 1.10.0 +- 1.12.0 azure_storage_blobs_cpp: -- 12.13.0 +- 12.14.0 azure_storage_files_datalake_cpp: - 12.12.0 bzip2: @@ -43,21 +43,21 @@ gflags: glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libopentelemetry_cpp: -- '1.20' +- '1.21' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -65,7 +65,7 @@ lz4_c: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -73,12 +73,12 @@ snappy: target_platform: - linux-aarch64 thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler - cuda_compiler_version + - cuda_compiler - docker_image zlib: - '1' diff --git a/.ci_support/linux_aarch64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml b/.ci_support/linux_aarch64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14.yaml similarity index 85% rename from .ci_support/linux_aarch64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml rename to .ci_support/linux_aarch64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14.yaml index 282989dbc..0c4a053ee 100644 --- a/.ci_support/linux_aarch64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml +++ b/.ci_support/linux_aarch64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14.yaml @@ -1,13 +1,13 @@ aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 azure_core_cpp: -- 1.14.0 +- 1.16.0 azure_identity_cpp: -- 1.10.0 +- 1.12.0 azure_storage_blobs_cpp: -- 12.13.0 +- 12.14.0 azure_storage_files_datalake_cpp: - 12.12.0 bzip2: @@ -15,7 +15,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '13' +- '14' c_stdlib: - sysroot c_stdlib_version: @@ -27,7 +27,7 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- None +- cuda-nvcc cuda_compiler_version: - None cuda_compiler_version_min: @@ -35,7 +35,7 @@ cuda_compiler_version_min: cxx_compiler: - gxx cxx_compiler_version: -- '13' +- '14' docker_image: - quay.io/condaforge/linux-anvil-x86_64:alma9 gflags: @@ -43,21 +43,21 @@ gflags: glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libopentelemetry_cpp: -- '1.20' +- '1.21' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -65,7 +65,7 @@ lz4_c: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -73,12 +73,12 @@ snappy: target_platform: - linux-aarch64 thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler - cuda_compiler_version + - cuda_compiler - docker_image zlib: - '1' diff --git a/.ci_support/linux_ppc64le_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml b/.ci_support/linux_ppc64le_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml index 462c615c0..6d83d88c2 100644 --- a/.ci_support/linux_ppc64le_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml +++ b/.ci_support/linux_ppc64le_c_compiler_version11cuda_compiler_version11.8cxx_compiler_version11.yaml @@ -1,13 +1,13 @@ aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 azure_core_cpp: -- 1.14.0 +- 1.16.0 azure_identity_cpp: -- 1.10.0 +- 1.12.0 azure_storage_blobs_cpp: -- 12.13.0 +- 12.14.0 azure_storage_files_datalake_cpp: - 12.12.0 bzip2: @@ -43,21 +43,21 @@ gflags: glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libopentelemetry_cpp: -- '1.20' +- '1.21' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -65,7 +65,7 @@ lz4_c: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -73,12 +73,12 @@ snappy: target_platform: - linux-ppc64le thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler - cuda_compiler_version + - cuda_compiler - docker_image zlib: - '1' diff --git a/.ci_support/linux_ppc64le_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml b/.ci_support/linux_ppc64le_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14.yaml similarity index 85% rename from .ci_support/linux_ppc64le_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml rename to .ci_support/linux_ppc64le_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14.yaml index 2500e271f..55969feee 100644 --- a/.ci_support/linux_ppc64le_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml +++ b/.ci_support/linux_ppc64le_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14.yaml @@ -1,13 +1,13 @@ aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 azure_core_cpp: -- 1.14.0 +- 1.16.0 azure_identity_cpp: -- 1.10.0 +- 1.12.0 azure_storage_blobs_cpp: -- 12.13.0 +- 12.14.0 azure_storage_files_datalake_cpp: - 12.12.0 bzip2: @@ -15,7 +15,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '13' +- '14' c_stdlib: - sysroot c_stdlib_version: @@ -27,7 +27,7 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- None +- cuda-nvcc cuda_compiler_version: - None cuda_compiler_version_min: @@ -35,7 +35,7 @@ cuda_compiler_version_min: cxx_compiler: - gxx cxx_compiler_version: -- '13' +- '14' docker_image: - quay.io/condaforge/linux-anvil-x86_64:alma9 gflags: @@ -43,21 +43,21 @@ gflags: glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libopentelemetry_cpp: -- '1.20' +- '1.21' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -65,7 +65,7 @@ lz4_c: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -73,12 +73,12 @@ snappy: target_platform: - linux-ppc64le thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler - cuda_compiler_version + - cuda_compiler - docker_image zlib: - '1' diff --git a/.ci_support/migrations/absl_grpc_proto_25Q2.yaml b/.ci_support/migrations/absl_grpc_proto_25Q2.yaml new file mode 100644 index 000000000..10b7a0955 --- /dev/null +++ b/.ci_support/migrations/absl_grpc_proto_25Q2.yaml @@ -0,0 +1,29 @@ +__migrator: + build_number: 1 + commit_message: Rebuild for libabseil 20250512, libgrpc 1.73 & libprotobuf 6.31.1 + kind: version + migration_number: 1 + exclude: + # core deps + - abseil-cpp + - grpc-cpp + - libprotobuf + # required for building/testing + - protobuf + - re2 + # bazel stack + - bazel + - grpc_java_plugin + - singlejar +libabseil: +- 20250512 +libgrpc: +- "1.73" +libprotobuf: +- 6.31.1 +# we need to leave this migration open until we're ready to move the global baseline, see +# https://github.com/conda-forge/conda-forge.github.io/issues/2467; grpc 1.72 requires 11.0, +# see https://github.com/grpc/grpc/commit/f122d248443c81592e748da1adb240cbf0a0231c +c_stdlib_version: # [osx] + - 11.0 # [osx] +migrator_ts: 1748506837.6039238 diff --git a/.ci_support/migrations/aws_crt_cpp0324.yaml b/.ci_support/migrations/aws_crt_cpp0324.yaml deleted file mode 100644 index a62e28610..000000000 --- a/.ci_support/migrations/aws_crt_cpp0324.yaml +++ /dev/null @@ -1,9 +0,0 @@ -__migrator: - build_number: 1 - commit_message: Rebuild for aws_crt_cpp 0.32.4 - kind: version - migration_number: 1 - automerge: true -aws_crt_cpp: -- 0.32.4 -migrator_ts: 1744842909.2272944 diff --git a/.ci_support/migrations/cuda118.yaml b/.ci_support/migrations/cuda118.yaml new file mode 100644 index 000000000..de3fb48c6 --- /dev/null +++ b/.ci_support/migrations/cuda118.yaml @@ -0,0 +1,70 @@ +migrator_ts: 1748496951 +__migrator: + kind: + version + migration_number: + 1 + build_number: + 1 + # This is intended as a _manual_ migrator to re-add CUDA 11.8, after we + # dropped it as version that's built by default; DO NOT unpause + paused: true + override_cbc_keys: + - cuda_compiler_stub + operation: key_add + check_solvable: false + primary_key: cuda_compiler_version + additional_zip_keys: + - cuda_compiler + - docker_image # [linux] + ordering: + cuda_compiler: + - None + - cuda-nvcc + - nvcc + cuda_compiler_version: + - 12.4 + - 12.6 + - 12.8 + - None + - 12.9 + - 11.8 + cuda_compiler_version_min: + - 12.4 + - 12.6 + - 12.8 + - 12.9 + - 11.8 + +cuda_compiler: # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - nvcc # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +cuda_compiler_version: # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.8 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +cuda_compiler_version_min: # [linux or win64] + - 11.8 # [linux or win64] + +c_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +cxx_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +fortran_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +docker_image: # [os.environ.get("BUILD_PLATFORM", "").startswith("linux-") and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + ### Docker images with CUDA 11.8 support + + # CUDA 11.8 builds (only x64 has a DEFAULT_LINUX_VERSION choice; alma9 not available) + - quay.io/condaforge/linux-anvil-x86_64-cuda11.8:cos7 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64" and os.environ.get("DEFAULT_LINUX_VERSION", "ubi8") == "cos7"] + - quay.io/condaforge/linux-anvil-x86_64-cuda11.8:ubi8 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64" and os.environ.get("DEFAULT_LINUX_VERSION", "ubi8") in ("ubi8", "alma8", "alma9")] + + # CUDA 11.8 arch: native compilation (build == target) + - quay.io/condaforge/linux-anvil-aarch64-cuda11.8:ubi8 # [aarch64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + - quay.io/condaforge/linux-anvil-ppc64le-cuda11.8:ubi8 # [ppc64le and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + + # CUDA 11.8 arch: cross-compilation (build != target) + - quay.io/condaforge/linux-anvil-x86_64-cuda11.8:ubi8 # [aarch64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-x86_64-cuda11.8:ubi8 # [ppc64le and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] diff --git a/.ci_support/migrations/libboost186.yaml b/.ci_support/migrations/libboost188.yaml similarity index 50% rename from .ci_support/migrations/libboost186.yaml rename to .ci_support/migrations/libboost188.yaml index 001c6a638..6eca3fb6a 100644 --- a/.ci_support/migrations/libboost186.yaml +++ b/.ci_support/migrations/libboost188.yaml @@ -1,14 +1,12 @@ __migrator: build_number: 1 kind: version - commit_message: "Rebuild for libboost 1.86" + commit_message: "Rebuild for libboost 1.88" migration_number: 1 -assimp: -- 5.4.2 libboost_devel: -- "1.86" +- "1.88" libboost_headers: -- "1.86" +- "1.88" libboost_python_devel: -- "1.86" -migrator_ts: 1723764795.6693385 +- "1.88" +migrator_ts: 1753251695.3315456 diff --git a/.ci_support/migrations/orc220.yaml b/.ci_support/migrations/orc220.yaml new file mode 100644 index 000000000..5840d7bcc --- /dev/null +++ b/.ci_support/migrations/orc220.yaml @@ -0,0 +1,8 @@ +__migrator: + build_number: 1 + commit_message: Rebuild for orc 2.2.0 + kind: version + migration_number: 1 +migrator_ts: 1754220388.1938324 +orc: +- 2.2.0 diff --git a/.ci_support/osx_64_.yaml b/.ci_support/osx_64_.yaml index 214c80336..eff3dc103 100644 --- a/.ci_support/osx_64_.yaml +++ b/.ci_support/osx_64_.yaml @@ -1,17 +1,17 @@ MACOSX_DEPLOYMENT_TARGET: -- '10.14' +- '11.0' MACOSX_SDK_VERSION: -- '10.14' +- '11.0' aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 azure_core_cpp: -- 1.14.0 +- 1.16.0 azure_identity_cpp: -- 1.10.0 +- 1.12.0 azure_storage_blobs_cpp: -- 12.13.0 +- 12.14.0 azure_storage_files_datalake_cpp: - 12.12.0 bzip2: @@ -19,11 +19,11 @@ bzip2: c_compiler: - clang c_compiler_version: -- '18' +- '19' c_stdlib: - macosx_deployment_target c_stdlib_version: -- '10.14' +- '11.0' channel_sources: - conda-forge channel_targets: @@ -33,27 +33,27 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '18' +- '19' gflags: - '2.2' glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libopentelemetry_cpp: -- '1.20' +- '1.21' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -63,7 +63,7 @@ macos_machine: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -71,7 +71,7 @@ snappy: target_platform: - osx-64 thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/.ci_support/osx_arm64_.yaml b/.ci_support/osx_arm64_.yaml index ff314a802..c5f62e1e1 100644 --- a/.ci_support/osx_arm64_.yaml +++ b/.ci_support/osx_arm64_.yaml @@ -3,15 +3,15 @@ MACOSX_DEPLOYMENT_TARGET: MACOSX_SDK_VERSION: - '11.0' aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 azure_core_cpp: -- 1.14.0 +- 1.16.0 azure_identity_cpp: -- 1.10.0 +- 1.12.0 azure_storage_blobs_cpp: -- 12.13.0 +- 12.14.0 azure_storage_files_datalake_cpp: - 12.12.0 bzip2: @@ -19,7 +19,7 @@ bzip2: c_compiler: - clang c_compiler_version: -- '18' +- '19' c_stdlib: - macosx_deployment_target c_stdlib_version: @@ -33,27 +33,27 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '18' +- '19' gflags: - '2.2' glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libopentelemetry_cpp: -- '1.20' +- '1.21' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -63,7 +63,7 @@ macos_machine: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -71,7 +71,7 @@ snappy: target_platform: - osx-arm64 thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/.ci_support/win_64_cuda_compilernvcccuda_compiler_version11.8.yaml b/.ci_support/win_64_cuda_compiler_version11.8.yaml similarity index 81% rename from .ci_support/win_64_cuda_compilernvcccuda_compiler_version11.8.yaml rename to .ci_support/win_64_cuda_compiler_version11.8.yaml index 88bfe0321..84244884e 100644 --- a/.ci_support/win_64_cuda_compilernvcccuda_compiler_version11.8.yaml +++ b/.ci_support/win_64_cuda_compiler_version11.8.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 bzip2: - '1' c_compiler: @@ -25,23 +25,23 @@ gflags: glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libcrc32c: - '1.1' libcurl: - '8' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -49,7 +49,7 @@ lz4_c: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -57,10 +57,10 @@ snappy: target_platform: - win-64 thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: -- - cuda_compiler - - cuda_compiler_version +- - cuda_compiler_version + - cuda_compiler zlib: - '1' zstd: diff --git a/.ci_support/win_64_cuda_compilerNonecuda_compiler_versionNone.yaml b/.ci_support/win_64_cuda_compiler_versionNone.yaml similarity index 79% rename from .ci_support/win_64_cuda_compilerNonecuda_compiler_versionNone.yaml rename to .ci_support/win_64_cuda_compiler_versionNone.yaml index 60a22e6ae..017061e76 100644 --- a/.ci_support/win_64_cuda_compilerNonecuda_compiler_versionNone.yaml +++ b/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.32.4 +- 0.33.1 aws_sdk_cpp: -- 1.11.510 +- 1.11.606 bzip2: - '1' c_compiler: @@ -13,7 +13,7 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- None +- cuda-nvcc cuda_compiler_version: - None cuda_compiler_version_min: @@ -25,23 +25,23 @@ gflags: glog: - '0.7' libabseil: -- '20250127' +- '20250512' libarrow: - '*' libboost_devel: -- '1.86' +- '1.88' libcrc32c: - '1.1' libcurl: - '8' libgoogle_cloud_devel: -- '2.36' +- '2.39' libgoogle_cloud_storage_devel: -- '2.36' +- '2.39' libgrpc: -- '1.71' +- '1.73' libprotobuf: -- 5.29.3 +- 6.31.1 libutf8proc: - '2.10' lz4_c: @@ -49,7 +49,7 @@ lz4_c: openssl: - '3' orc: -- 2.1.1 +- 2.2.0 re2: - 2024.07.02 snappy: @@ -57,10 +57,10 @@ snappy: target_platform: - win-64 thrift_cpp: -- 0.21.0 +- 0.22.0 zip_keys: -- - cuda_compiler - - cuda_compiler_version +- - cuda_compiler_version + - cuda_compiler zlib: - '1' zstd: diff --git a/.gitattributes b/.gitattributes index 18f114a1f..85ccb8f4f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -24,4 +24,5 @@ bld.bat text eol=crlf /README.md linguist-generated=true azure-pipelines.yml linguist-generated=true build-locally.py linguist-generated=true +pixi.toml linguist-generated=true shippable.yml linguist-generated=true diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 827a29fbc..c6830decb 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -89,7 +89,8 @@ else ( startgroup "Uploading packages" ) 2> /dev/null if [[ "${UPLOAD_PACKAGES}" != "False" ]] && [[ "${IS_PR_BUILD}" == "False" ]]; then - upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" + # upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" + bash ${FEEDSTOCK_ROOT}/.scripts/upload_binary.sh fi ( endgroup "Uploading packages" ) 2> /dev/null @@ -97,4 +98,4 @@ fi ( startgroup "Final checks" ) 2> /dev/null -touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" \ No newline at end of file +touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 6ee04aa12..311d80bd2 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -86,6 +86,7 @@ docker pull "${DOCKER_IMAGE}" docker run ${DOCKER_RUN_ARGS} \ -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \ -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z,delegated \ + -v "${SECRETS_DIR}":/home/secrets:rw,z,delegated \ -e CONFIG \ -e HOST_USER_ID \ -e UPLOAD_PACKAGES \ @@ -103,6 +104,8 @@ docker run ${DOCKER_RUN_ARGS} \ -e BINSTAR_TOKEN \ -e FEEDSTOCK_TOKEN \ -e STAGING_BINSTAR_TOKEN \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY \ "${DOCKER_IMAGE}" \ bash \ "/home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh" @@ -111,4 +114,4 @@ docker run ${DOCKER_RUN_ARGS} \ test -f "$DONE_CANARY" # This closes the last group opened in `build_steps.sh` -( endgroup "Final checks" ) 2> /dev/null \ No newline at end of file +( endgroup "Final checks" ) 2> /dev/null diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index 0e3c39422..361edeb2c 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -117,4 +117,4 @@ else fi ( endgroup "Uploading packages" ) 2> /dev/null -fi \ No newline at end of file +fi diff --git a/.scripts/run_win_build.bat b/.scripts/run_win_build.bat index 283e14388..27c552b1a 100755 --- a/.scripts/run_win_build.bat +++ b/.scripts/run_win_build.bat @@ -25,7 +25,7 @@ set "MICROMAMBA_EXE=%MICROMAMBA_TMPDIR%\micromamba.exe" echo Downloading micromamba %MICROMAMBA_VERSION% if not exist "%MICROMAMBA_TMPDIR%" mkdir "%MICROMAMBA_TMPDIR%" -certutil -urlcache -split -f "%MICROMAMBA_URL%" "%MICROMAMBA_EXE%" +powershell -ExecutionPolicy Bypass -Command "(New-Object Net.WebClient).DownloadFile('%MICROMAMBA_URL%', '%MICROMAMBA_EXE%')" if !errorlevel! neq 0 exit /b !errorlevel! echo Creating environment @@ -148,4 +148,4 @@ if /i "%CI%" == "azure" ( echo ##[endgroup] exit /b ) -exit /b \ No newline at end of file +exit /b diff --git a/.scripts/upload_binary.sh b/.scripts/upload_binary.sh new file mode 100644 index 000000000..2f0c1affe --- /dev/null +++ b/.scripts/upload_binary.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -exo pipefail + +echo "********** Publishing to Artifactory **********" +USERNAME=`cat /home/secrets/secret_file | grep artifactory.ci.username | cut -f 2 -d' '` +TOKEN=`cat /home/secrets/secret_file | grep artifactory.ci.token | cut -f 2 -d' '` +ANACONDA_TOKEN=`cat /home/secrets/secret_file | grep anaconda.org.token | cut -f 2 -d' '` + +CHANNEL_NAME="bodo.ai-platform" + +for package in `ls ${FEEDSTOCK_ROOT}/build_artifacts/linux-64/*.conda`; do + package_name=`basename $package` + echo "Package Name: $package_name" + + curl -u${USERNAME}:${TOKEN} -T $package "https://bodo.jfrog.io/artifactory/${CHANNEL_NAME}/linux-64/$package_name" + anaconda -t $ANACONDA_TOKEN upload -u bodo.ai -c bodo.ai $package --label main --force +done +curl -X POST https://$USERNAME:$TOKEN@bodo.jfrog.io/artifactory/api/conda/$CHANNEL_NAME/reindex + +# Reindex Conda +curl -s -X POST "https://$USERNAME:$TOKEN@bodo.jfrog.io/artifactory/api/conda/$CHANNEL_NAME/reindex?async=0" diff --git a/README.md b/README.md index c0f928751..eb8968448 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,15 @@ Package license: Apache-2.0 Summary: A meta-package to select Arrow build variant +About arrow-utils +----------------- + +Home: http://github.com/apache/arrow + +Package license: Apache-2.0 + +Summary: Executables for manipulating Apache arrow files + About libarrow -------------- @@ -40,6 +49,15 @@ Package license: Apache-2.0 Summary: C++ libraries for Apache Arrow Acero +About libarrow-dataset +---------------------- + +Home: http://github.com/apache/arrow + +Package license: Apache-2.0 + +Summary: C++ libraries for Apache Arrow Dataset + About libarrow-flight --------------------- @@ -49,50 +67,50 @@ Package license: Apache-2.0 Summary: C++ libraries for Apache Arrow Flight -About libarrow-gandiva ----------------------- +About libarrow-flight-sql +------------------------- Home: http://github.com/apache/arrow Package license: Apache-2.0 -Summary: C++ libraries for Apache Arrow Gandiva +Summary: C++ libraries for Apache Arrow Flight SQL -About libparquet ----------------- +About libarrow-gandiva +---------------------- Home: http://github.com/apache/arrow Package license: Apache-2.0 -Summary: C++ libraries for Apache Parquet +Summary: C++ libraries for Apache Arrow Gandiva -About libarrow-dataset ----------------------- +About libarrow-substrait +------------------------ Home: http://github.com/apache/arrow Package license: Apache-2.0 -Summary: C++ libraries for Apache Arrow Dataset +Summary: C++ libraries for Apache Arrow Substrait -About libarrow-flight-sql -------------------------- +About libparquet +---------------- Home: http://github.com/apache/arrow Package license: Apache-2.0 -Summary: C++ libraries for Apache Arrow Flight SQL +Summary: C++ libraries for Apache Parquet -About libarrow-substrait ------------------------- +About parquet-utils +------------------- Home: http://github.com/apache/arrow Package license: Apache-2.0 -Summary: C++ libraries for Apache Arrow Substrait +Summary: Executables for inspecting Apache Parquet files Current build status ==================== @@ -119,10 +137,10 @@ Current build status - linux_64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13 + linux_64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14 - variant + variant @@ -133,10 +151,10 @@ Current build status - linux_aarch64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13 + linux_aarch64_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14 - variant + variant @@ -147,10 +165,10 @@ Current build status - linux_ppc64le_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13 + linux_ppc64le_c_compiler_version14cuda_compiler_versionNonecxx_compiler_version14 - variant + variant @@ -168,17 +186,17 @@ Current build status - win_64_cuda_compilerNonecuda_compiler_versionNone + win_64_cuda_compiler_version11.8 - variant + variant - win_64_cuda_compilernvcccuda_compiler_version11.8 + win_64_cuda_compiler_versionNone - variant + variant @@ -195,6 +213,7 @@ Current release info | Name | Downloads | Version | Platforms | | --- | --- | --- | --- | | [![Conda Recipe](https://img.shields.io/badge/recipe-apache--arrow--proc-green.svg)](https://anaconda.org/conda-forge/apache-arrow-proc) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/apache-arrow-proc.svg)](https://anaconda.org/conda-forge/apache-arrow-proc) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/apache-arrow-proc.svg)](https://anaconda.org/conda-forge/apache-arrow-proc) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/apache-arrow-proc.svg)](https://anaconda.org/conda-forge/apache-arrow-proc) | +| [![Conda Recipe](https://img.shields.io/badge/recipe-arrow--utils-green.svg)](https://anaconda.org/conda-forge/arrow-utils) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/arrow-utils.svg)](https://anaconda.org/conda-forge/arrow-utils) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/arrow-utils.svg)](https://anaconda.org/conda-forge/arrow-utils) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/arrow-utils.svg)](https://anaconda.org/conda-forge/arrow-utils) | | [![Conda Recipe](https://img.shields.io/badge/recipe-libarrow-green.svg)](https://anaconda.org/conda-forge/libarrow) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libarrow.svg)](https://anaconda.org/conda-forge/libarrow) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libarrow.svg)](https://anaconda.org/conda-forge/libarrow) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libarrow.svg)](https://anaconda.org/conda-forge/libarrow) | | [![Conda Recipe](https://img.shields.io/badge/recipe-libarrow--acero-green.svg)](https://anaconda.org/conda-forge/libarrow-acero) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libarrow-acero.svg)](https://anaconda.org/conda-forge/libarrow-acero) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libarrow-acero.svg)](https://anaconda.org/conda-forge/libarrow-acero) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libarrow-acero.svg)](https://anaconda.org/conda-forge/libarrow-acero) | | [![Conda Recipe](https://img.shields.io/badge/recipe-libarrow--all-green.svg)](https://anaconda.org/conda-forge/libarrow-all) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libarrow-all.svg)](https://anaconda.org/conda-forge/libarrow-all) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libarrow-all.svg)](https://anaconda.org/conda-forge/libarrow-all) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libarrow-all.svg)](https://anaconda.org/conda-forge/libarrow-all) | @@ -204,6 +223,7 @@ Current release info | [![Conda Recipe](https://img.shields.io/badge/recipe-libarrow--gandiva-green.svg)](https://anaconda.org/conda-forge/libarrow-gandiva) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libarrow-gandiva.svg)](https://anaconda.org/conda-forge/libarrow-gandiva) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libarrow-gandiva.svg)](https://anaconda.org/conda-forge/libarrow-gandiva) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libarrow-gandiva.svg)](https://anaconda.org/conda-forge/libarrow-gandiva) | | [![Conda Recipe](https://img.shields.io/badge/recipe-libarrow--substrait-green.svg)](https://anaconda.org/conda-forge/libarrow-substrait) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libarrow-substrait.svg)](https://anaconda.org/conda-forge/libarrow-substrait) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libarrow-substrait.svg)](https://anaconda.org/conda-forge/libarrow-substrait) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libarrow-substrait.svg)](https://anaconda.org/conda-forge/libarrow-substrait) | | [![Conda Recipe](https://img.shields.io/badge/recipe-libparquet-green.svg)](https://anaconda.org/conda-forge/libparquet) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libparquet.svg)](https://anaconda.org/conda-forge/libparquet) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libparquet.svg)](https://anaconda.org/conda-forge/libparquet) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libparquet.svg)](https://anaconda.org/conda-forge/libparquet) | +| [![Conda Recipe](https://img.shields.io/badge/recipe-parquet--utils-green.svg)](https://anaconda.org/conda-forge/parquet-utils) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/parquet-utils.svg)](https://anaconda.org/conda-forge/parquet-utils) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/parquet-utils.svg)](https://anaconda.org/conda-forge/parquet-utils) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/parquet-utils.svg)](https://anaconda.org/conda-forge/parquet-utils) | Installing arrow-cpp ==================== @@ -215,16 +235,16 @@ conda config --add channels conda-forge conda config --set channel_priority strict ``` -Once the `conda-forge` channel has been enabled, `apache-arrow-proc, libarrow, libarrow-acero, libarrow-all, libarrow-dataset, libarrow-flight, libarrow-flight-sql, libarrow-gandiva, libarrow-substrait, libparquet` can be installed with `conda`: +Once the `conda-forge` channel has been enabled, `apache-arrow-proc, arrow-utils, libarrow, libarrow-acero, libarrow-all, libarrow-dataset, libarrow-flight, libarrow-flight-sql, libarrow-gandiva, libarrow-substrait, libparquet, parquet-utils` can be installed with `conda`: ``` -conda install apache-arrow-proc libarrow libarrow-acero libarrow-all libarrow-dataset libarrow-flight libarrow-flight-sql libarrow-gandiva libarrow-substrait libparquet +conda install apache-arrow-proc arrow-utils libarrow libarrow-acero libarrow-all libarrow-dataset libarrow-flight libarrow-flight-sql libarrow-gandiva libarrow-substrait libparquet parquet-utils ``` or with `mamba`: ``` -mamba install apache-arrow-proc libarrow libarrow-acero libarrow-all libarrow-dataset libarrow-flight libarrow-flight-sql libarrow-gandiva libarrow-substrait libparquet +mamba install apache-arrow-proc arrow-utils libarrow libarrow-acero libarrow-all libarrow-dataset libarrow-flight libarrow-flight-sql libarrow-gandiva libarrow-substrait libparquet parquet-utils ``` It is possible to list all of the versions of `apache-arrow-proc` available on your platform with `conda`: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index eff4ad66b..2ed88ae42 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,6 +2,8 @@ # update the conda-forge.yml and/or the recipe/meta.yaml. # -*- mode: yaml -*- +trigger: none +pr: none stages: - stage: Check jobs: @@ -29,5 +31,3 @@ stages: dependsOn: Check jobs: - template: ./.azure-pipelines/azure-pipelines-linux.yml - - template: ./.azure-pipelines/azure-pipelines-osx.yml - - template: ./.azure-pipelines/azure-pipelines-win.yml \ No newline at end of file diff --git a/build-locally.py b/build-locally.py index 02c23747b..9dfe44039 100755 --- a/build-locally.py +++ b/build-locally.py @@ -106,9 +106,7 @@ def main(args=None): action="store_true", help="Setup debug environment using `conda debug`", ) - p.add_argument( - "--output-id", help="If running debug, specify the output to setup." - ) + p.add_argument("--output-id", help="If running debug, specify the output to setup.") ns = p.parse_args(args=args) verify_config(ns) @@ -124,9 +122,7 @@ def main(args=None): elif ns.config.startswith("win"): run_win_build(ns) finally: - recipe_license_file = os.path.join( - "recipe", "recipe-scripts-license.txt" - ) + recipe_license_file = os.path.join("recipe", "recipe-scripts-license.txt") if os.path.exists(recipe_license_file): os.remove(recipe_license_file) diff --git a/recipe/bld.bat b/recipe/bld.bat index cf1b7c596..c2ed88fc2 100644 --- a/recipe/bld.bat +++ b/recipe/bld.bat @@ -21,7 +21,7 @@ cmake -G "Ninja" ^ -DARROW_BOOST_USE_SHARED:BOOL=ON ^ -DARROW_BUILD_STATIC:BOOL=OFF ^ -DARROW_BUILD_TESTS:BOOL=ON ^ - -DARROW_BUILD_UTILITIES:BOOL=OFF ^ + -DARROW_BUILD_UTILITIES:BOOL=ON ^ -DARROW_COMPUTE:BOOL=ON ^ -DARROW_CSV:BOOL=ON ^ -DARROW_DATASET:BOOL=ON ^ @@ -39,6 +39,7 @@ cmake -G "Ninja" ^ -DARROW_ORC:BOOL=ON ^ -DARROW_PACKAGE_PREFIX="%LIBRARY_PREFIX%" ^ -DARROW_PARQUET:BOOL=ON ^ + -DPARQUET_BUILD_EXECUTABLES:BOOL=ON ^ -DARROW_S3:BOOL=ON ^ -DARROW_SIMD_LEVEL:STRING=NONE ^ -DARROW_SUBSTRAIT:BOOL=ON ^ diff --git a/recipe/build.sh b/recipe/build.sh index a26c2617f..0d14fab70 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -48,6 +48,12 @@ fi # reusable variable for dependencies we cannot yet unvendor export READ_RECIPE_META_YAML_WHY_NOT=OFF +# Enable SCCache +export SCCACHE_BUCKET=engine-codebuild-cache +export SCCACHE_REGION=us-east-2 +export SCCACHE_S3_USE_SSL=true +export SCCACHE_S3_SERVER_SIDE_ENCRYPTION=true + # for available switches see # https://github.com/apache/arrow/blame/apache-arrow-12.0.0/cpp/cmake_modules/DefineOptions.cmake # placeholder in ARROW_GDB_INSTALL_DIR must match _la_placeholder in activate.sh @@ -58,7 +64,7 @@ cmake -GNinja \ -DARROW_BUILD_BENCHMARKS=OFF \ -DARROW_BUILD_STATIC=OFF \ -DARROW_BUILD_TESTS=ON \ - -DARROW_BUILD_UTILITIES=OFF \ + -DARROW_BUILD_UTILITIES=ON \ -DARROW_COMPUTE=ON \ -DARROW_CSV=ON \ -DARROW_CXXFLAGS="${CXXFLAGS}" \ @@ -80,11 +86,13 @@ cmake -GNinja \ -DARROW_ORC=ON \ -DARROW_PACKAGE_PREFIX=$PREFIX \ -DARROW_PARQUET=ON \ + -DPARQUET_BUILD_EXECUTABLES=ON \ -DARROW_S3=ON \ -DARROW_SIMD_LEVEL=NONE \ -DARROW_SUBSTRAIT=ON \ -DARROW_USE_GLOG=ON \ -DARROW_USE_LD_GOLD=ON \ + -DARROW_USE_SCCACHE=ON \ -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_LZ4=ON \ diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml index 499d9fbb1..7cdfa120a 100644 --- a/recipe/conda_build_config.yaml +++ b/recipe/conda_build_config.yaml @@ -6,10 +6,6 @@ libarrow_all: # on osx, keep this in sync with llvm_version in meta.yaml. c_compiler_version: # [osx] - - 18 # [osx] + - 19 # [osx] cxx_compiler_version: # [osx] - - 18 # [osx] -c_compiler: # [win] - - vs2022 # [win] -cxx_compiler: # [win] - - vs2022 # [win] + - 19 # [osx] diff --git a/recipe/install-libarrow.bat b/recipe/install-libarrow.bat index e7011b579..9b817dfbd 100644 --- a/recipe/install-libarrow.bat +++ b/recipe/install-libarrow.bat @@ -73,6 +73,10 @@ if [%PKG_NAME%] == [libarrow] ( move .\temp_prefix\lib\cmake\Parquet\* %LIBRARY_LIB%\cmake\Parquet mkdir %LIBRARY_PREFIX%\include\parquet xcopy /s /y .\temp_prefix\include\parquet %LIBRARY_PREFIX%\include\parquet +) else if [%PKG_NAME%] == [parquet-utils] ( + copy .\temp_prefix\bin\parquet-*.exe %LIBRARY_BIN% +) else if [%PKG_NAME%] == [arrow-utils] ( + copy .\temp_prefix\bin\arrow-*.exe %LIBRARY_BIN% ) else if [%PKG_NAME%] == [libarrow-all] ( REM libarrow-all: install everything else (whatever ends up in this output REM should generally be installed into the appropriate libarrow-). diff --git a/recipe/install-libarrow.sh b/recipe/install-libarrow.sh index 23cf22afb..4b56a0cc4 100644 --- a/recipe/install-libarrow.sh +++ b/recipe/install-libarrow.sh @@ -69,6 +69,10 @@ elif [[ "${PKG_NAME}" == "libparquet" ]]; then cp ./temp_prefix/lib/pkgconfig/parquet.pc $PREFIX/lib/pkgconfig cp -R ./temp_prefix/lib/cmake/Parquet/. $PREFIX/lib/cmake/Parquet cp -R ./temp_prefix/include/parquet/. $PREFIX/include/parquet +elif [[ "${PKG_NAME}" == "parquet-utils" ]]; then + cp ./temp_prefix/bin/parquet-* $PREFIX/bin +elif [[ "${PKG_NAME}" == "arrow-utils" ]]; then + cp ./temp_prefix/bin/arrow-* $PREFIX/bin elif [[ "${PKG_NAME}" == "libarrow-all" ]]; then # libarrow-all: install everything else (whatever ends up in this output # should generally be installed into the appropriate libarrow-). diff --git a/recipe/meta.yaml b/recipe/meta.yaml index cb30f0e49..678aa4964 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -3,7 +3,7 @@ {% set build_ext_version = "5.0.0" %} {% set build_ext = "cuda" if cuda_enabled else "cpu" %} {% set proc_build_number = "0" %} -{% set llvm_version = "18" %} +{% set llvm_version = "19" %} # see https://github.com/apache/arrow/blob/apache-arrow-10.0.1/cpp/CMakeLists.txt#L88-L90 {% set so_version = (version.split(".")[0] | int * 100 + version.split(".")[1] | int) ~ "." ~ version.split(".")[2] ~ ".0" %} @@ -28,6 +28,13 @@ source: - patches/0004-GH-45225-C-Upgrade-ORC-to-2.1.0-45226.patch # backport https://github.com/apache/arrow/pull/45859 - patches/0005-MINOR-C-Fix-OTel-1.19-compat-after-silent-breaking-c.patch + # backport https://github.com/apache/arrow/pull/46136 + - patches/0006-GH-46134-CI-C-Explicit-conversion-of-possible-absl-s.patch + # backport https://github.com/apache/arrow/pull/46113 + - patches/0007-GH-46111-C-CI-Fix-boost-1.88-on-MinGW-46113.patch + # backport https://github.com/apache/arrow/pull/46160 + - patches/0008-GH-46159-CI-C-Stop-using-possibly-missing-boost-proc.patch + - patches/0009-Bodo-Changes.patch # testing-submodules not part of release tarball - git_url: https://github.com/apache/arrow-testing.git @@ -38,10 +45,15 @@ source: folder: cpp/submodules/parquet-testing build: - number: 8 + number: 0 # for cuda support, building with one version is enough to be compatible with # all later versions, since arrow is only using libcuda, and not libcudart. skip: true # [cuda_compiler_version not in ("None", cuda_compiler_version_min)] + # arrow promises API- & ABI-compatibility along SemVer, see #1096 + script_env: + # Necessary for sccache to find AWS + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY requirements: build: @@ -63,6 +75,7 @@ requirements: - make # [unix] # where orc ends up looking (because during build, CONDA_PREFIX==BUILD_PREFIX) - tzdata + - sccache host: # for required dependencies, see # https://github.com/apache/arrow/blob/apache-arrow-11.0.0/cpp/cmake_modules/ThirdpartyToolchain.cmake#L46-L75 @@ -159,6 +172,8 @@ outputs: - {{ pin_subpackage("libarrow-gandiva", exact=True) }} - {{ pin_subpackage("libarrow-substrait", exact=True) }} - {{ pin_subpackage("libparquet", exact=True) }} + - {{ pin_subpackage("arrow-utils", exact=True) }} + - {{ pin_subpackage("parquet-utils", exact=True) }} run: - {{ pin_subpackage("libarrow", exact=True) }} - {{ pin_subpackage("libarrow-acero", exact=True) }} @@ -168,6 +183,8 @@ outputs: - {{ pin_subpackage("libarrow-gandiva", exact=True) }} - {{ pin_subpackage("libarrow-substrait", exact=True) }} - {{ pin_subpackage("libparquet", exact=True) }} + - {{ pin_subpackage("arrow-utils", exact=True) }} + - {{ pin_subpackage("parquet-utils", exact=True) }} test: commands: # absence of test bits @@ -324,6 +341,9 @@ outputs: - {{ compiler("cxx") }} host: - {{ pin_subpackage("libarrow", exact=True) }} + - libabseil # [osx] + - libprotobuf # [osx] + - libopentelemetry-cpp # [osx] run: - {{ pin_subpackage("libarrow", exact=True) }} # run-constraints handled by libarrow, since we depend on it with exact=True @@ -371,6 +391,9 @@ outputs: - {{ pin_subpackage("libarrow", exact=True) }} - {{ pin_subpackage("libarrow-acero", exact=True) }} - {{ pin_subpackage("libparquet", exact=True) }} + - libabseil # [osx] + - libprotobuf # [osx] + - libopentelemetry-cpp # [osx] run: - {{ pin_subpackage("libarrow", exact=True) }} - {{ pin_subpackage("libarrow-acero", exact=True) }} @@ -424,6 +447,7 @@ outputs: - libabseil - libgrpc - libprotobuf + - libopentelemetry-cpp # [osx] run: - {{ pin_subpackage("libarrow", exact=True) }} # run-constraints handled by libarrow, since we depend on it with exact=True @@ -475,6 +499,8 @@ outputs: - {{ pin_subpackage("libarrow-flight", exact=True) }} - libabseil - libprotobuf + - libgrpc # [osx] + - libopentelemetry-cpp # [osx] run: - {{ pin_subpackage("libarrow", exact=True) }} - {{ pin_subpackage("libarrow-flight", exact=True) }} @@ -628,6 +654,9 @@ outputs: - {{ pin_subpackage("libarrow", exact=True) }} - openssl - thrift-cpp + - libabseil # [osx] + - libprotobuf # [osx] + - libopentelemetry-cpp # [osx] run: - {{ pin_subpackage("libarrow", exact=True) }} # run-constraints handled by libarrow, since we depend on it with exact=True @@ -655,6 +684,82 @@ outputs: - LICENSE.txt summary: C++ libraries for Apache Parquet + - name: parquet-utils + script: install-libarrow.sh # [unix] + script: install-libarrow.bat # [win] + version: {{ version }} + build: + string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} + requirements: + build: + - cmake <4 + - ninja + # for strong run-exports + - {{ stdlib("c") }} + - {{ compiler("c") }} + - {{ compiler("cxx") }} + host: + - {{ pin_subpackage("libarrow", exact=True) }} + - {{ pin_subpackage("libparquet", exact=True) }} + run: + - {{ pin_subpackage("libarrow", exact=True) }} + - {{ pin_subpackage("libparquet", exact=True) }} + + test: + commands: + # bin + - test -f $PREFIX/bin/parquet-dump-arrow-statistics # [unix] + - test -f $PREFIX/bin/parquet-dump-footer # [unix] + - test -f $PREFIX/bin/parquet-dump-schema # [unix] + - test -f $PREFIX/bin/parquet-reader # [unix] + - test -f $PREFIX/bin/parquet-scan # [unix] + - if not exist %LIBRARY_BIN%\parquet-dump-arrow-statistics.exe exit 1 # [win] + - if not exist %LIBRARY_BIN%\parquet-dump-footer.exe exit 1 # [win] + - if not exist %LIBRARY_BIN%\parquet-dump-schema.exe exit 1 # [win] + - if not exist %LIBRARY_BIN%\parquet-reader.exe exit 1 # [win] + - if not exist %LIBRARY_BIN%\parquet-scan.exe exit 1 # [win] + + about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: + - LICENSE.txt + summary: Executables for inspecting Apache Parquet files + + - name: arrow-utils + script: install-libarrow.sh # [unix] + script: install-libarrow.bat # [win] + version: {{ version }} + build: + string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} + requirements: + build: + - cmake <4 + - ninja + # for strong run-exports + - {{ stdlib("c") }} + - {{ compiler("c") }} + - {{ compiler("cxx") }} + host: + - {{ pin_subpackage("libarrow", exact=True) }} + run: + - {{ pin_subpackage("libarrow", exact=True) }} + + test: + commands: + # bin + - test -f $PREFIX/bin/arrow-file-to-stream # [unix] + - test -f $PREFIX/bin/arrow-stream-to-file # [unix] + - if not exist %LIBRARY_BIN%\arrow-file-to-stream.exe exit 1 # [win] + - if not exist %LIBRARY_BIN%\arrow-stream-to-file.exe exit 1 # [win] + + about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: + - LICENSE.txt + summary: Executables for manipulating Apache arrow files + about: home: http://github.com/apache/arrow license: Apache-2.0 diff --git a/recipe/patches/0001-disable-gcsfs_test.patch b/recipe/patches/0001-disable-gcsfs_test.patch index c8541189b..76bf99f3e 100644 --- a/recipe/patches/0001-disable-gcsfs_test.patch +++ b/recipe/patches/0001-disable-gcsfs_test.patch @@ -1,7 +1,7 @@ From e35ee3c6edc44b9bc3bd998afc4b3ad3227b6a56 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sat, 2 Nov 2024 15:41:34 +1100 -Subject: [PATCH 1/5] disable gcsfs_test +Subject: [PATCH 1/8] disable gcsfs_test it cannot work unless we package https://github.com/googleapis/storage-testbench, which however has extremely tight dependencies on protobuf etc., making it very diff --git a/recipe/patches/0002-skip-NonExistentBucket-test-on-osx.patch b/recipe/patches/0002-skip-NonExistentBucket-test-on-osx.patch index d8cef70cb..ae729bb71 100644 --- a/recipe/patches/0002-skip-NonExistentBucket-test-on-osx.patch +++ b/recipe/patches/0002-skip-NonExistentBucket-test-on-osx.patch @@ -1,7 +1,7 @@ From f9e6ab1ac71c1bc69905adc397302d14bf0e92f1 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 24 Nov 2024 20:22:35 +1100 -Subject: [PATCH 2/5] skip NonExistentBucket test on osx +Subject: [PATCH 2/8] skip NonExistentBucket test on osx --- cpp/src/arrow/filesystem/s3fs_test.cc | 4 ++++ diff --git a/recipe/patches/0003-add-missing-include-in-cpp-src-parquet-size_statisti.patch b/recipe/patches/0003-add-missing-include-in-cpp-src-parquet-size_statisti.patch index 02b61f997..04dab0b06 100644 --- a/recipe/patches/0003-add-missing-include-in-cpp-src-parquet-size_statisti.patch +++ b/recipe/patches/0003-add-missing-include-in-cpp-src-parquet-size_statisti.patch @@ -1,7 +1,7 @@ From cf86fabb1dda76ba45b97e84d0566540e3bcc242 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 17 Feb 2025 11:56:43 +1100 -Subject: [PATCH 3/5] add missing include in cpp/src/parquet/size_statistics.cc +Subject: [PATCH 3/8] add missing include in cpp/src/parquet/size_statistics.cc Suggested-By: Bryce Mecum --- diff --git a/recipe/patches/0004-GH-45225-C-Upgrade-ORC-to-2.1.0-45226.patch b/recipe/patches/0004-GH-45225-C-Upgrade-ORC-to-2.1.0-45226.patch index 1882bb38f..f3b412e3a 100644 --- a/recipe/patches/0004-GH-45225-C-Upgrade-ORC-to-2.1.0-45226.patch +++ b/recipe/patches/0004-GH-45225-C-Upgrade-ORC-to-2.1.0-45226.patch @@ -1,7 +1,7 @@ From c403b19c68babb050b98a0a5203153dd36ab1db3 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Tue, 14 Jan 2025 10:25:08 +0800 -Subject: [PATCH 4/5] GH-45225: [C++] Upgrade ORC to 2.1.0 (#45226) +Subject: [PATCH 4/8] GH-45225: [C++] Upgrade ORC to 2.1.0 (#45226) ### Rationale for this change diff --git a/recipe/patches/0005-MINOR-C-Fix-OTel-1.19-compat-after-silent-breaking-c.patch b/recipe/patches/0005-MINOR-C-Fix-OTel-1.19-compat-after-silent-breaking-c.patch index 2c8621bbb..1f75d1377 100644 --- a/recipe/patches/0005-MINOR-C-Fix-OTel-1.19-compat-after-silent-breaking-c.patch +++ b/recipe/patches/0005-MINOR-C-Fix-OTel-1.19-compat-after-silent-breaking-c.patch @@ -1,7 +1,7 @@ From 4d2513bec967c2a671f96039fbc1e91a627a36fa Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 19 Mar 2025 09:34:50 +0900 -Subject: [PATCH 5/5] MINOR: [C++] Fix OTel 1.19 compat after silent breaking +Subject: [PATCH 5/8] MINOR: [C++] Fix OTel 1.19 compat after silent breaking change --- diff --git a/recipe/patches/0006-GH-46134-CI-C-Explicit-conversion-of-possible-absl-s.patch b/recipe/patches/0006-GH-46134-CI-C-Explicit-conversion-of-possible-absl-s.patch new file mode 100644 index 000000000..616990e9e --- /dev/null +++ b/recipe/patches/0006-GH-46134-CI-C-Explicit-conversion-of-possible-absl-s.patch @@ -0,0 +1,78 @@ +From 78e297360e6a335904fabed75ec0b2dd0cf7ade4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= +Date: Mon, 14 Apr 2025 23:46:23 +0200 +Subject: [PATCH 6/8] GH-46134: [CI][C++] Explicit conversion of possible + `absl::string_view` on protobuf methods to `std::string` (#46136) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +### Rationale for this change + +Protobuf v30.2 uses `absl::string_view` instead of `const char*` as seen here: +https://github.com/protocolbuffers/protobuf/commit/a9ad51f5b6a19eacc934bcb51db6282ec1fabb8c +This is breaking on our CI. + +### What changes are included in this PR? + +Explicitly convert `google::protobuf::EnumValueDescriptor::name()` and `google::protobuf::Message::descriptor()->full_name()` from `absl::string_view` to `std::string` + +### Are these changes tested? + +Via CI, MinGW CI jobs use protobuf v30.2 and those are fixed with this change. + +### Are there any user-facing changes? + +No + +* GitHub Issue: #46134 + +Authored-by: Raúl Cumplido +Signed-off-by: Sutou Kouhei +--- + cpp/src/arrow/engine/substrait/expression_internal.cc | 2 +- + cpp/src/arrow/engine/substrait/serde.cc | 4 ++-- + cpp/src/arrow/engine/substrait/util_internal.cc | 2 +- + 3 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/cpp/src/arrow/engine/substrait/expression_internal.cc b/cpp/src/arrow/engine/substrait/expression_internal.cc +index 16dcba8944..02e453fbff 100644 +--- a/cpp/src/arrow/engine/substrait/expression_internal.cc ++++ b/cpp/src/arrow/engine/substrait/expression_internal.cc +@@ -142,7 +142,7 @@ std::string EnumToString(int value, const google::protobuf::EnumDescriptor* desc + if (value_desc == nullptr) { + return "unknown"; + } +- return value_desc->name(); ++ return std::string(value_desc->name()); + } + + Result FromProto(const substrait::Expression::ReferenceSegment* ref, +diff --git a/cpp/src/arrow/engine/substrait/serde.cc b/cpp/src/arrow/engine/substrait/serde.cc +index 6b4c05a3b1..db2dcb5928 100644 +--- a/cpp/src/arrow/engine/substrait/serde.cc ++++ b/cpp/src/arrow/engine/substrait/serde.cc +@@ -62,8 +62,8 @@ Status ParseFromBufferImpl(const Buffer& buf, const std::string& full_name, + template + Result ParseFromBuffer(const Buffer& buf) { + Message message; +- ARROW_RETURN_NOT_OK( +- ParseFromBufferImpl(buf, Message::descriptor()->full_name(), &message)); ++ ARROW_RETURN_NOT_OK(ParseFromBufferImpl( ++ buf, std::string(Message::descriptor()->full_name()), &message)); + return message; + } + +diff --git a/cpp/src/arrow/engine/substrait/util_internal.cc b/cpp/src/arrow/engine/substrait/util_internal.cc +index 89034784ab..35e3baf94a 100644 +--- a/cpp/src/arrow/engine/substrait/util_internal.cc ++++ b/cpp/src/arrow/engine/substrait/util_internal.cc +@@ -30,7 +30,7 @@ std::string EnumToString(int value, const google::protobuf::EnumDescriptor& desc + if (value_desc == nullptr) { + return "unknown"; + } +- return value_desc->name(); ++ return std::string(value_desc->name()); + } + + std::unique_ptr CreateVersion() { diff --git a/recipe/patches/0007-GH-46111-C-CI-Fix-boost-1.88-on-MinGW-46113.patch b/recipe/patches/0007-GH-46111-C-CI-Fix-boost-1.88-on-MinGW-46113.patch new file mode 100644 index 000000000..d25f734a8 --- /dev/null +++ b/recipe/patches/0007-GH-46111-C-CI-Fix-boost-1.88-on-MinGW-46113.patch @@ -0,0 +1,61 @@ +From ff5a5f41d654b840670835b4f28d6ee630b2d12e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= +Date: Mon, 14 Apr 2025 15:04:44 +0200 +Subject: [PATCH 7/8] GH-46111: [C++][CI] Fix boost 1.88 on MinGW (#46113) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +### Rationale for this change + +Boost has remove their `include/boost/process/v1.hpp` + +### What changes are included in this PR? + +Instead of using their old include file (which was a wrapper for the different includes) add the individual includes. + +### Are these changes tested? + +Via CI. + +### Are there any user-facing changes? + +No +* GitHub Issue: #46111 + +Authored-by: Raúl Cumplido +Signed-off-by: Raúl Cumplido +--- + cpp/src/arrow/testing/process.cc | 19 ++++++++++++++++++- + 1 file changed, 18 insertions(+), 1 deletion(-) + +diff --git a/cpp/src/arrow/testing/process.cc b/cpp/src/arrow/testing/process.cc +index 5f04b31aa1..b1c4fa5550 100644 +--- a/cpp/src/arrow/testing/process.cc ++++ b/cpp/src/arrow/testing/process.cc +@@ -57,7 +57,24 @@ + # define BOOST_USE_WINDOWS_H = 1 + # endif + # ifdef BOOST_PROCESS_HAVE_V1 +-# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include + # else + # include + # endif diff --git a/recipe/patches/0008-GH-46159-CI-C-Stop-using-possibly-missing-boost-proc.patch b/recipe/patches/0008-GH-46159-CI-C-Stop-using-possibly-missing-boost-proc.patch new file mode 100644 index 000000000..7d29e887d --- /dev/null +++ b/recipe/patches/0008-GH-46159-CI-C-Stop-using-possibly-missing-boost-proc.patch @@ -0,0 +1,64 @@ +From c36208c83c5b1f8513e8cad7ecd0fb7a32ea414c Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= +Date: Thu, 17 Apr 2025 12:06:54 +0200 +Subject: [PATCH 8/8] GH-46159: [CI][C++] Stop using possibly missing + boost/process/v2.hpp on boost 1.88 and use individual includes (#46160) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +### Rationale for this change + +Similar to https://github.com/apache/arrow/pull/46113 but for boost v2. +The files `v1.hpp` and `v2.hpp` have been removed from `include/boost/process` on `1.88`: +https://github.com/boostorg/process/tree/develop/include/boost/process +but were available on `1.87`: +https://github.com/boostorg/process/blob/boost-1.87.0/include/boost/process/v1.hpp +https://github.com/boostorg/process/blob/boost-1.87.0/include/boost/process/v2.hpp + +### What changes are included in this PR? + +Stop using include for possible missing file and add individual includes + +### Are these changes tested? + +Via CI + +### Are there any user-facing changes? + +No + +* GitHub Issue: #46159 + +Authored-by: Raúl Cumplido +Signed-off-by: Jacob Wujciak-Jens +--- + cpp/src/arrow/testing/process.cc | 13 ++++++++++--- + 1 file changed, 10 insertions(+), 3 deletions(-) + +diff --git a/cpp/src/arrow/testing/process.cc b/cpp/src/arrow/testing/process.cc +index b1c4fa5550..45b92af0dd 100644 +--- a/cpp/src/arrow/testing/process.cc ++++ b/cpp/src/arrow/testing/process.cc +@@ -39,11 +39,18 @@ + # ifdef __APPLE__ + # include + # endif +-# include + # include +-# else +-# include + # endif ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include ++# include + # include + # else + // We need BOOST_USE_WINDOWS_H definition with MinGW when we use diff --git a/recipe/patches/0009-Bodo-Changes.patch b/recipe/patches/0009-Bodo-Changes.patch new file mode 100644 index 000000000..f23b64c6b --- /dev/null +++ b/recipe/patches/0009-Bodo-Changes.patch @@ -0,0 +1,278 @@ +diff --git a/cpp/src/arrow/dataset/file_parquet.cc b/cpp/src/arrow/dataset/file_parquet.cc +index 1f8b6cc488..6ac6642fdb 100644 +--- a/cpp/src/arrow/dataset/file_parquet.cc ++++ b/cpp/src/arrow/dataset/file_parquet.cc +@@ -26,16 +26,23 @@ + + #include "arrow/compute/cast.h" + #include "arrow/compute/exec.h" ++#include "arrow/dataset/dataset.h" + #include "arrow/dataset/dataset_internal.h" + #include "arrow/dataset/parquet_encryption_config.h" + #include "arrow/dataset/scanner.h" + #include "arrow/filesystem/path_util.h" ++#include "arrow/memory_pool.h" ++#include "arrow/record_batch.h" ++#include "arrow/result.h" + #include "arrow/table.h" ++#include "arrow/type.h" ++#include "arrow/type_fwd.h" + #include "arrow/util/checked_cast.h" + #include "arrow/util/future.h" + #include "arrow/util/iterator.h" + #include "arrow/util/logging.h" + #include "arrow/util/range.h" ++#include "arrow/util/thread_pool.h" + #include "arrow/util/tracing_internal.h" + #include "parquet/arrow/reader.h" + #include "parquet/arrow/schema.h" +@@ -555,6 +562,68 @@ Future> ParquetFileFormat::GetReader + }); + } + ++struct CastingGenerator { ++ CastingGenerator(RecordBatchGenerator source, std::shared_ptr final_schema, ++ const std::unordered_set& cols_to_skip, ++ MemoryPool* pool = default_memory_pool()) ++ : source_(source), ++ final_schema_(final_schema), ++ cols_to_skip_(cols_to_skip), ++ exec_ctx(std::make_shared(pool)) {} ++ ++ Future> operator()() { ++ return this->source_().Then([this](const std::shared_ptr& next) ++ -> Result> { ++ if (IsIterationEnd(next) || this->final_schema_ == nullptr) { ++ return next; ++ } ++ std::vector> out_cols; ++ std::vector> out_schema_fields; ++ ++ bool changed = false; ++ for (const auto& field : this->final_schema_->fields()) { ++ FieldRef field_ref = FieldRef(field->name()); ++ ARROW_ASSIGN_OR_RAISE(std::shared_ptr column, ++ field_ref.GetOneOrNone(*next)); ++ if (column) { ++ if (this->cols_to_skip_.count(field->name())) { ++ // Maintain the original input type. ++ out_schema_fields.emplace_back(field->WithType(column->type())); ++ out_cols.emplace_back(std::move(column)); ++ continue; ++ } ++ if (!column->type()->Equals(field->type())) { ++ // Referenced field was present but didn't have the expected type. ++ ARROW_ASSIGN_OR_RAISE( ++ auto converted, ++ compute::Cast(column, field->type(), compute::CastOptions::Safe(), ++ this->exec_ctx.get())); ++ column = converted.make_array(); ++ changed = true; ++ } ++ out_schema_fields.emplace_back(field->Copy()); ++ out_cols.emplace_back(std::move(column)); ++ // XXX Do we need to handle the else case? What happens when the column ++ // doesn't exist, e.g. all null or all the same value? ++ } ++ } ++ ++ if (changed) { ++ return RecordBatch::Make(std::make_shared(std::move(out_schema_fields), ++ next->schema()->metadata()), ++ next->num_rows(), std::move(out_cols)); ++ } else { ++ return next; ++ } ++ }); ++ } ++ ++ RecordBatchGenerator source_; ++ std::shared_ptr final_schema_; ++ const std::unordered_set& cols_to_skip_; ++ std::shared_ptr exec_ctx; ++}; ++ + struct SlicingGenerator { + SlicingGenerator(RecordBatchGenerator source, int64_t batch_size) + : state(std::make_shared(source, batch_size)) {} +@@ -617,6 +686,9 @@ Result ParquetFileFormat::ScanBatchesAsync( + [this, options, parquet_fragment, pre_filtered, + row_groups](const std::shared_ptr& reader) mutable + -> Result { ++ // Since we already do the batching through the SlicingGenerator, we don't need the ++ // reader to batch its output. ++ reader->set_batch_size(std::numeric_limits::max()); + // Ensure that parquet_fragment has FileMetaData + RETURN_NOT_OK(parquet_fragment->EnsureCompleteMetadata(reader.get())); + if (!pre_filtered) { +@@ -633,12 +705,24 @@ Result ParquetFileFormat::ScanBatchesAsync( + kParquetTypeName, options.get(), default_fragment_scan_options)); + int batch_readahead = options->batch_readahead; + int64_t rows_to_readahead = batch_readahead * options->batch_size; +- ARROW_ASSIGN_OR_RAISE(auto generator, +- reader->GetRecordBatchGenerator( +- reader, row_groups, column_projection, +- ::arrow::internal::GetCpuThreadPool(), rows_to_readahead)); ++ // Modified this to pass the executor in scan_options instead of always using the ++ // default CPU thread pool. ++ // XXX Should we get it from options->fragment_scan_options instead?? ++ auto cpu_executor = options->exec_context.executor() ++ ? options->exec_context.executor() ++ : ::arrow::internal::GetCpuThreadPool(); ++ ARROW_ASSIGN_OR_RAISE(auto generator, reader->GetRecordBatchGenerator( ++ reader, row_groups, column_projection, ++ cpu_executor, rows_to_readahead)); ++ // We need to skip casting the dictionary columns since the dataset_schema doesn't ++ // have the dictionary-encoding information. Parquet reader will return them with the ++ // dictionary type, which is what we eventually want. ++ const std::unordered_set& dict_cols = ++ parquet_fragment->parquet_format_.reader_options.dict_columns; ++ RecordBatchGenerator casted = CastingGenerator( ++ std::move(generator), options->dataset_schema, dict_cols, options->pool); + RecordBatchGenerator sliced = +- SlicingGenerator(std::move(generator), options->batch_size); ++ SlicingGenerator(std::move(casted), options->batch_size); + if (batch_readahead == 0) { + return sliced; + } +diff --git a/cpp/src/arrow/dataset/scanner.cc b/cpp/src/arrow/dataset/scanner.cc +index a856a792a2..5c10dfc6ac 100644 +--- a/cpp/src/arrow/dataset/scanner.cc ++++ b/cpp/src/arrow/dataset/scanner.cc +@@ -355,8 +355,10 @@ class OneShotFragment : public Fragment { + ARROW_ASSIGN_OR_RAISE( + auto background_gen, + MakeBackgroundGenerator(std::move(batch_it_), options->io_context.executor())); +- return MakeTransferredGenerator(std::move(background_gen), +- ::arrow::internal::GetCpuThreadPool()); ++ auto cpu_executor = options->exec_context.executor() ++ ? options->exec_context.executor() ++ : ::arrow::internal::GetCpuThreadPool(); ++ return MakeTransferredGenerator(std::move(background_gen), cpu_executor); + } + std::string type_name() const override { return "one-shot"; } + +@@ -382,7 +384,7 @@ Result AsyncScanner::ScanBatches() { + [this](::arrow::internal::Executor* executor) { + return ScanBatchesAsync(executor); + }, +- scan_options_->use_threads); ++ scan_options_->use_threads, this->async_cpu_executor()); + } + + Result AsyncScanner::ScanBatchesUnordered() { +@@ -390,7 +392,7 @@ Result AsyncScanner::ScanBatchesUnordered() { + [this](::arrow::internal::Executor* executor) { + return ScanBatchesUnorderedAsync(executor); + }, +- scan_options_->use_threads); ++ scan_options_->use_threads, this->async_cpu_executor()); + } + + Result> AsyncScanner::ToTable() { +@@ -400,7 +402,7 @@ Result> AsyncScanner::ToTable() { + } + + Result AsyncScanner::ScanBatchesUnorderedAsync() { +- return ScanBatchesUnorderedAsync(::arrow::internal::GetCpuThreadPool(), ++ return ScanBatchesUnorderedAsync(this->async_cpu_executor(), + /*sequence_fragments=*/false); + } + +@@ -601,7 +603,7 @@ Result> AsyncScanner::Head(int64_t num_rows) { + } + + Result AsyncScanner::ScanBatchesAsync() { +- return ScanBatchesAsync(::arrow::internal::GetCpuThreadPool()); ++ return ScanBatchesAsync(this->async_cpu_executor()); + } + + Result AsyncScanner::ScanBatchesAsync( +@@ -778,7 +780,7 @@ Future AsyncScanner::CountRowsAsync(Executor* executor) { + } + + Future AsyncScanner::CountRowsAsync() { +- return CountRowsAsync(::arrow::internal::GetCpuThreadPool()); ++ return CountRowsAsync(this->async_cpu_executor()); + } + + Result AsyncScanner::CountRows() { +diff --git a/cpp/src/arrow/dataset/scanner.h b/cpp/src/arrow/dataset/scanner.h +index d2de267897..1c605c1bf2 100644 +--- a/cpp/src/arrow/dataset/scanner.h ++++ b/cpp/src/arrow/dataset/scanner.h +@@ -107,6 +107,11 @@ struct ARROW_DS_EXPORT ScanOptions { + /// Note: The IOContext executor will be ignored if use_threads is set to false + io::IOContext io_context; + ++ /// ExecContext for any CPU tasks ++ /// ++ /// Note: The ExecContext executor will be ignored if use_threads is set to false ++ compute::ExecContext exec_context; ++ + /// If true the scanner will scan in parallel + /// + /// Note: If true, this will use threads from both the cpu_executor and the +@@ -442,6 +447,11 @@ class ARROW_DS_EXPORT Scanner { + TaggedRecordBatchIterator scan); + + const std::shared_ptr scan_options_; ++ ++ ::arrow::internal::Executor* async_cpu_executor() const { ++ return scan_options_->exec_context.executor() ? scan_options_->exec_context.executor() ++ : ::arrow::internal::GetCpuThreadPool(); ++ } + }; + + /// \brief ScannerBuilder is a factory class to construct a Scanner. It is used +diff --git a/cpp/src/arrow/util/thread_pool.h b/cpp/src/arrow/util/thread_pool.h +index 44b1e227b0..218edc60ca 100644 +--- a/cpp/src/arrow/util/thread_pool.h ++++ b/cpp/src/arrow/util/thread_pool.h +@@ -20,6 +20,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -591,6 +592,21 @@ typename Fut::SyncType RunSynchronously(FnOnce get_future, + } + } + ++template ++Iterator IterateSynchronously( ++ FnOnce()>>(Executor*)> get_gen, bool use_threads, ++ Executor* executor) { ++ if (use_threads) { ++ auto maybe_gen = std::move(get_gen)(executor); ++ if (!maybe_gen.ok()) { ++ return MakeErrorIterator(maybe_gen.status()); ++ } ++ return MakeGeneratorIterator(*maybe_gen); ++ } else { ++ return SerialExecutor::IterateGenerator(std::move(get_gen)); ++ } ++} ++ + /// \brief Potentially iterate an async generator serially (if use_threads is false) + /// \see IterateGenerator + /// +@@ -605,15 +621,7 @@ typename Fut::SyncType RunSynchronously(FnOnce get_future, + template + Iterator IterateSynchronously( + FnOnce()>>(Executor*)> get_gen, bool use_threads) { +- if (use_threads) { +- auto maybe_gen = std::move(get_gen)(GetCpuThreadPool()); +- if (!maybe_gen.ok()) { +- return MakeErrorIterator(maybe_gen.status()); +- } +- return MakeGeneratorIterator(*maybe_gen); +- } else { +- return SerialExecutor::IterateGenerator(std::move(get_gen)); +- } ++ return IterateSynchronously(std::move(get_gen), use_threads, GetCpuThreadPool()); + } + + } // namespace internal