DX-114802: Fix Mac build to actually use VCPKG for LLVM. (#16)

* Change release naming scheme to add commit ids.

Add action url to release

Add action url to release

* Mac vcpkg root setting.

Old grpc uninstall causing error?

fix vcpkg root error

Fix vcpgkg llvm for mac.

vcpkg

vcpkg

Go back to brew for build tools. vcpkg just for llvm

Update macos path

fix path to system files

vcpkg

vcpkg

vcpkg

fix re2 dependency

Make JNI CMake args more robust

- Check if protobuf_ep-install exists before adding Protobuf_ROOT
- Check if re2_ep-install exists before adding re2_ROOT
- This handles cases where system dependencies are used instead of bundled

Use vcpkg RE2 instead of bundled RE2 for macOS Gandiva build

vcpkg installs RE2 as a dependency of LLVM. When we use bundled RE2,
the Arrow C++ build compiles Gandiva against vcpkg's RE2 headers
(which use std::string_view API) but links against bundled RE2
(which uses StringPiece API), causing linker errors.

This change uses vcpkg's RE2 for both compilation and linking when
vcpkg LLVM is used, ensuring ABI compatibility.

* Add username to tag

fix release notes

* Fix merge error

* Fix quotes

* fix env var addressing

Author: lriggs

.github/workflows/jarbuild.yml

@@ -228,12 +228,13 @@ jobs:
         with:
           repository: Microsoft/vcpkg
           path: arrow/vcpkg
+          fetch-depth: 0
       - name: Install vcpkg
         run: |
           cd arrow/vcpkg
           ./bootstrap-vcpkg.sh
-          echo "VCPKG_ROOT=${PWD}/arrow/vcpkg" >> ${GITHUB_ENV}
-          echo "${PWD}/arrow/vcpkg" >> ${GITHUB_PATH}
+          echo "VCPKG_ROOT_LOCAL=${PWD}" >> ${GITHUB_ENV}
+          echo "${PWD}" >> ${GITHUB_PATH}
       - name: Clean up disk space
         run: |
           echo "=== Free disk space before cleanup ==="
@@ -284,25 +285,28 @@ jobs:
           brew bundle --file=arrow/cpp/Brewfile
 
           # Clean up any existing LLVM installations in favor of vcpkg.
-          brew uninstall llvm || :
+          # Need to uninstall all versioned LLVM packages (llvm@18, llvm@17, etc.)
+          for llvm_pkg in $(brew list | grep -E '^llvm(@[0-9]+)?$'); do
+            brew uninstall "${llvm_pkg}" || :
+          done
 
           # We want to link aws-sdk-cpp statically but Homebrew's
           # aws-sdk-cpp provides only shared library. If we have
           # Homebrew's aws-sdk-cpp, our build mix Homebrew's
           # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
           # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
-          brew uninstall aws-sdk-cpp
+          brew uninstall aws-sdk-cpp || :
           # We want to use bundled RE2 for static linking. If
           # Homebrew's RE2 is installed, its header file may be used.
           # We uninstall Homebrew's RE2 to ensure using bundled RE2.
           brew uninstall grpc || : # gRPC depends on RE2
           brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
-          brew uninstall re2
+          brew uninstall re2 || :
           # We want to use bundled Protobuf for static linking. If
           # Homebrew's Protobuf is installed, its library file may be
           # used on test  We uninstall Homebrew's Protobuf to ensure using
           # bundled Protobuf.
-          brew uninstall protobuf
+          brew uninstall protobuf || :
 
           echo ""
           echo "=== Free disk space before LLVM build ==="
@@ -312,7 +316,8 @@ jobs:
           # Use vcpkg to install LLVM.
           vcpkg install \
             --clean-after-build \
-            --x-install-root=${VCPKG_ROOT}/installed \
+            --vcpkg-root=${VCPKG_ROOT_LOCAL} \
+            --x-install-root=${VCPKG_ROOT_LOCAL}/installed \
             --x-manifest-root=arrow/ci/vcpkg \
             --overlay-ports=arrow/ci/vcpkg/overlay/llvm/ \
             --x-feature=gandiva-llvm
@@ -558,7 +563,7 @@ jobs:
           # GH-499: How to create release notes?
           echo "Creating release: ${{ steps.commit_ids.outputs.release_tag }}"
           gh release create "${{ steps.commit_ids.outputs.release_tag }}" \
-            -n "Action URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
+            -n "Release ${{ steps.commit_ids.outputs.release_name }} RC${{ steps.commit_ids.outputs.rc }}<br>Triggered by: ${{ github.actor }}<br>Action URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID<br>arrow_branch: ${{github.event.inputs.ARROW_BRANCH}}<br>arrow_repo: ${{github.event.inputs.ARROW_REPO}}<br>release_tag_name: ${{github.event.inputs.RELEASE_TAG_NAME}}<br>arrow-java branch: ${{github.ref_name}}" \
             --prerelease \
             --repo ${GITHUB_REPOSITORY} \
             --title "Apache Arrow Java ${{ steps.commit_ids.outputs.version }} RC${{ steps.commit_ids.outputs.rc }} (arrow-java: ${{ steps.commit_ids.outputs.arrow_java_commit }}, arrow: ${{ steps.commit_ids.outputs.arrow_commit }})"

ci/scripts/jni_macos_build.sh

@@ -78,6 +78,55 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
+# Determine vcpkg triplet based on architecture
+vcpkg_arch="$(arch)"
+case "${vcpkg_arch}" in
+arm64)
+  vcpkg_triplet="arm64-osx"
+  ;;
+i386|x86_64)
+  vcpkg_triplet="x64-osx"
+  ;;
+*)
+  vcpkg_triplet="arm64-osx"
+  ;;
+esac
+
+# Set LLVM_DIR to point to vcpkg-installed LLVM if VCPKG_ROOT_LOCAL is set
+llvm_dir_arg=""
+gandiva_cxx_flags=""
+osx_sysroot_arg=""
+re2_source_arg="-Dre2_SOURCE=BUNDLED"
+if [ -n "${VCPKG_ROOT_LOCAL:-}" ]; then
+  vcpkg_installed="${VCPKG_ROOT_LOCAL}/installed/${vcpkg_triplet}"
+  llvm_cmake_dir="${vcpkg_installed}/share/llvm"
+  if [ -d "${llvm_cmake_dir}" ]; then
+    llvm_dir_arg="-DLLVM_DIR=${llvm_cmake_dir}"
+
+    # vcpkg's clang needs to know where to find system headers
+    # Arrow's GandivaAddBitcode.cmake uses CMAKE_OSX_SYSROOT to set SDKROOT env var
+    sdk_path="$(xcrun --show-sdk-path)"
+    if [ -d "${sdk_path}" ]; then
+      osx_sysroot_arg="-DCMAKE_OSX_SYSROOT=${sdk_path}"
+    fi
+
+    # Also pass the C++ standard library include path via ARROW_GANDIVA_PC_CXX_FLAGS
+    xcode_path="$(xcode-select -p)"
+    cxx_include_path="${xcode_path}/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1"
+    if [ -d "${cxx_include_path}" ]; then
+      gandiva_cxx_flags="-DARROW_GANDIVA_PC_CXX_FLAGS=-stdlib=libc++;-isystem;${cxx_include_path}"
+    fi
+
+    # Use vcpkg's RE2 since it's installed as a dependency of LLVM
+    # This ensures ABI compatibility - vcpkg's RE2 uses std::string_view API
+    # which matches what vcpkg's LLVM and Abseil expect
+    re2_cmake_dir="${vcpkg_installed}/share/re2"
+    if [ -d "${re2_cmake_dir}" ]; then
+      re2_source_arg="-Dre2_ROOT=${vcpkg_installed}"
+    fi
+  fi
+fi
+
 cmake \
   -S "${arrow_dir}/cpp" \
   -B "${build_dir}/cpp" \
@@ -100,10 +149,13 @@ cmake \
   -DCMAKE_INSTALL_PREFIX="${install_dir}" \
   -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \
   -DGTest_SOURCE=BUNDLED \
+  ${llvm_dir_arg} \
+  ${osx_sysroot_arg} \
+  ${gandiva_cxx_flags} \
   -DPARQUET_BUILD_EXAMPLES=OFF \
   -DPARQUET_BUILD_EXECUTABLES=OFF \
   -DPARQUET_REQUIRE_ENCRYPTION=OFF \
-  -Dre2_SOURCE=BUNDLED \
+  ${re2_source_arg} \
   -GNinja
 cmake --build "${build_dir}/cpp" --target install
 github_actions_group_end
@@ -125,7 +177,27 @@ if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then
   github_actions_group_end
 fi
 
-export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install"
+# Pass paths to dependencies so the JNI build can find them
+# Build up the JNI CMake args based on what's available
+jni_cmake_args="${llvm_dir_arg}"
+
+# Add Protobuf path if bundled, otherwise CMake will find system Protobuf
+if [ -d "${build_dir}/cpp/protobuf_ep-install" ]; then
+  jni_cmake_args="${jni_cmake_args} -DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install"
+fi
+
+# RE2 path for the JNI build - prefer vcpkg's RE2 if we used it for the C++ build,
+# otherwise fall back to bundled RE2 if available
+if [ -n "${VCPKG_ROOT_LOCAL:-}" ]; then
+  vcpkg_re2_dir="${VCPKG_ROOT_LOCAL}/installed/${vcpkg_triplet}"
+  if [ -d "${vcpkg_re2_dir}/share/re2" ]; then
+    jni_cmake_args="${jni_cmake_args} -Dre2_ROOT=${vcpkg_re2_dir}"
+  fi
+elif [ -d "${build_dir}/cpp/re2_ep-install" ]; then
+  jni_cmake_args="${jni_cmake_args} -Dre2_ROOT=${build_dir}/cpp/re2_ep-install"
+fi
+
+export JAVA_JNI_CMAKE_ARGS="${jni_cmake_args}"
 "${source_dir}/ci/scripts/jni_build.sh" \
   "${source_dir}" \
   "${install_dir}" \