Skip to content

Commit 64cfa7c

Browse files
raulcdpitrou
andauthored
GH-47367: [Packaging][Python] Patch vcpkg to show logs and install newer Windows SDK for vs_buildtools (#47484)
### Rationale for this change Our Windows wheels have been failing for a long time with some errors finding header files: ``` Python313\Lib\subprocess.py:419: CalledProcessError ---------------------------- Captured stdout call ----------------------------- Compiling extensions.pyx because it changed. [1/1] Cythonizing extensions.pyx Extension module: <setuptools.extension.Extension('extensions') at 0x24849772490> ['C:\\Python313\\Lib\\site-packages\\numpy\\_core\\include', 'C:\\Python313\\Lib\\site-packages\\pyarrow\\include'] ['arrow_python', 'arrow'] ['C:\\Python313\\Lib\\site-packages\\pyarrow', 'C:\\Python313\\Lib\\site-packages\\pyarrow.libs'] running build_ext building 'extensions' extension creating build\temp.win-amd64-cpython-313\Release "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.44.35207\bin\HostX86\x64\cl.exe" /c /nologo /O2 /W3 /GL /DNDEBUG /MD -IC:\Python313\Lib\site-packages\numpy\_core\include -IC:\Python313\Lib\site-packages\pyarrow\include -IC:\Python313\include -IC:\Python313\Include "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.44.35207\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\VS\include" /EHsc /Tpextensions.cpp /Fobuild\temp.win-amd64-cpython-313\Release\extensions.obj -D_ENABLE_EXTENDED_ALIGNED_STORAGE /std:c++17 extensions.cpp C:\Python313\include\pyconfig.h(59): fatal error C1083: Cannot open include file: 'io.h': No such file or directory ``` We also couldn't re-build our images because vcpkg was failing to build dependencies. ### What changes are included in this PR? When installing through vs_buildtools the different components we were using a deprecated Windows SDK. Installing VCTools with all recommended packages fixes both the vcpkg build and the test failures on the wheel but due to image size we have decided to only upgrade the Windows SDK at the moment. Link to MSVC VCTools package: https://learn.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2022#desktop-development-with-c In order to understand the vcpkg failure I patched vcpkg so it prints the build log error. Patching vcpkg is also part of the PR so it can be used in the future. ### Are these changes tested? Yes ### Are there any user-facing changes? No * GitHub Issue: #47367 Lead-authored-by: Raúl Cumplido <[email protected]> Co-authored-by: Antoine Pitrou <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
1 parent c558163 commit 64cfa7c

File tree

5 files changed

+70
-4
lines changed

5 files changed

+70
-4
lines changed

.env

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,8 @@ VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625" # 2024.06.15 Release
9898
# ci/docker/python-*-windows-*.dockerfile or the vcpkg config.
9999
# This is a workaround for our CI problem that "archery docker build" doesn't
100100
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
101-
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-02-25
102-
PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-02-25
101+
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-09-04
102+
PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-09-04
103103

104104
# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan".
105105
# See https://github.com/conan-io/conan-docker-tools#readme and

ci/docker/python-wheel-windows-test-vs2022-base.dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ RUN `
3535
--installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools" `
3636
--add Microsoft.VisualStudio.Component.VC.CoreBuildTools `
3737
--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
38-
--add Microsoft.VisualStudio.Component.Windows10SDK.20348 `
38+
--add Microsoft.VisualStudio.Component.Windows11SDK.26100 `
3939
--add Microsoft.VisualStudio.Component.VC.CMake.Project `
4040
|| IF "%ERRORLEVEL%"=="3010" EXIT 0) `
4141
&& del /q vs_buildtools.exe

ci/docker/python-wheel-windows-vs2022-base.dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ RUN `
6767
--installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools" `
6868
--add Microsoft.VisualStudio.Component.VC.CoreBuildTools `
6969
--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
70-
--add Microsoft.VisualStudio.Component.Windows10SDK.20348 `
70+
--add Microsoft.VisualStudio.Component.Windows11SDK.26100 `
7171
--add Microsoft.VisualStudio.Component.VC.CMake.Project `
7272
|| IF "%ERRORLEVEL%"=="3010" EXIT 0) `
7373
&& del /q vs_buildtools.exe

ci/scripts/install_vcpkg.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ fi
2626

2727
arrow_dir=$(cd -- "$(dirname -- "$0")/../.." && pwd -P)
2828
default_vcpkg_ports_patch="${arrow_dir}/ci/vcpkg/ports.patch"
29+
vcpkg_patch="${arrow_dir}/ci/vcpkg/vcpkg.patch"
2930

3031
vcpkg_destination=$1
3132
vcpkg_version=${2:-}
@@ -42,6 +43,9 @@ pushd "${vcpkg_destination}"
4243

4344
git checkout "${vcpkg_version}"
4445

46+
git apply --verbose --ignore-whitespace "${vcpkg_patch}"
47+
echo "Patch successfully applied to the VCPKG files!"
48+
4549
if [[ "${OSTYPE:-}" == "msys" ]]; then
4650
./bootstrap-vcpkg.bat -disableMetrics
4751
else

ci/vcpkg/vcpkg.patch

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
diff --git a/scripts/cmake/vcpkg_execute_build_process.cmake b/scripts/cmake/vcpkg_execute_build_process.cmake
2+
index 60fd5b587a..c8dc021af8 100644
3+
--- a/scripts/cmake/vcpkg_execute_build_process.cmake
4+
+++ b/scripts/cmake/vcpkg_execute_build_process.cmake
5+
@@ -131,6 +131,26 @@ function(vcpkg_execute_build_process)
6+
endif()
7+
endforeach()
8+
z_vcpkg_prettify_command_line(pretty_command ${arg_COMMAND})
9+
+ # --- Try to print error logs
10+
+ # Split the string by newline characters
11+
+ string(REGEX MATCHALL "[^\n]+" file_list ${stringified_logs})
12+
+
13+
+ # Iterate over the list and print content of each file
14+
+ foreach(file IN LISTS file_list)
15+
+ string(STRIP "${file}" file_stripped)
16+
+
17+
+ # Print filename
18+
+ message(STATUS "===")
19+
+ message(STATUS "=== Build Failed. Content of ${file_stripped}:")
20+
+
21+
+ # Read the content of the file
22+
+ file(READ ${file_stripped} file_content)
23+
+
24+
+ # Print the content
25+
+ message(STATUS "${file_content}")
26+
+ message(STATUS "=== End of content of ${file_stripped}")
27+
+ endforeach()
28+
+ # ---
29+
message(FATAL_ERROR
30+
" Command failed: ${pretty_command}\n"
31+
" Working Directory: ${arg_WORKING_DIRECTORY}\n"
32+
diff --git a/scripts/cmake/vcpkg_execute_required_process.cmake b/scripts/cmake/vcpkg_execute_required_process.cmake
33+
index 830aa409fd..90452d857b 100644
34+
--- a/scripts/cmake/vcpkg_execute_required_process.cmake
35+
+++ b/scripts/cmake/vcpkg_execute_required_process.cmake
36+
@@ -109,6 +109,26 @@ Halting portfile execution.
37+
endforeach()
38+
39+
z_vcpkg_prettify_command_line(pretty_command ${arg_COMMAND})
40+
+ # --- Try to print error logs
41+
+ # Split the string by newline characters
42+
+ string(REGEX MATCHALL "[^\n]+" file_list ${stringified_logs})
43+
+
44+
+ # Iterate over the list and print content of each file
45+
+ foreach(file IN LISTS file_list)
46+
+ string(STRIP "${file}" file_stripped)
47+
+
48+
+ # Print filename
49+
+ message(STATUS "===")
50+
+ message(STATUS "=== Build Failed. Content of ${file_stripped}:")
51+
+
52+
+ # Read the content of the file
53+
+ file(READ ${file_stripped} file_content)
54+
+
55+
+ # Print the content
56+
+ message(STATUS "${file_content}")
57+
+ message(STATUS "=== End of content of ${file_stripped}")
58+
+ endforeach()
59+
+ # ---
60+
message(FATAL_ERROR
61+
" Command failed: ${pretty_command}\n"
62+
" Working Directory: ${arg_WORKING_DIRECTORY}\n"

0 commit comments

Comments
 (0)