Skip to content

Commit 038fa53

Browse files
chinazhangchaochinazhangchao
committed
fix windows build issue
1 parent 63238ab commit 038fa53

File tree

8 files changed

+175
-50
lines changed

8 files changed

+175
-50
lines changed

‎backends/xnnpack/CMakeLists.txt‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -73,17 +73,33 @@ foreach(fbs_file ${_xnnpack_schema__srcs})
7373
endforeach()
7474

7575
# Generate the headers from the .fbs files.
76-
add_custom_command(
77-
OUTPUT ${_xnnpack_schema__outputs}
78-
COMMAND
79-
${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o
80-
"${_xnnpack_schema__include_dir}/executorch/backends/xnnpack/serialization"
81-
${_xnnpack_schema__srcs}
82-
COMMAND mv ${_xnnpack_flatbuffer__outputs} ${_xnnpack_schema__outputs}
83-
WORKING_DIRECTORY ${EXECUTORCH_ROOT}
84-
COMMENT "Generating xnnpack_schema headers"
85-
VERBATIM
86-
)
76+
if(WIN32)
77+
add_custom_command(
78+
OUTPUT ${_xnnpack_schema__outputs}
79+
COMMAND
80+
${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o
81+
"${_xnnpack_schema__include_dir}/executorch/backends/xnnpack/serialization"
82+
${_xnnpack_schema__srcs}
83+
COMMAND
84+
powershell -Command
85+
"Move-Item -Path ${_xnnpack_flatbuffer__outputs} -Destination ${_xnnpack_schema__outputs}"
86+
WORKING_DIRECTORY ${EXECUTORCH_ROOT}
87+
COMMENT "Generating xnnpack_schema headers"
88+
VERBATIM
89+
)
90+
else()
91+
add_custom_command(
92+
OUTPUT ${_xnnpack_schema__outputs}
93+
COMMAND
94+
${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o
95+
"${_xnnpack_schema__include_dir}/executorch/backends/xnnpack/serialization"
96+
${_xnnpack_schema__srcs}
97+
COMMAND mv ${_xnnpack_flatbuffer__outputs} ${_xnnpack_schema__outputs}
98+
WORKING_DIRECTORY ${EXECUTORCH_ROOT}
99+
COMMENT "Generating xnnpack_schema headers"
100+
VERBATIM
101+
)
102+
endif()
87103

88104
add_library(xnnpack_schema INTERFACE ${_xnnpack_schema__outputs})
89105
set_target_properties(xnnpack_schema PROPERTIES LINKER_LANGUAGE CXX)

‎build/resolve_buck.py‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
import platform
1212
import stat
1313
import sys
14-
import tempfile
1514
import urllib.request
1615

1716
from dataclasses import dataclass
@@ -85,6 +84,12 @@ class BuckInfo:
8584
archive_name="buck2-x86_64-apple-darwin.zst",
8685
target_versions=["3eb1ae97ea963086866b4d2d9ffa966d"],
8786
),
87+
("windows", "x86_64"): BuckInfo(
88+
archive_name="buck2-x86_64-pc-windows-msvc.exe.zst",
89+
target_versions=[
90+
"bf1685c4c4ddd9de4592b5a955cb7326fd01e6c4d5f561643422bed961a17401"
91+
],
92+
),
8893
}
8994

9095

@@ -135,6 +140,8 @@ def resolve_buck2(args: argparse.Namespace) -> Union[str, int]:
135140
os_family = "linux"
136141
elif sys.platform.startswith("darwin"):
137142
os_family = "darwin"
143+
elif sys.platform.startswith("win"):
144+
os_family = "windows"
138145

139146
platform_key = (os_family, arch)
140147
if platform_key not in BUCK_PLATFORM_MAP:
@@ -193,12 +200,14 @@ def resolve_buck2(args: argparse.Namespace) -> Union[str, int]:
193200

194201
buck2_archive_url = f"https://github.com/facebook/buck2/releases/download/{target_buck_version}/{buck_info.archive_name}"
195202

196-
with tempfile.NamedTemporaryFile() as archive_file:
203+
try:
197204
print(f"Downloading buck2 from {buck2_archive_url}...", file=sys.stderr)
198-
urllib.request.urlretrieve(buck2_archive_url, archive_file.name)
205+
# This change is based on https://github.com/posit-dev/py-shiny/issues/287
206+
# It works around PermissionError on Windows. This happens because urlretrieve is trying to open an already-open temporary file by name, which isn't permitted on Windows.
207+
archive_file, _ = urllib.request.urlretrieve(buck2_archive_url)
199208

200209
# Extract and chmod.
201-
with open(archive_file.name, "rb") as f:
210+
with open(archive_file, "rb") as f:
202211
data = f.read()
203212
decompressed_bytes = zstd.decompress(data)
204213

@@ -207,6 +216,8 @@ def resolve_buck2(args: argparse.Namespace) -> Union[str, int]:
207216

208217
file_stat = os.stat(buck2_local_path)
209218
os.chmod(buck2_local_path, file_stat.st_mode | stat.S_IEXEC)
219+
finally:
220+
os.remove(archive_file)
210221

211222
return buck2_local_path
212223

‎devtools/CMakeLists.txt‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -157,19 +157,46 @@ file(MAKE_DIRECTORY
157157
${_program_schema__include_dir}/executorch/devtools/bundled_program
158158
)
159159

160-
add_custom_command(
161-
OUTPUT ${_etdump_schema__outputs}
162-
COMMAND
163-
# Note that the flatcc project actually writes its outputs into the source
164-
# tree instead of under the binary directory, and there's no way to change
165-
# that behavior.
166-
${_flatcc_source_dir}/bin/flatcc -cwr -o
167-
${_program_schema__include_dir}/executorch/devtools/etdump
168-
${_etdump_schema__srcs}
169-
COMMAND rm -rf ${_etdump_schema_cleanup_paths}
170-
DEPENDS ${_etdump_schema_gen_dep}
171-
COMMENT "Generating etdump headers"
172-
)
160+
# Note that the flatcc project actually writes its outputs into the source
161+
# tree instead of under the binary directory, and there's no way to change
162+
# that behavior.
163+
if(WIN32)
164+
# The binary directory for Windows will have build type like Debug/Release at the end.
165+
set(_flatcc_bin_path ${_flatcc_source_dir}/bin/${CMAKE_BUILD_TYPE}/flatcc)
166+
else()
167+
set(_flatcc_bin_path ${_flatcc_source_dir}/bin/flatcc)
168+
endif()
169+
170+
if(WIN32)
171+
add_custom_command(
172+
OUTPUT ${_etdump_schema__outputs}
173+
COMMAND
174+
# Note that the flatcc project actually writes its outputs into the source
175+
# tree instead of under the binary directory, and there's no way to change
176+
# that behavior.
177+
${_flatcc_bin_path} -cwr -o
178+
${_program_schema__include_dir}/executorch/devtools/etdump
179+
${_etdump_schema__srcs}
180+
# COMMAND powershell -Command "Remove-Item -Path " ${_etdump_schema_cleanup_paths} " -Force -ErrorAction SilentlyContinue"
181+
COMMAND powershell -Command "if (" "'${_etdump_schema_cleanup_paths}'" "-ne '') { Remove-Item -Path " "'${_etdump_schema_cleanup_paths}'" " -Force -ErrorAction SilentlyContinue }"
182+
DEPENDS ${_etdump_schema_gen_dep}
183+
COMMENT "Generating etdump headers"
184+
)
185+
else()
186+
add_custom_command(
187+
OUTPUT ${_etdump_schema__outputs}
188+
COMMAND
189+
# Note that the flatcc project actually writes its outputs into the source
190+
# tree instead of under the binary directory, and there's no way to change
191+
# that behavior.
192+
${_flatcc_bin_path} -cwr -o
193+
${_program_schema__include_dir}/executorch/devtools/etdump
194+
${_etdump_schema__srcs}
195+
COMMAND rm -rf ${_etdump_schema_cleanup_paths}
196+
DEPENDS ${_etdump_schema_gen_dep}
197+
COMMENT "Generating etdump headers"
198+
)
199+
endif()
173200

174201
add_library(
175202
etdump ${CMAKE_CURRENT_SOURCE_DIR}/etdump/etdump_flatcc.cpp

‎install_requirements.bat‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,8 @@ rem This batch file provides a basic functionality similar to the bash script.
77

88
cd /d "%~dp0"
99

10-
rem Find the names of the python tools to use (replace with your actual python installation)
11-
if "%PYTHON_EXECUTABLE%"=="" (
12-
if "%CONDA_DEFAULT_ENV%"=="" OR "%CONDA_DEFAULT_ENV%"=="base" OR NOT EXIST "python" (
13-
set PYTHON_EXECUTABLE=python3
14-
) else (
15-
set PYTHON_EXECUTABLE=python
16-
)
17-
)
10+
rem Under windows it's always python
11+
set PYTHON_EXECUTABLE=python
1812

1913
"%PYTHON_EXECUTABLE%" install_requirements.py %*
2014

‎runtime/platform/compat_unistd.h‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
/**
10+
* @file
11+
* unistd.h related macros for POSIX/Windows compatibility.
12+
*/
13+
#pragma once
14+
15+
#if defined(_WIN32) && !defined(_WIN64)
16+
#error \
17+
"You're trying to build ExecuTorch with a too old version of Windows. We need Windows 64-bit."
18+
#endif
19+
20+
#if !defined(_WIN64)
21+
#include <unistd.h>
22+
#else
23+
#include <io.h>
24+
#define O_RDONLY _O_RDONLY
25+
#define open _open
26+
#define close _close
27+
#define read _read
28+
#define write _write
29+
#define stat _stat64
30+
#define fstat _fstat64
31+
#define off_t _off_t
32+
#define lseek _lseeki64
33+
34+
#include <executorch/runtime/platform/compiler.h> // For ssize_t.
35+
#include <windows.h>
36+
// To avoid conflicts with std::numeric_limits<int32_t>::max() in
37+
// file_data_loader.cpp.
38+
#undef max
39+
40+
inline ssize_t pread(int fd, void* buf, size_t nbytes, size_t offset) {
41+
OVERLAPPED overlapped; /* The offset for ReadFile. */
42+
memset(&overlapped, 0, sizeof(overlapped));
43+
overlapped.Offset = offset;
44+
overlapped.OffsetHigh = offset >> 32;
45+
46+
BOOL result; /* The result of ReadFile. */
47+
DWORD bytes_read; /* The number of bytes read. */
48+
HANDLE file = (HANDLE)_get_osfhandle(fd);
49+
50+
result = ReadFile(file, buf, nbytes, &bytes_read, &overlapped);
51+
DWORD error = GetLastError();
52+
if (!result) {
53+
if (error == ERROR_IO_PENDING) {
54+
result = GetOverlappedResult(file, &overlapped, &bytes_read, TRUE);
55+
if (!result) {
56+
error = GetLastError();
57+
}
58+
}
59+
}
60+
if (!result) {
61+
// Translate error into errno.
62+
switch (error) {
63+
case ERROR_HANDLE_EOF:
64+
errno = 0;
65+
break;
66+
default:
67+
errno = EIO;
68+
break;
69+
}
70+
return -1;
71+
}
72+
return bytes_read;
73+
}
74+
75+
#endif // !defined(_WIN64)

‎runtime/platform/compiler.h‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -100,14 +100,8 @@
100100
#endif // (__cplusplus) >= 202002L
101101

102102
/// Define a C symbol with weak linkage.
103-
#ifdef _MSC_VER
104-
// There currently doesn't seem to be a great way to do this in Windows and
105-
// given that weak linkage is not really critical on Windows, we'll just leave
106-
// it as a stub.
107-
#define ET_WEAK
108-
#else
103+
// Building on Windows also need this. Windows build uses clang-cl compiler, which supports __attribute__((weak)).
109104
#define ET_WEAK __attribute__((weak))
110-
#endif
111105

112106
/**
113107
* Annotation marking a function as printf-like, providing compiler support

‎runtime/platform/targets.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ def define_common_targets():
6868
"log.h",
6969
"profiler.h",
7070
"runtime.h",
71+
"compat_unistd.h",
7172
],
7273
srcs = [
7374
"abort.cpp",

‎setup.py‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ def src_path(self, installer: "InstallerBuildExt") -> Path:
225225
self.src = self.src.replace("%BUILD_TYPE%", cfg)
226226
else:
227227
# Remove %BUILD_TYPE% from the path.
228-
self.src = self.src.replace("/%BUILD_TYPE%", "")
228+
self.src = self.src.replace("%BUILD_TYPE%/", "")
229229

230230
# Construct the full source path, resolving globs. If there are no glob
231231
# pattern characters, this will just ensure that the source file exists.
@@ -263,7 +263,7 @@ def __init__(
263263
output is in a subdirectory named after the build type. For single-
264264
config generators (like Makefile Generators or Ninja), this placeholder
265265
will be removed.
266-
src_name: The name of the file to install
266+
src_name: The name of the file to install.
267267
dst: The path to install to, relative to the root of the pip
268268
package. If dst ends in "/", it is treated as a directory.
269269
Otherwise it is treated as a filename.
@@ -305,20 +305,25 @@ def dst_path(self, installer: "InstallerBuildExt") -> Path:
305305
class BuiltExtension(_BaseExtension):
306306
"""An extension that installs a python extension that was built by cmake."""
307307

308-
def __init__(self, src: str, modpath: str):
308+
def __init__(self, src_dir: str, src_name: str, modpath: str):
309309
"""Initializes a BuiltExtension.
310310
311311
Args:
312-
src: The path to the file to install (typically a shared library),
313-
relative to the cmake-out directory. May be an fnmatch-style
314-
glob that matches exactly one file. If the path ends in `.so`,
312+
src_dir: The directory of the file to install, relative to the cmake-out
313+
directory. A placeholder %BUILD_TYPE% will be replaced with the build
314+
type for multi-config generators (like Visual Studio) where the build
315+
output is in a subdirectory named after the build type. For single-
316+
config generators (like Makefile Generators or Ninja), this placeholder
317+
will be removed.
318+
src_name: The name of the file to install. If the path ends in `.so`,
315319
this class will also look for similarly-named `.dylib` files.
316320
modpath: The dotted path of the python module that maps to the
317321
extension.
318322
"""
319323
assert (
320324
"/" not in modpath
321325
), f"modpath must be a dotted python module path: saw '{modpath}'"
326+
src = os.path.join(src_dir, src_name)
322327
# This is a real extension, so use the modpath as the name.
323328
super().__init__(src=src, dst=modpath, name=modpath)
324329

@@ -658,7 +663,9 @@ def get_ext_modules() -> List[Extension]:
658663
# portable kernels, and a selection of backends. This lets users
659664
# load and execute .pte files from python.
660665
BuiltExtension(
661-
"_portable_lib.*", "executorch.extension.pybindings._portable_lib"
666+
src_dir="%BUILD_TYPE%/", # Set the src directory based on build configuration for windows.
667+
src_name="_portable_lib.cp*", # Rename _portable_lib.* to _portable_lib.cp* to avoid _portable_lib.lib is selected on windows.
668+
modpath="executorch.extension.pybindings._portable_lib",
662669
)
663670
)
664671
if ShouldBuild.llama_custom_ops():

0 commit comments

Comments
 (0)