Skip to content

Commit b22b792

Browse files
authored
Merge branch 'vcmi:main' into main
2 parents 6878bf2 + 1857724 commit b22b792

File tree

10 files changed

+287
-4
lines changed

10 files changed

+287
-4
lines changed

.github/workflows/rebuildDependencies.yml

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,18 @@ jobs:
139139
git sparse-checkout set \
140140
recipes/minizip \
141141
recipes/flac \
142+
recipes/onnx \
143+
recipes/onnxruntime \
142144
$recipePathQt \
143145
144146
git checkout
145147
148+
# Patch conan recipe to support msvc version 192
149+
git apply --ignore-whitespace ../conan_patches/onnxruntime/recipe.diff
150+
146151
# versions must be synced with: conan_patches/<package>/conandata.yml
147152
# if no custom patches are required for a package, it should be removed from here
148-
for p in minizip/1.3.1 flac/1.4.2 ; do
153+
for p in minizip/1.3.1 flac/1.4.2 onnx/1.16.2 onnxruntime/1.18.1 ; do
149154
IFS_OLD="$IFS"
150155
IFS=/
151156
read package version <<<"$p"
@@ -225,7 +230,9 @@ jobs:
225230
- name: Remove build requirements' binaries
226231
run: |
227232
graphFile='graph.json'
228-
packageListFile='pkglist.json'
233+
hostPackagesFile='packages-host.json'
234+
buildPackagesFile='packages-build.json'
235+
buildFilteredPackagesFile='packages-build-filtered.txt'
229236
230237
conan graph info . \
231238
$CONAN_PROFILES \
@@ -234,12 +241,30 @@ jobs:
234241
--build=never \
235242
--no-remote \
236243
> "$graphFile"
244+
237245
conan list \
238246
--graph "$graphFile" \
239247
--graph-context=build-only \
240248
--format=json \
241-
> "$packageListFile"
242-
conan remove --list "$packageListFile" --confirm
249+
> $buildPackagesFile
250+
251+
conan list \
252+
--graph "$graphFile" \
253+
--graph-context=host \
254+
--format=json \
255+
> $hostPackagesFile
256+
257+
# Some are both build and host requirements and must not be removed
258+
# => exclude them from the list
259+
python3 -c '
260+
import json, sys
261+
build_dict = json.load(open(sys.argv[1]))["Local Cache"] # => {"pkg1": {...}, "pkg2": {...}, ...}
262+
host_list = json.load(open(sys.argv[2]))["Local Cache"] # => ["pkg1", "pkg2", ...]
263+
filtered_dict = {k: v for k, v in build_dict.items() if k not in host_list}
264+
json.dump({"Local Cache": filtered_dict}, open(sys.argv[3], "w"))
265+
' "$buildPackagesFile" "$hostPackagesFile" "$buildFilteredPackagesFile"
266+
267+
conan remove --list "$buildFilteredPackagesFile" --confirm
243268
244269
- name: Create list of built packages
245270
run: |

conan_patches/onnx/conandata.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
patches:
2+
"1.16.2":
3+
- patch_file: "patches/apple-pre-2019.diff"
4+
patch_description: |
5+
Fixes errors for older apple targets:
6+
error: 'basic_fstream' is unavailable: introduced in macOS 10.15
7+
error: 'basic_fstream' is unavailable: introduced in iOS 13.0
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
--- a/onnx/common/file_utils.h
2+
+++ b/onnx/common/file_utils.h
3+
@@ -17,8 +17,7 @@ namespace ONNX_NAMESPACE {
4+
5+
template <typename T>
6+
void LoadProtoFromPath(const std::string proto_path, T& proto) {
7+
- std::filesystem::path proto_u8_path = std::filesystem::u8path(proto_path);
8+
- std::fstream proto_stream(proto_u8_path, std::ios::in | std::ios::binary);
9+
+ std::fstream proto_stream(proto_path, std::ios::in | std::ios::binary);
10+
if (!proto_stream.good()) {
11+
fail_check("Unable to open proto file: ", proto_path, ". Please check if it is a valid proto. ");
12+
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
patches:
2+
"1.18.1":
3+
- patch_file: "patches/android-64.diff"
4+
patch_description: |
5+
Fixes error for arm64-v8a targets:
6+
onnxruntime/core/mlas/lib/mlasi.h:366:11: error: unknown type name 'bfloat16_t'
7+
- patch_file: "patches/msvc-x86.diff"
8+
patch_description: |
9+
Fixes detection of x86 windows targets during cross-compilation.
10+
- patch_file: "patches/win7.diff"
11+
patch_description: |
12+
Fixes compile errors on for windows 7 targets:
13+
error C3861: 'CreateFile2': identifier not found
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
--- a/cmake/onnxruntime_mlas.cmake
2+
+++ b/cmake/onnxruntime_mlas.cmake
3+
@@ -352,7 +362,7 @@ else()
4+
${MLAS_SRC_DIR}/sqnbitgemm_kernel_neon.cpp
5+
)
6+
set_source_files_properties(${MLAS_SRC_DIR}/sqnbitgemm_kernel_neon.cpp
7+
- PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+dotprod")
8+
+ PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+dotprod+bf16")
9+
if (NOT APPLE)
10+
set(mlas_platform_srcs
11+
${mlas_platform_srcs}
12+
@@ -368,13 +378,13 @@ else()
13+
${MLAS_SRC_DIR}/qgemm_kernel_ummla.cpp
14+
${MLAS_SRC_DIR}/sbgemm_kernel_neon.cpp
15+
)
16+
- set_source_files_properties(${MLAS_SRC_DIR}/aarch64/HalfGemmKernelNeon.S PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+fp16 ")
17+
- set_source_files_properties(${MLAS_SRC_DIR}/aarch64/QgemmS8S8KernelSmmla.S PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+i8mm ")
18+
- set_source_files_properties(${MLAS_SRC_DIR}/aarch64/QgemmU8X8KernelUmmla.S PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+i8mm ")
19+
+ set_source_files_properties(${MLAS_SRC_DIR}/aarch64/HalfGemmKernelNeon.S PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+fp16+bf16 ")
20+
+ set_source_files_properties(${MLAS_SRC_DIR}/aarch64/QgemmS8S8KernelSmmla.S PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+i8mm+bf16 ")
21+
+ set_source_files_properties(${MLAS_SRC_DIR}/aarch64/QgemmU8X8KernelUmmla.S PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+i8mm+bf16 ")
22+
set_source_files_properties(${MLAS_SRC_DIR}/aarch64/SbgemmKernelNeon.S PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+bf16 ")
23+
- set_source_files_properties(${MLAS_SRC_DIR}/activate_fp16.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+fp16 ")
24+
- set_source_files_properties(${MLAS_SRC_DIR}/dwconv.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+fp16 ")
25+
- set_source_files_properties(${MLAS_SRC_DIR}/pooling_fp16.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+fp16 ")
26+
+ set_source_files_properties(${MLAS_SRC_DIR}/activate_fp16.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+fp16+bf16 ")
27+
+ set_source_files_properties(${MLAS_SRC_DIR}/dwconv.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+fp16+bf16 ")
28+
+ set_source_files_properties(${MLAS_SRC_DIR}/pooling_fp16.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+fp16+bf16 ")
29+
set_source_files_properties(${MLAS_SRC_DIR}/sbgemm_kernel_neon.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+bf16 ")
30+
endif()
31+
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
--- a/cmake/adjust_global_compile_flags.cmake
2+
+++ b/cmake/adjust_global_compile_flags.cmake
3+
@@ -226,13 +226,17 @@ endmacro()
4+
5+
#Set global compile flags for all the source code(including third_party code like protobuf)
6+
#This section must be before any add_subdirectory, otherwise build may fail because /MD,/MT mismatch
7+
+message(STATUS "MSVC=${MSVC} | CMAKE_VS_PLATFORM_NAME=${CMAKE_VS_PLATFORM_NAME} | CMAKE_C_COMPILER_ARCHITECTURE_ID=${CMAKE_C_COMPILER_ARCHITECTURE_ID} | CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
8+
if (MSVC)
9+
if (CMAKE_VS_PLATFORM_NAME)
10+
# Multi-platform generator
11+
set(onnxruntime_target_platform ${CMAKE_VS_PLATFORM_NAME})
12+
+ elseif (CMAKE_C_COMPILER_ARCHITECTURE_ID)
13+
+ set(onnxruntime_target_platform ${CMAKE_C_COMPILER_ARCHITECTURE_ID})
14+
else()
15+
set(onnxruntime_target_platform ${CMAKE_SYSTEM_PROCESSOR})
16+
endif()
17+
+ message(STATUS "Temporary onnxruntime_target_platform=${onnxruntime_target_platform}")
18+
if (onnxruntime_target_platform STREQUAL "ARM64")
19+
set(onnxruntime_target_platform "ARM64")
20+
enable_language(ASM_MARMASM)
21+
@@ -241,18 +245,19 @@ if (MSVC)
22+
elseif (onnxruntime_target_platform STREQUAL "ARM" OR CMAKE_GENERATOR MATCHES "ARM")
23+
set(onnxruntime_target_platform "ARM")
24+
enable_language(ASM_MARMASM)
25+
- elseif (onnxruntime_target_platform STREQUAL "x64" OR onnxruntime_target_platform STREQUAL "x86_64" OR onnxruntime_target_platform STREQUAL "AMD64" OR CMAKE_GENERATOR MATCHES "Win64")
26+
+ elseif (onnxruntime_target_platform MATCHES "[Xx]64" OR onnxruntime_target_platform MATCHES "[Xx]86_64" OR onnxruntime_target_platform STREQUAL "AMD64" OR CMAKE_GENERATOR MATCHES "Win64")
27+
set(onnxruntime_target_platform "x64")
28+
enable_language(ASM_MASM)
29+
- elseif (onnxruntime_target_platform STREQUAL "Win32" OR onnxruntime_target_platform STREQUAL "x86" OR onnxruntime_target_platform STREQUAL "i386" OR onnxruntime_target_platform STREQUAL "i686")
30+
+ elseif (onnxruntime_target_platform STREQUAL "Win32" OR onnxruntime_target_platform MATCHES "[Xx]86" OR onnxruntime_target_platform STREQUAL "i386" OR onnxruntime_target_platform STREQUAL "i686")
31+
set(onnxruntime_target_platform "x86")
32+
enable_language(ASM_MASM)
33+
message("Enabling SAFESEH for x86 build")
34+
set(CMAKE_ASM_MASM_FLAGS "${CMAKE_ASM_MASM_FLAGS} /safeseh")
35+
else()
36+
- message(FATAL_ERROR "Unknown CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
37+
+ message(FATAL_ERROR "Unknown onnxruntime_target_platform: ${onnxruntime_target_platform}")
38+
endif()
39+
40+
+ message(STATUS "Final onnxruntime_target_platform=${onnxruntime_target_platform}")
41+
42+
#Always enable exception handling, even for Windows ARM
43+
if (NOT onnxruntime_DISABLE_EXCEPTIONS)
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
--- a/onnxruntime/test/shared_lib/test_model_loading.cc
2+
+++ b/onnxruntime/test/shared_lib/test_model_loading.cc
3+
@@ -227,7 +227,7 @@ using ScopedFileDescriptor = ScopedResource<FileDescriptorTraits>;
4+
5+
void FileMmap(const ORTCHAR_T* file_path, void*& mapped_base) {
6+
#ifdef _WIN32
7+
- wil::unique_hfile file_handle{CreateFile2(file_path, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, NULL)};
8+
+ wil::unique_hfile file_handle{CreateFileW(file_path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr)};
9+
ASSERT_TRUE(file_handle.get() != INVALID_HANDLE_VALUE);
10+
11+
wil::unique_hfile file_mapping_handle{
12+
--- a/onnxruntime/core/platform/windows/env.cc
13+
+++ b/onnxruntime/core/platform/windows/env.cc
14+
@@ -314,7 +314,7 @@ PIDType WindowsEnv::GetSelfPid() const {
15+
16+
Status WindowsEnv::GetFileLength(_In_z_ const ORTCHAR_T* file_path, size_t& length) const {
17+
wil::unique_hfile file_handle{
18+
- CreateFile2(file_path, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, OPEN_EXISTING, NULL)};
19+
+ CreateFileW(file_path, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr)};
20+
if (file_handle.get() == INVALID_HANDLE_VALUE) {
21+
const auto error_code = GetLastError();
22+
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "open file ", ToUTF8String(Basename(file_path)), " fail, errcode = ", error_code, " - ", std::system_category().message(error_code));
23+
@@ -361,7 +361,7 @@ Status WindowsEnv::ReadFileIntoBuffer(_In_z_ const ORTCHAR_T* const file_path, c
24+
ORT_RETURN_IF_NOT(offset >= 0, "offset < 0");
25+
ORT_RETURN_IF_NOT(length <= buffer.size(), "length > buffer.size()");
26+
wil::unique_hfile file_handle{
27+
- CreateFile2(file_path, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, NULL)};
28+
+ CreateFileW(file_path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr)};
29+
if (file_handle.get() == INVALID_HANDLE_VALUE) {
30+
const auto error_code = GetLastError();
31+
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "open file ", ToUTF8String(Basename(file_path)), " fail, errcode = ", error_code, " - ", std::system_category().message(error_code));
32+
@@ -414,7 +414,7 @@ Status WindowsEnv::MapFileIntoMemory(_In_z_ const ORTCHAR_T* file_path,
33+
}
34+
35+
wil::unique_hfile file_handle{
36+
- CreateFile2(file_path, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, NULL)};
37+
+ CreateFileW(file_path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr)};
38+
if (file_handle.get() == INVALID_HANDLE_VALUE) {
39+
const auto error_code = GetLastError();
40+
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
41+
@@ -600,16 +600,14 @@ common::Status WindowsEnv::GetCanonicalPath(
42+
PathString& canonical_path) const {
43+
// adapted from MSVC STL std::filesystem::canonical() implementation
44+
// https://github.com/microsoft/STL/blob/ed3cbf36416a385828e7a5987ca52cb42882d84b/stl/inc/filesystem#L2986
45+
- CREATEFILE2_EXTENDED_PARAMETERS param;
46+
- memset(&param, 0, sizeof(param));
47+
- param.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS);
48+
- param.dwFileFlags = FILE_FLAG_BACKUP_SEMANTICS;
49+
- wil::unique_hfile file_handle{CreateFile2(
50+
- path.c_str(),
51+
- FILE_READ_ATTRIBUTES,
52+
- FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
53+
- OPEN_EXISTING,
54+
- &param)};
55+
+ wil::unique_hfile file_handle{CreateFileW(
56+
+ path.c_str(), // LPCWSTR lpFileName
57+
+ FILE_READ_ATTRIBUTES, // DWORD dwDesiredAccess
58+
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, // DWORD dwShareMode
59+
+ nullptr, // LPSECURITY_ATTRIBUTES lpSecurityAttributes
60+
+ OPEN_EXISTING, // DWORD dwCreationDisposition
61+
+ FILE_FLAG_BACKUP_SEMANTICS, // DWORD dwFlagsAndAttributes
62+
+ nullptr)}; // HANDLE hTemplateFile
63+
64+
if (file_handle.get() == INVALID_HANDLE_VALUE) {
65+
const auto error_code = GetLastError();
66+
--- a/onnxruntime/core/platform/windows/env_time.cc
67+
+++ b/onnxruntime/core/platform/windows/env_time.cc
68+
@@ -18,6 +18,7 @@ limitations under the License.
69+
70+
#include <time.h>
71+
#include <windows.h>
72+
+#include <sysinfoapi.h>
73+
#include <chrono>
74+
#include <numeric>
75+
#include <algorithm>
76+
@@ -28,14 +29,10 @@ namespace {
77+
78+
class WindowsEnvTime : public EnvTime {
79+
public:
80+
- WindowsEnvTime() : GetSystemTimePreciseAsFileTime_(GetSystemTimePreciseAsFileTime) {}
81+
+ WindowsEnvTime() = default;
82+
83+
uint64_t NowMicros() override {
84+
- if (GetSystemTimePreciseAsFileTime_ != NULL) {
85+
- // GetSystemTimePreciseAsFileTime function is only available in latest
86+
- // versions of Windows, so we need to check for its existence here.
87+
- // All std::chrono clocks on Windows proved to return
88+
- // values that may repeat, which is not good enough for some uses.
89+
+ // System time in 100-ns ticks since 1601-01-01
90+
constexpr int64_t kUnixEpochStartTicks = 116444736000000000i64;
91+
constexpr int64_t kFtToMicroSec = 10;
92+
93+
@@ -43,7 +40,8 @@ class WindowsEnvTime : public EnvTime {
94+
// just any microseconds because it is often used as an argument
95+
// to TimedWait() on condition variable
96+
FILETIME system_time;
97+
- GetSystemTimePreciseAsFileTime_(&system_time);
98+
+ // Get current system time as FILETIME (available on all supported Windows versions)
99+
+ GetSystemTimeAsFileTime(&system_time);
100+
101+
LARGE_INTEGER li;
102+
li.LowPart = system_time.dwLowDateTime;
103+
@@ -54,16 +52,11 @@ class WindowsEnvTime : public EnvTime {
104+
li.QuadPart /= kFtToMicroSec;
105+
return li.QuadPart;
106+
}
107+
- using namespace std::chrono;
108+
- return duration_cast<microseconds>(system_clock::now().time_since_epoch())
109+
- .count();
110+
- }
111+
-
112+
- void SleepForMicroseconds(int64_t micros) { Sleep(static_cast<DWORD>(micros) / 1000); }
113+
114+
- private:
115+
- typedef VOID(WINAPI* FnGetSystemTimePreciseAsFileTime)(LPFILETIME);
116+
- FnGetSystemTimePreciseAsFileTime GetSystemTimePreciseAsFileTime_;
117+
+ void SleepForMicroseconds(int64_t micros) {
118+
+ DWORD ms = static_cast<DWORD>((micros + 999) / 1000);
119+
+ Sleep(ms);
120+
+ }
121+
};
122+
123+
} // namespace
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
diff --git a/recipes/onnxruntime/all/conanfile.py b/recipes/onnxruntime/all/conanfile.py
2+
index 0757ecdc0..bb8cca8bf 100644
3+
--- a/recipes/onnxruntime/all/conanfile.py
4+
+++ b/recipes/onnxruntime/all/conanfile.py
5+
@@ -47,7 +52,7 @@ class OnnxRuntimeConan(ConanFile):
6+
def _compilers_minimum_version(self):
7+
return {
8+
"Visual Studio": "17",
9+
- "msvc": "193",
10+
+ "msvc": "192",
11+
"gcc": "9",
12+
"clang": "5",
13+
"apple-clang": "10",

conan_profiles/base/common

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,19 @@ sdl_mixer/*:modplug=False
219219
sdl_mixer/*:nativemidi=False
220220
sdl_mixer/*:tinymidi=False
221221

222+
# libsquish
223+
# accelerate as much as possible
224+
libsquish/*:openmp=True
225+
libsquish/*:sse2_intrinsics=True
226+
222227
# transitive deps, stuff that's not needed
223228
# doesn't link to bzip2 & zlib from "system" recipes
224229
pcre2/*:build_pcre2grep=False
225230
sqlite3/*:build_executable=False
226231
opusfile/*:http=False
227232
zstd/*:build_programs=False
233+
234+
# Prevents onnxruntime->date->libcurl dependency
235+
# NOTE: use_system_tz_db is specific to date/3.0.1
236+
# Newer date recipes would need the tz_db=system option instead.
237+
date/*:use_system_tz_db=True

conanfile.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ class VCMI(ConanFile):
1919
"sdl_image/[^2.8.2]",
2020
"sdl_mixer/[^2.8.0]",
2121
"sdl_ttf/[^2.0.18]",
22+
"libsquish/[^1.15]",
2223
]
2324
_launcherRequires = [
2425
"xz_utils/[^5.2.5]", # innoextract
@@ -28,11 +29,13 @@ class VCMI(ConanFile):
2829
options = {
2930
"target_pre_windows10": [True, False],
3031
"with_ffmpeg": [True, False],
32+
"with_onnxruntime": [True, False],
3133
"lua_lib": [None, "luajit", "lua"]
3234
}
3335
default_options = {
3436
"target_pre_windows10": False,
3537
"with_ffmpeg": True,
38+
"with_onnxruntime": True,
3639
"lua_lib": "luajit",
3740
}
3841

@@ -87,6 +90,9 @@ def requirements(self):
8790
else:
8891
self.requires("qt/[~5.15.2]")
8992

93+
if self.options.with_onnxruntime:
94+
self.requires("onnxruntime/1.18.1")
95+
9096
def validate(self):
9197
# FFmpeg
9298
if is_msvc(self) and self.options.with_ffmpeg and self.dependencies["ffmpeg"].options.shared != True:

0 commit comments

Comments
 (0)