diff --git a/.ci/docker/ci_commit_pins/pytorch.txt b/.ci/docker/ci_commit_pins/pytorch.txt index bb8caf3ffeb..f1719bacd64 100644 --- a/.ci/docker/ci_commit_pins/pytorch.txt +++ b/.ci/docker/ci_commit_pins/pytorch.txt @@ -1 +1 @@ -5616fa4a68718ead203314a3467f7dd9547153ae +c620d0b5c7e8679413d620624725471223ce8359 diff --git a/.ci/docker/common/install_conda.sh b/.ci/docker/common/install_conda.sh index ac3e4fc1a0e..8c1c7da63d4 100755 --- a/.ci/docker/common/install_conda.sh +++ b/.ci/docker/common/install_conda.sh @@ -13,7 +13,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" install_miniconda() { BASE_URL="https://repo.anaconda.com/miniconda" CONDA_FILE="Miniconda3-py${PYTHON_VERSION//./}_${MINICONDA_VERSION}-Linux-x86_64.sh" - if [[ $(uname -m) == "aarch64" ]]; then + if [[ $(uname -m) == "aarch64" ]]; then CONDA_FILE="Miniconda3-py${PYTHON_VERSION//./}_${MINICONDA_VERSION}-Linux-aarch64.sh" fi @@ -71,4 +71,8 @@ fix_conda_ubuntu_libstdcxx() { install_miniconda install_python install_pip_dependencies -fix_conda_ubuntu_libstdcxx +# Hack breaks the job on aarch64 but is still necessary everywhere +# else. +if [ "$(uname -m)" != "aarch64" ]; then + fix_conda_ubuntu_libstdcxx +fi diff --git a/.ci/docker/conda-env-ci.txt b/.ci/docker/conda-env-ci.txt index 292a87ace64..995f9635a0d 100644 --- a/.ci/docker/conda-env-ci.txt +++ b/.ci/docker/conda-env-ci.txt @@ -1,4 +1,4 @@ -cmake=3.26.4 +cmake=3.31.2 ninja=1.10.2 libuv llvm-openmp diff --git a/.ci/scripts/check_c10_sync.sh b/.ci/scripts/check_c10_sync.sh index 69dfd7ea543..67bc8a3e4c0 100755 --- a/.ci/scripts/check_c10_sync.sh +++ b/.ci/scripts/check_c10_sync.sh @@ -12,3 +12,4 @@ pushd pytorch git checkout "$pytorch_pin" popd "$(dirname "${BASH_SOURCE[0]}")"/compare_dirs.sh runtime/core/portable_type/c10/c10 pytorch/c10 +"$(dirname "${BASH_SOURCE[0]}")"/compare_dirs.sh runtime/core/portable_type/c10/torch/standalone pytorch/torch/standalone diff --git a/install_requirements.py b/install_requirements.py index 66768426a99..641f40ee1b7 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -71,7 +71,7 @@ def python_is_compatible(): # # NOTE: If you're changing, make the corresponding change in .ci/docker/ci_commit_pins/pytorch.txt # by picking the hash from the same date in https://hud.pytorch.org/hud/pytorch/pytorch/nightly/ -NIGHTLY_VERSION = "dev20250601" +NIGHTLY_VERSION = "dev20250616" def install_requirements(use_pytorch_nightly): diff --git a/runtime/core/portable_type/c10/README.md b/runtime/core/portable_type/c10/README.md index 104a6717ba7..361d4b51c44 100644 --- a/runtime/core/portable_type/c10/README.md +++ b/runtime/core/portable_type/c10/README.md @@ -11,3 +11,10 @@ would cause all headers in that directory to be includeable with `#include `. In particular, that includes `runtime/core/portable_type/complex.h`, which would shadow the C99 `complex.h` standard header. + +`torch/standalone` has been added as an extra "even more bottom of +stack" directory in PyTorch, so we have to add it to our sync +here. The extra "stutter" c10 directory causing `c10/torch/standlone` +is unfortunately awkward; perhaps we can rename the top-level +directory to `pytorch_embedded_mirror` when we have extra time to work +through CI failures. diff --git a/runtime/core/portable_type/c10/c10/macros/Export.h b/runtime/core/portable_type/c10/c10/macros/Export.h index 21808de77a3..28068bd34d4 100644 --- a/runtime/core/portable_type/c10/c10/macros/Export.h +++ b/runtime/core/portable_type/c10/c10/macros/Export.h @@ -1,95 +1,11 @@ #ifndef C10_MACROS_EXPORT_H_ #define C10_MACROS_EXPORT_H_ -/* Header file to define the common scaffolding for exported symbols. - * - * Export is by itself a quite tricky situation to deal with, and if you are - * hitting this file, make sure you start with the background here: - * - Linux: https://gcc.gnu.org/wiki/Visibility - * - Windows: - * https://docs.microsoft.com/en-us/cpp/cpp/dllexport-dllimport?view=vs-2017 - * - * Do NOT include this file directly. Instead, use c10/macros/Macros.h - */ - -// You do not need to edit this part of file unless you are changing the core -// pytorch export abstractions. -// -// This part defines the C10 core export and import macros. This is controlled -// by whether we are building shared libraries or not, which is determined -// during build time and codified in c10/core/cmake_macros.h. -// When the library is built as a shared lib, EXPORT and IMPORT will contain -// visibility attributes. If it is being built as a static lib, then EXPORT -// and IMPORT basically have no effect. - -// As a rule of thumb, you should almost NEVER mix static and shared builds for -// libraries that depend on c10. AKA, if c10 is built as a static library, we -// recommend everything dependent on c10 to be built statically. If c10 is built -// as a shared library, everything dependent on it should be built as shared. In -// the PyTorch project, all native libraries shall use the macro -// C10_BUILD_SHARED_LIB to check whether pytorch is building shared or static -// libraries. - -// For build systems that do not directly depend on CMake and directly build -// from the source directory (such as Buck), one may not have a cmake_macros.h -// file at all. In this case, the build system is responsible for providing -// correct macro definitions corresponding to the cmake_macros.h.in file. -// -// In such scenarios, one should define the macro -// C10_USING_CUSTOM_GENERATED_MACROS -// to inform this header that it does not need to include the cmake_macros.h -// file. - #ifndef C10_USING_CUSTOM_GENERATED_MACROS #include #endif // C10_USING_CUSTOM_GENERATED_MACROS -#ifdef _WIN32 -#define C10_HIDDEN -#if defined(C10_BUILD_SHARED_LIBS) -#define C10_EXPORT __declspec(dllexport) -#define C10_IMPORT __declspec(dllimport) -#else -#define C10_EXPORT -#define C10_IMPORT -#endif -#else // _WIN32 -#if defined(__GNUC__) -#define C10_EXPORT __attribute__((__visibility__("default"))) -#define C10_HIDDEN __attribute__((__visibility__("hidden"))) -#else // defined(__GNUC__) -#define C10_EXPORT -#define C10_HIDDEN -#endif // defined(__GNUC__) -#define C10_IMPORT C10_EXPORT -#endif // _WIN32 - -#ifdef NO_EXPORT -#undef C10_EXPORT -#define C10_EXPORT -#endif - -// Definition of an adaptive XX_API macro, that depends on whether you are -// building the library itself or not, routes to XX_EXPORT and XX_IMPORT. -// Basically, you will need to do this for each shared library that you are -// building, and the instruction is as follows: assuming that you are building -// a library called libawesome.so. You should: -// (1) for your cmake target (usually done by "add_library(awesome, ...)"), -// define a macro called AWESOME_BUILD_MAIN_LIB using -// target_compile_options. -// (2) define the AWESOME_API macro similar to the one below. -// And in the source file of your awesome library, use AWESOME_API to -// annotate public symbols. - -// Here, for the C10 library, we will define the macro C10_API for both import -// and export. - -// This one is being used by libc10.so -#ifdef C10_BUILD_MAIN_LIB -#define C10_API C10_EXPORT -#else -#define C10_API C10_IMPORT -#endif +#include // This one is being used by libtorch.so #ifdef CAFFE2_BUILD_MAIN_LIB @@ -159,4 +75,4 @@ #define C10_API_ENUM #endif -#endif // C10_MACROS_MACROS_H_ +#endif // C10_MACROS_EXPORT_H_ diff --git a/runtime/core/portable_type/c10/c10/targets.bzl b/runtime/core/portable_type/c10/c10/targets.bzl index cb41bd0bb8e..5f964e65e87 100644 --- a/runtime/core/portable_type/c10/c10/targets.bzl +++ b/runtime/core/portable_type/c10/c10/targets.bzl @@ -112,7 +112,9 @@ def define_common_targets(): "//executorch/...", "@EXECUTORCH_CLIENTS", ], - deps = select({ + exported_deps = [ + "//executorch/runtime/core/portable_type/c10/torch/standalone:torch_standalone_headers", + ] + select({ "DEFAULT": [], # Half-inl.h depends on vec_half.h from ATen, but only when building for x86. "ovr_config//cpu:x86_64": [ diff --git a/runtime/core/portable_type/c10/torch/standalone/TARGETS b/runtime/core/portable_type/c10/torch/standalone/TARGETS new file mode 100644 index 00000000000..0a42614a385 --- /dev/null +++ b/runtime/core/portable_type/c10/torch/standalone/TARGETS @@ -0,0 +1,5 @@ +load(":targets.bzl", "define_common_targets") + +oncall("executorch") + +define_common_targets() diff --git a/runtime/core/portable_type/c10/torch/standalone/macros/Export.h b/runtime/core/portable_type/c10/torch/standalone/macros/Export.h new file mode 100644 index 00000000000..183aeab5634 --- /dev/null +++ b/runtime/core/portable_type/c10/torch/standalone/macros/Export.h @@ -0,0 +1,87 @@ +#pragma once + +/* Header file to define the common scaffolding for exported symbols. + * + * Export is by itself a quite tricky situation to deal with, and if you are + * hitting this file, make sure you start with the background here: + * - Linux: https://gcc.gnu.org/wiki/Visibility + * - Windows: + * https://docs.microsoft.com/en-us/cpp/cpp/dllexport-dllimport?view=vs-2017 + * + * Do NOT include this file directly. Instead, use c10/macros/Macros.h + */ + +// You do not need to edit this part of file unless you are changing the core +// pytorch export abstractions. +// +// This part defines the C10 core export and import macros. This is controlled +// by whether we are building shared libraries or not, which is determined +// during build time and codified in c10/core/cmake_macros.h. +// When the library is built as a shared lib, EXPORT and IMPORT will contain +// visibility attributes. If it is being built as a static lib, then EXPORT +// and IMPORT basically have no effect. + +// As a rule of thumb, you should almost NEVER mix static and shared builds for +// libraries that depend on c10. AKA, if c10 is built as a static library, we +// recommend everything dependent on c10 to be built statically. If c10 is built +// as a shared library, everything dependent on it should be built as shared. In +// the PyTorch project, all native libraries shall use the macro +// C10_BUILD_SHARED_LIB to check whether pytorch is building shared or static +// libraries. + +// For build systems that do not directly depend on CMake and directly build +// from the source directory (such as Buck), one may not have a cmake_macros.h +// file at all. In this case, the build system is responsible for providing +// correct macro definitions corresponding to the cmake_macros.h.in file. +// +// In such scenarios, one should define the macro +// C10_USING_CUSTOM_GENERATED_MACROS +// to inform this header that it does not need to include the cmake_macros.h +// file. + +#ifdef _WIN32 +#define C10_HIDDEN +#if defined(C10_BUILD_SHARED_LIBS) +#define C10_EXPORT __declspec(dllexport) +#define C10_IMPORT __declspec(dllimport) +#else +#define C10_EXPORT +#define C10_IMPORT +#endif +#else // _WIN32 +#if defined(__GNUC__) +#define C10_EXPORT __attribute__((__visibility__("default"))) +#define C10_HIDDEN __attribute__((__visibility__("hidden"))) +#else // defined(__GNUC__) +#define C10_EXPORT +#define C10_HIDDEN +#endif // defined(__GNUC__) +#define C10_IMPORT C10_EXPORT +#endif // _WIN32 + +#ifdef NO_EXPORT +#undef C10_EXPORT +#define C10_EXPORT +#endif + +// Definition of an adaptive XX_API macro, that depends on whether you are +// building the library itself or not, routes to XX_EXPORT and XX_IMPORT. +// Basically, you will need to do this for each shared library that you are +// building, and the instruction is as follows: assuming that you are building +// a library called libawesome.so. You should: +// (1) for your cmake target (usually done by "add_library(awesome, ...)"), +// define a macro called AWESOME_BUILD_MAIN_LIB using +// target_compile_options. +// (2) define the AWESOME_API macro similar to the one below. +// And in the source file of your awesome library, use AWESOME_API to +// annotate public symbols. + +// Here, for the C10 library, we will define the macro C10_API for both import +// and export. + +// This one is being used by libc10.so +#ifdef C10_BUILD_MAIN_LIB +#define C10_API C10_EXPORT +#else +#define C10_API C10_IMPORT +#endif diff --git a/runtime/core/portable_type/c10/torch/standalone/targets.bzl b/runtime/core/portable_type/c10/torch/standalone/targets.bzl new file mode 100644 index 00000000000..1faf1173a4a --- /dev/null +++ b/runtime/core/portable_type/c10/torch/standalone/targets.bzl @@ -0,0 +1,14 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +def define_common_targets(): + """Defines targets that should be shared between fbcode and xplat. + + The directory containing this targets.bzl file should also contain both + TARGETS and BUCK files that call this function. + """ + + runtime.cxx_library( + name = "torch_standalone_headers", + exported_headers = glob(["**/*.h"]), + header_namespace = "torch/standalone", + )