Bump pinned PT commit to 20250616 and mirror torch/standalone to ExecuTorch

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-5616fa4a68718ead203314a3467f7dd9547153ae
+c620d0b5c7e8679413d620624725471223ce8359

.ci/docker/common/install_conda.sh

@@ -13,7 +13,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 install_miniconda() {
   BASE_URL="https://repo.anaconda.com/miniconda"
   CONDA_FILE="Miniconda3-py${PYTHON_VERSION//./}_${MINICONDA_VERSION}-Linux-x86_64.sh"
-  if [[ $(uname -m) == "aarch64" ]]; then 
+  if [[ $(uname -m) == "aarch64" ]]; then
     CONDA_FILE="Miniconda3-py${PYTHON_VERSION//./}_${MINICONDA_VERSION}-Linux-aarch64.sh"
   fi
 
@@ -71,4 +71,8 @@ fix_conda_ubuntu_libstdcxx() {
 install_miniconda
 install_python
 install_pip_dependencies
-fix_conda_ubuntu_libstdcxx
+# Hack breaks the job on aarch64 but is still necessary everywhere
+# else.
+if [ "$(uname -m)" != "aarch64" ]; then
+    fix_conda_ubuntu_libstdcxx
+fi

.ci/docker/conda-env-ci.txt

@@ -1,4 +1,4 @@
-cmake=3.26.4
+cmake=3.31.2
 ninja=1.10.2
 libuv
 llvm-openmp

.ci/scripts/check_c10_sync.sh

@@ -12,3 +12,4 @@ pushd pytorch
 git checkout "$pytorch_pin"
 popd
 "$(dirname "${BASH_SOURCE[0]}")"/compare_dirs.sh runtime/core/portable_type/c10/c10 pytorch/c10
+"$(dirname "${BASH_SOURCE[0]}")"/compare_dirs.sh runtime/core/portable_type/c10/torch/standalone pytorch/torch/standalone

install_requirements.py

@@ -71,7 +71,7 @@ def python_is_compatible():
 #
 # NOTE: If you're changing, make the corresponding change in .ci/docker/ci_commit_pins/pytorch.txt
 # by picking the hash from the same date in https://hud.pytorch.org/hud/pytorch/pytorch/nightly/
-NIGHTLY_VERSION = "dev20250601"
+NIGHTLY_VERSION = "dev20250616"
 
 
 def install_requirements(use_pytorch_nightly):

runtime/core/portable_type/c10/README.md

@@ -11,3 +11,10 @@ would cause all headers in that directory to be includeable with
 `#include <foo.h>`. In particular, that includes
 `runtime/core/portable_type/complex.h`, which would shadow the C99
 `complex.h` standard header.
+
+`torch/standalone` has been added as an extra "even more bottom of
+stack" directory in PyTorch, so we have to add it to our sync
+here. The extra "stutter" c10 directory causing `c10/torch/standlone`
+is unfortunately awkward; perhaps we can rename the top-level
+directory to `pytorch_embedded_mirror` when we have extra time to work
+through CI failures.

runtime/core/portable_type/c10/c10/macros/Export.h

@@ -1,95 +1,11 @@
 #ifndef C10_MACROS_EXPORT_H_
 #define C10_MACROS_EXPORT_H_
 
-/* Header file to define the common scaffolding for exported symbols.
- *
- * Export is by itself a quite tricky situation to deal with, and if you are
- * hitting this file, make sure you start with the background here:
- * - Linux: https://gcc.gnu.org/wiki/Visibility
- * - Windows:
- * https://docs.microsoft.com/en-us/cpp/cpp/dllexport-dllimport?view=vs-2017
- *
- * Do NOT include this file directly. Instead, use c10/macros/Macros.h
- */
-
-// You do not need to edit this part of file unless you are changing the core
-// pytorch export abstractions.
-//
-// This part defines the C10 core export and import macros. This is controlled
-// by whether we are building shared libraries or not, which is determined
-// during build time and codified in c10/core/cmake_macros.h.
-// When the library is built as a shared lib, EXPORT and IMPORT will contain
-// visibility attributes. If it is being built as a static lib, then EXPORT
-// and IMPORT basically have no effect.
-
-// As a rule of thumb, you should almost NEVER mix static and shared builds for
-// libraries that depend on c10. AKA, if c10 is built as a static library, we
-// recommend everything dependent on c10 to be built statically. If c10 is built
-// as a shared library, everything dependent on it should be built as shared. In
-// the PyTorch project, all native libraries shall use the macro
-// C10_BUILD_SHARED_LIB to check whether pytorch is building shared or static
-// libraries.
-
-// For build systems that do not directly depend on CMake and directly build
-// from the source directory (such as Buck), one may not have a cmake_macros.h
-// file at all. In this case, the build system is responsible for providing
-// correct macro definitions corresponding to the cmake_macros.h.in file.
-//
-// In such scenarios, one should define the macro
-//     C10_USING_CUSTOM_GENERATED_MACROS
-// to inform this header that it does not need to include the cmake_macros.h
-// file.
-
 #ifndef C10_USING_CUSTOM_GENERATED_MACROS
 #include <c10/macros/cmake_macros.h>
 #endif // C10_USING_CUSTOM_GENERATED_MACROS
 
-#ifdef _WIN32
-#define C10_HIDDEN
-#if defined(C10_BUILD_SHARED_LIBS)
-#define C10_EXPORT __declspec(dllexport)
-#define C10_IMPORT __declspec(dllimport)
-#else
-#define C10_EXPORT
-#define C10_IMPORT
-#endif
-#else // _WIN32
-#if defined(__GNUC__)
-#define C10_EXPORT __attribute__((__visibility__("default")))
-#define C10_HIDDEN __attribute__((__visibility__("hidden")))
-#else // defined(__GNUC__)
-#define C10_EXPORT
-#define C10_HIDDEN
-#endif // defined(__GNUC__)
-#define C10_IMPORT C10_EXPORT
-#endif // _WIN32
-
-#ifdef NO_EXPORT
-#undef C10_EXPORT
-#define C10_EXPORT
-#endif
-
-// Definition of an adaptive XX_API macro, that depends on whether you are
-// building the library itself or not, routes to XX_EXPORT and XX_IMPORT.
-// Basically, you will need to do this for each shared library that you are
-// building, and the instruction is as follows: assuming that you are building
-// a library called libawesome.so. You should:
-// (1) for your cmake target (usually done by "add_library(awesome, ...)"),
-//     define a macro called AWESOME_BUILD_MAIN_LIB using
-//     target_compile_options.
-// (2) define the AWESOME_API macro similar to the one below.
-// And in the source file of your awesome library, use AWESOME_API to
-// annotate public symbols.
-
-// Here, for the C10 library, we will define the macro C10_API for both import
-// and export.
-
-// This one is being used by libc10.so
-#ifdef C10_BUILD_MAIN_LIB
-#define C10_API C10_EXPORT
-#else
-#define C10_API C10_IMPORT
-#endif
+#include <torch/standalone/macros/Export.h>
 
 // This one is being used by libtorch.so
 #ifdef CAFFE2_BUILD_MAIN_LIB
@@ -159,4 +75,4 @@
 #define C10_API_ENUM
 #endif
 
-#endif // C10_MACROS_MACROS_H_
+#endif // C10_MACROS_EXPORT_H_

runtime/core/portable_type/c10/c10/targets.bzl

@@ -78,7 +78,9 @@ def define_common_targets():
             "//executorch/...",
             "@EXECUTORCH_CLIENTS",
         ],
-        deps = select({
+        exported_deps = [
+            "//executorch/runtime/core/portable_type/c10/torch/standalone:torch_standalone_headers",
+        ] + select({
             "DEFAULT": [],
             # Half-inl.h depends on vec_half.h from ATen, but only when building for x86.
             "ovr_config//cpu:x86_64": [

runtime/core/portable_type/c10/torch/standalone/TARGETS

@@ -0,0 +1,5 @@
+load(":targets.bzl", "define_common_targets")
+
+oncall("executorch")
+
+define_common_targets()

runtime/core/portable_type/c10/torch/standalone/macros/Export.h

@@ -0,0 +1,87 @@
+#pragma once
+
+/* Header file to define the common scaffolding for exported symbols.
+ *
+ * Export is by itself a quite tricky situation to deal with, and if you are
+ * hitting this file, make sure you start with the background here:
+ * - Linux: https://gcc.gnu.org/wiki/Visibility
+ * - Windows:
+ * https://docs.microsoft.com/en-us/cpp/cpp/dllexport-dllimport?view=vs-2017
+ *
+ * Do NOT include this file directly. Instead, use c10/macros/Macros.h
+ */
+
+// You do not need to edit this part of file unless you are changing the core
+// pytorch export abstractions.
+//
+// This part defines the C10 core export and import macros. This is controlled
+// by whether we are building shared libraries or not, which is determined
+// during build time and codified in c10/core/cmake_macros.h.
+// When the library is built as a shared lib, EXPORT and IMPORT will contain
+// visibility attributes. If it is being built as a static lib, then EXPORT
+// and IMPORT basically have no effect.
+
+// As a rule of thumb, you should almost NEVER mix static and shared builds for
+// libraries that depend on c10. AKA, if c10 is built as a static library, we
+// recommend everything dependent on c10 to be built statically. If c10 is built
+// as a shared library, everything dependent on it should be built as shared. In
+// the PyTorch project, all native libraries shall use the macro
+// C10_BUILD_SHARED_LIB to check whether pytorch is building shared or static
+// libraries.
+
+// For build systems that do not directly depend on CMake and directly build
+// from the source directory (such as Buck), one may not have a cmake_macros.h
+// file at all. In this case, the build system is responsible for providing
+// correct macro definitions corresponding to the cmake_macros.h.in file.
+//
+// In such scenarios, one should define the macro
+//     C10_USING_CUSTOM_GENERATED_MACROS
+// to inform this header that it does not need to include the cmake_macros.h
+// file.
+
+#ifdef _WIN32
+#define C10_HIDDEN
+#if defined(C10_BUILD_SHARED_LIBS)
+#define C10_EXPORT __declspec(dllexport)
+#define C10_IMPORT __declspec(dllimport)
+#else
+#define C10_EXPORT
+#define C10_IMPORT
+#endif
+#else // _WIN32
+#if defined(__GNUC__)
+#define C10_EXPORT __attribute__((__visibility__("default")))
+#define C10_HIDDEN __attribute__((__visibility__("hidden")))
+#else // defined(__GNUC__)
+#define C10_EXPORT
+#define C10_HIDDEN
+#endif // defined(__GNUC__)
+#define C10_IMPORT C10_EXPORT
+#endif // _WIN32
+
+#ifdef NO_EXPORT
+#undef C10_EXPORT
+#define C10_EXPORT
+#endif
+
+// Definition of an adaptive XX_API macro, that depends on whether you are
+// building the library itself or not, routes to XX_EXPORT and XX_IMPORT.
+// Basically, you will need to do this for each shared library that you are
+// building, and the instruction is as follows: assuming that you are building
+// a library called libawesome.so. You should:
+// (1) for your cmake target (usually done by "add_library(awesome, ...)"),
+//     define a macro called AWESOME_BUILD_MAIN_LIB using
+//     target_compile_options.
+// (2) define the AWESOME_API macro similar to the one below.
+// And in the source file of your awesome library, use AWESOME_API to
+// annotate public symbols.
+
+// Here, for the C10 library, we will define the macro C10_API for both import
+// and export.
+
+// This one is being used by libc10.so
+#ifdef C10_BUILD_MAIN_LIB
+#define C10_API C10_EXPORT
+#else
+#define C10_API C10_IMPORT
+#endif

runtime/core/portable_type/c10/torch/standalone/targets.bzl

@@ -0,0 +1,14 @@
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+
+def define_common_targets():
+    """Defines targets that should be shared between fbcode and xplat.
+
+    The directory containing this targets.bzl file should also contain both
+    TARGETS and BUCK files that call this function.
+    """
+
+    runtime.cxx_library(
+        name = "torch_standalone_headers",
+        exported_headers = glob(["**/*.h"]),
+        header_namespace = "torch/standalone",
+    )