Update install_requirements.sh to coremltools 9.0 (#15839)

As titled

Co-authored-by: Scott Roy <[email protected]>

Author: pytorchbot

.ci/scripts/utils.sh

@@ -46,8 +46,8 @@ install_pip_dependencies() {
 
 install_domains() {
   echo "Install torchvision and torchaudio"
-  pip install --no-use-pep517 --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}"
-  pip install --no-use-pep517 --user "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}"
+  pip install --no-build-isolation --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}"
+  pip install --no-build-isolation --user "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}"
 }
 
 install_pytorch_and_domains() {

backends/apple/coreml/scripts/build_tests.sh

@@ -30,7 +30,8 @@ rm -rf "$CMAKE_EXECUTORCH_BUILD_DIR_PATH"
 
 cmake "$EXECUTORCH_ROOT_PATH" -B"$CMAKE_EXECUTORCH_BUILD_DIR_PATH" \
 -DCMAKE_TOOLCHAIN_FILE="$IOS_TOOLCHAIN_PATH" \
--DPLATFORM=MAC_UNIVERSAL \
+-DPLATFORM=MAC_ARM64 \
+-DCMAKE_OSX_ARCHITECTURES=arm64 \
 -DDEPLOYMENT_TARGET=13.0 \
 -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \
 -DEXECUTORCH_BUILD_XNNPACK=OFF
@@ -44,7 +45,8 @@ rm -rf "$CMAKE_PROTOBUF_BUILD_DIR_PATH"
 
 cmake "$PROTOBUF_DIR_PATH/cmake" -B"$CMAKE_PROTOBUF_BUILD_DIR_PATH" \
 -DCMAKE_TOOLCHAIN_FILE="$IOS_TOOLCHAIN_PATH" \
--DPLATFORM=MAC_UNIVERSAL \
+-DPLATFORM=MAC_ARM64 \
+-DCMAKE_OSX_ARCHITECTURES=arm64 \
 -DDEPLOYMENT_TARGET=13.0 \
 -Dprotobuf_BUILD_TESTS=OFF \
 -Dprotobuf_BUILD_EXAMPLES=OFF \
@@ -55,7 +57,8 @@ cmake --build "$CMAKE_PROTOBUF_BUILD_DIR_PATH"  -j9 -t libprotobuf-lite
 
 # Copy required libraries
 echo "ExecuTorch: Copying libraries"
-mkdir "$LIBRARIES_DIR_PATH"
+rm -rf $LIBRARIES_DIR_PATH
+mkdir -p "$LIBRARIES_DIR_PATH"
 cp -f "$CMAKE_EXECUTORCH_BUILD_DIR_PATH/libexecutorch.a" "$LIBRARIES_DIR_PATH"
 cp -f "$CMAKE_EXECUTORCH_BUILD_DIR_PATH/libexecutorch_core.a" "$LIBRARIES_DIR_PATH"
 cp -f "$CMAKE_PROTOBUF_BUILD_DIR_PATH/libprotobuf-lite.a" "$LIBRARIES_DIR_PATH"

backends/apple/coreml/scripts/generate_test_models.sh

@@ -15,7 +15,9 @@ COREML_DIR_PATH="$EXECUTORCH_ROOT_PATH/backends/apple/coreml"
 
 cd "$EXECUTORCH_ROOT_PATH"
 
-mkdir "$COREML_DIR_PATH/runtime/test/models/"
+rm -rf "$COREML_DIR_PATH/runtime/test/models/"
+mkdir -p "$COREML_DIR_PATH/runtime/test/models/"
+
 #Generate models
 cd "$EXECUTORCH_ROOT_PATH"
 

backends/apple/coreml/scripts/install_requirements.sh

@@ -5,17 +5,25 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+set -euo pipefail
+
 SCRIPT_DIR_PATH="$(
     cd -- "$(dirname "$0")" >/dev/null 2>&1
     pwd -P
 )"
 
 # TODO(jathu): remove the need to fetch coremltools to build deps for coreml_executor_runner.
 # Keep this version in sync with: pyproject.toml
-COREMLTOOLS_VERSION="9.0b1"
+COREMLTOOLS_VERSION="9.0"
 
-red=`tput setaf 1`
-green=`tput setaf 2`
+# Safe colors (no TERM noise in CI)
+if command -v tput >/dev/null 2>&1 && [ -t 1 ] && [ -n "${TERM:-}" ]; then
+    red="$(tput setaf 1)"
+    green="$(tput setaf 2)"
+    reset="$(tput sgr0)"
+else
+    red=""; green=""; reset=""
+fi
 
 EXECUTORCH_ROOT_PATH=$(realpath "$SCRIPT_DIR_PATH/../../../../")
 COREML_DIR_PATH="$EXECUTORCH_ROOT_PATH/backends/apple/coreml"
@@ -25,30 +33,79 @@ PROTOBUF_FILES_DIR_PATH="$COREMLTOOLS_DIR_PATH/build/mlmodel/format/"
 cd "$EXECUTORCH_ROOT_PATH"
 
 rm -rf "$COREML_DIR_PATH/third-party"
-mkdir "$COREML_DIR_PATH/third-party"
+mkdir -p "$COREML_DIR_PATH/third-party"
 
-echo "${green}ExecuTorch: Cloning coremltools."
-git clone --depth 1 --branch "${COREMLTOOLS_VERSION}" "https://github.com/apple/coremltools.git" $COREMLTOOLS_DIR_PATH
-cd $COREMLTOOLS_DIR_PATH
+echo "${green}ExecuTorch: Cloning coremltools.${reset}"
+git clone --depth 1 --branch "${COREMLTOOLS_VERSION}" "https://github.com/apple/coremltools.git" "$COREMLTOOLS_DIR_PATH"
+cd "$COREMLTOOLS_DIR_PATH"
 
 STATUS=$?
 if [ $STATUS -ne 0 ]; then
-    echo "${red}ExecuTorch: Failed to clone coremltools."
+    echo "${red}ExecuTorch: Failed to clone coremltools.${reset}"
     exit 1
 fi
 
-echo "${green}ExecuTorch: Installing coremltools dependencies."
-pip install -r "$COREMLTOOLS_DIR_PATH/reqs/build.pip"
+# ---------------------------------------------------------------------
+# Host toolchain / SDK setup JUST for coremltools build
+# ---------------------------------------------------------------------
+HOST_SDKROOT="${SDKROOT:-}"
+HOST_CC="${CC:-}"
+HOST_CXX="${CXX:-}"
+HOST_CFLAGS="${CFLAGS:-}"
+HOST_CXXFLAGS="${CXXFLAGS:-}"
+
+if [[ "$(uname)" == "Darwin" ]]; then
+    # Only pick macOS SDK if nothing else is specified
+    if [[ -z "$HOST_SDKROOT" ]]; then
+        HOST_SDKROOT="$(xcrun --sdk macosx --show-sdk-path)"
+    fi
+    if [[ -z "$HOST_CC" ]]; then
+        HOST_CC="$(xcrun --find clang)"
+    fi
+    if [[ -z "$HOST_CXX" ]]; then
+        HOST_CXX="$(xcrun --find clang++)"
+    fi
+    # Only add -isysroot if caller didn't already set CFLAGS/CXXFLAGS
+    if [[ -z "$HOST_CFLAGS" && -n "$HOST_SDKROOT" ]]; then
+        HOST_CFLAGS="-isysroot ${HOST_SDKROOT}"
+    fi
+    if [[ -z "$HOST_CXXFLAGS" && -n "$HOST_SDKROOT" ]]; then
+        HOST_CXXFLAGS="-isysroot ${HOST_SDKROOT}"
+    fi
+fi
+
+echo "${green}ExecuTorch: Installing coremltools dependencies.${reset}"
+SDKROOT="$HOST_SDKROOT" \
+CC="$HOST_CC" \
+CXX="$HOST_CXX" \
+CFLAGS="$HOST_CFLAGS" \
+CXXFLAGS="$HOST_CXXFLAGS" \
+python -m pip install -r "$COREMLTOOLS_DIR_PATH/reqs/build.pip"
 STATUS=$?
 if [ $STATUS -ne 0 ]; then
-    echo "${red}ExecuTorch: Failed to install coremltools dependencies."
+    echo "${red}ExecuTorch: Failed to install coremltools dependencies.${reset}"
     exit 1
 fi
 
-mkdir "$COREMLTOOLS_DIR_PATH/build"
+mkdir -p "$COREMLTOOLS_DIR_PATH/build"
+
+echo "${green}ExecuTorch: Configuring coremltools CMake build.${reset}"
+SDKROOT="$HOST_SDKROOT" \
+CC="$HOST_CC" \
+CXX="$HOST_CXX" \
+CFLAGS="$HOST_CFLAGS" \
+CXXFLAGS="$HOST_CXXFLAGS" \
 cmake -S "$COREMLTOOLS_DIR_PATH" -B "$COREMLTOOLS_DIR_PATH/build"
+
+echo "${green}ExecuTorch: Building mlmodel target.${reset}"
+SDKROOT="$HOST_SDKROOT" \
+CC="$HOST_CC" \
+CXX="$HOST_CXX" \
+CFLAGS="$HOST_CFLAGS" \
+CXXFLAGS="$HOST_CXXFLAGS" \
 cmake --build "$COREMLTOOLS_DIR_PATH/build" --parallel --target mlmodel
 
-echo "${green}ExecuTorch: Copying protobuf files."
+echo "${green}ExecuTorch: Copying protobuf files.${reset}"
+rm -rf "$COREML_DIR_PATH/runtime/sdk/format/"
 mkdir -p "$COREML_DIR_PATH/runtime/sdk/format/"
 cp -rf "$PROTOBUF_FILES_DIR_PATH" "$COREML_DIR_PATH/runtime/sdk/format/"