Merge branch 'main' into integrating-vision-kernels

Author: cad-rlc

.ci/docker/build.sh

@@ -43,6 +43,10 @@ case "${IMAGE_NAME}" in
     ARM_SDK=yes
     CLANG_VERSION=12
     ;;
+  executorch-ubuntu-22.04-zephyr-sdk)
+    ZEPHYR_SDK=yes
+    GCC_VERSION=11
+    ;;
   executorch-ubuntu-22.04-qnn-sdk)
     QNN_SDK=yes
     CLANG_VERSION=12
@@ -87,6 +91,7 @@ docker build \
   --build-arg "LINTRUNNER=${LINTRUNNER:-}" \
   --build-arg "BUILD_DOCS=${BUILD_DOCS}" \
   --build-arg "ARM_SDK=${ARM_SDK:-}" \
+  --build-arg "ZEPHYR_SDK=${ZEPHYR_SDK:-}" \
   --build-arg "QNN_SDK=${QNN_SDK:-}" \
   --build-arg "MEDIATEK_SDK=${MEDIATEK_SDK:-}" \
   --build-arg "ANDROID_NDK_VERSION=${ANDROID_NDK_VERSION:-}" \

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-ab43fe4bdf5ccd82897f0e982c451a0127bd175e
+6fc0ad22f0a07b6f38d138861c56a765d5a9bb02

.ci/docker/common/install_zephyr.sh

@@ -0,0 +1,92 @@
+
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+# Double check if the NDK version is set
+[ -n "${ZEPHYR_SDK}" ]
+
+install_prerequiresites() {
+    rm /var/lib/dpkg/info/libc-bin.*
+    apt-get clean
+    apt-get -y update
+    apt-get install -y libc-bin
+    apt-get -y update
+    apt-get clean
+    apt-get install --no-install-recommends -y dos2unix
+    apt-get install --no-install-recommends -y ca-certificates
+    apt-get install -y --reinstall libc-bin
+    apt-get install --no-install-recommends -y file
+    apt-get install --no-install-recommends -y locales
+    apt-get install --no-install-recommends -y git
+    apt-get install --no-install-recommends -y build-essential
+    apt-get install --no-install-recommends -y cmake
+    apt-get install --no-install-recommends -y ninja-build gperf
+    apt-get install --no-install-recommends -y device-tree-compiler
+    apt-get install --no-install-recommends -y wget
+    apt-get install --no-install-recommends -y curl
+    apt-get install --no-install-recommends -y xz-utils
+    apt-get install --no-install-recommends -y dos2unix
+    apt-get install --no-install-recommends -y vim
+    apt-get install --no-install-recommends -y nano
+    apt-get install --no-install-recommends -y mc
+    apt-get install --no-install-recommends -y openssh-server
+    apt-get install -y gdb
+
+    # Zephyr SDK relies on python 3.12
+    apt install software-properties-common -y
+    add-apt-repository ppa:deadsnakes/ppa -y
+    apt update
+    apt install -y python3.12 python3.12-dev python3.12-venv python3-pip
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
+
+    # Upgrade cmake ot 3.24
+    apt update
+    apt install cmake
+    apt install software-properties-common lsb-release
+    apt update
+    test -f /usr/share/doc/kitware-archive-keyring/copyright || \
+        wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+    "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list > /dev/null
+    apt update
+    apt install cmake
+
+    # Install additional required software for Zephyr
+    apt install --no-install-recommends -y ccache \
+        dfu-util \
+        python3-setuptools \
+        python3-tk \
+        python3-wheel \
+        make \
+        gcc \
+        libsdl2-dev \
+        libmagic1 \
+        xterm \
+        telnet \
+        net-tools
+    apt install --no-install-recommends -y gcc-multilib g++-multilib
+    apt-get clean -y
+    apt-get autoremove --purge -y
+    rm -rf /var/lib/apt/lists/*
+    wget https://apt.kitware.com/kitware-archive.sh && \
+        chmod +x kitware-archive.sh && \
+        ./kitware-archive.sh && \
+        rm -f kitware-archive.sh
+    useradd -d /home/zephyruser -m -s /bin/bash zephyruser
+}
+
+install_sdk() {
+    wget https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.0/zephyr-sdk-0.16.0_linux-x86_64.tar.xz && \
+         tar -xf zephyr-sdk-0.16.0_linux-x86_64.tar.xz && \
+         rm -f zephyr-sdk-0.16.0_linux-x86_64.tar.xz && \
+         cd zephyr-sdk-0.16.0/ && \
+         ./setup.sh -c -t arm-zephyr-eabi
+}
+
+install_prerequiresites
+install_sdk

.ci/docker/ubuntu/Dockerfile

@@ -84,6 +84,11 @@ RUN rm install_android.sh
 
 ARG ARM_SDK
 
+ARG ZEPHYR_SDK
+COPY ./common/install_zephyr.sh install_zephyr.sh
+RUN if [ -n "${ZEPHYR_SDK}" ]; then bash ./install_zephyr.sh; fi
+RUN rm install_zephyr.sh
+
 ARG QNN_SDK
 
 ARG MEDIATEK_SDK

.ci/scripts/build-qnn-sdk.sh

@@ -33,6 +33,8 @@ set_up_aot() {
       -DEXECUTORCH_BUILD_DEVTOOLS=ON \
       -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
       -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+      -DEXECUTORCH_BUILD_EXTENSION_EXTENSION_LLM=ON \
+      -DEXECUTORCH_BUILD_EXTENSION_EXTENSION_LLM_RUNNER=ON \
       -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
       -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
       -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \

.ci/scripts/build_llama_android.sh

@@ -19,7 +19,7 @@ install_executorch_and_backend_lib() {
   echo "Installing executorch and xnnpack backend"
   clean_executorch_install_folders
   mkdir cmake-android-out
-  ANDROID_NDK=/opt/ndk
+  ANDROID_NDK=${ANDROID_NDK:-/opt/ndk}
   BUCK2=buck2
   ANDROID_ABI=arm64-v8a
   cmake --preset llm \

.ci/scripts/setup-qnn-deps.sh

@@ -7,47 +7,7 @@
 
 set -ex
 
-verify_pkg_installed() {
-  echo $(dpkg-query -W --showformat='${Status}\n' $1|grep "install ok installed")
-}
+source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh"
 
-install_qnn() {
-  echo "Start installing qnn."
-  QNN_INSTALLATION_DIR=/tmp/qnn
-  mkdir -p "${QNN_INSTALLATION_DIR}"
-
-  curl -Lo /tmp/v2.28.0.24.10.29.zip "https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v2.28.0.241029.zip"
-  echo "Finishing downloading qnn sdk."
-  unzip -qo /tmp/v2.28.0.24.10.29.zip -d /tmp
-  echo "Finishing unzip qnn sdk."
-
-
-  # Print the content for manual verification
-  ls -lah "/tmp/qairt"
-  mv "/tmp/qairt"/* "${QNN_INSTALLATION_DIR}"
-  echo "Finishing installing qnn '${QNN_INSTALLATION_DIR}' ."
-
-  ls -lah "${QNN_INSTALLATION_DIR}"
-}
-
-setup_libc++() {
-  clang_version=$1
-  sudo apt-get update
-  pkgs_to_check=("libc++-${clang_version}-dev")
-  j=0
-  while [ $j -lt ${#pkgs_to_check[*]} ]; do
-    install_status=$(verify_pkg_installed ${pkgs_to_check[$j]})
-    if [ "$install_status" == "" ]; then
-      sudo apt-get install -y ${pkgs_to_check[$j]}
-      if [[ $? -ne 0 ]]; then
-        echo "ERROR: Failed to install required packages for libc++"
-        exit 1
-      fi
-    fi
-    j=$(( $j +1));
-  done
-}
-
-# This needs to match with the clang version from the Docker image
-setup_libc++ 12
+setup_libcpp 12
 install_qnn

.ci/scripts/test_ane_static_llama.sh

@@ -28,6 +28,6 @@ pushd $EXECUTORCH_ROOT/examples/apple/coreml/llama
 # Download stories llama110m artifacts
 download_stories_model_artifacts
 
-python export.py -n model.pte -p params.json -c stories110M.pt --seq_length 32 --max_seq_length 64 --dtype fp16 --coreml-quantize c4w
+python export.py -n model.pte -p params.json -c stories110M.pt --seq_length 32 --max_seq_length 64 --dtype fp16 --coreml-quantize c4w --embedding-quantize 4,32
 
 popd

.ci/scripts/test_llama.sh

@@ -150,6 +150,7 @@ cmake_install_executorch_libraries() {
     echo "Installing libexecutorch.a, libextension_module.so, libportable_ops_lib.a"
     rm -rf cmake-out
     retry cmake --preset llm \
+        -DBUILD_TESTING=OFF \
         -DCMAKE_INSTALL_PREFIX=cmake-out \
         -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \
         -DEXECUTORCH_BUILD_QNN="$QNN" \

.ci/scripts/test_llama_torchao_lowbit.sh

@@ -29,6 +29,8 @@ cmake -DPYTHON_EXECUTABLE=python \
     -DEXECUTORCH_ENABLE_LOGGING=1 \
     -DCMAKE_BUILD_TYPE=Release \
     -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_LLM=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_LLM_RUNNER=ON \
     -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
     -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
     -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \