Update on "Introduce NamedDataStore"

Introduce NamedDataStore for weight sharing.
Rename 'NamedBlobStore' --> 'NamedDataStore' to mirror 'NamedDataMap' in the runtime.


The NamedDataStore exposes two methods:
- add_named_data: add a blob to the store
- get_named_data_store_output: return the contents of the store, to pass to serialization.

Invariants on the NamedDataStore
- Keys are unique regardless of whether they are in PTE or external file.
- Different keys can point to the same data.

NamedDataStore is used in D69764150. It's owned by the EdgeProgramManager.

Differential Revision: [D69764094](https://our.internmc.facebook.com/intern/diff/D69764094/)

[ghstack-poisoned]

Author: lucylq

.buckconfig

@@ -8,14 +8,15 @@
   root = .
   prelude = third-party/prelude
   shim = shim
+  shim_et = shim_et
 
 [repository_aliases]
   config = prelude
   ovr_config = prelude
-  toolchains = shim
-  fbcode = shim
-  fbcode_macros = shim
-  fbsource = shim
+  toolchains = shim_et
+  fbcode = shim_et
+  fbcode_macros = shim_et
+  fbsource = shim_et
   buck = shim
 
 [cxx]

.ci/scripts/gather_test_models.py

@@ -90,7 +90,7 @@ def model_should_run_on_event(model: str, event: str) -> bool:
     We put higher priority and fast models to pull request and rest to push.
     """
     if event == "pull_request":
-        return model in ["mv3", "vit"]
+        return model in ["mv3", "vit", "qwen2_5"]  # TODO: remove, just to test the ci
     elif event == "push":
         # These are super slow. Only run it periodically
         return model not in ["dl3", "edsr", "emformer_predict"]

.ci/scripts/setup-linux.sh

@@ -22,7 +22,7 @@ fi
 # have already been installed, so we use PyTorch build from source here instead
 # of nightly. This allows CI to test against latest commits from PyTorch
 install_executorch "use-pt-pinned-commit"
-build_executorch_runner "${BUILD_TOOL}"
+build_executorch_runner "${BUILD_TOOL}" "${2:-Release}"
 
 if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then
   do_not_use_nightly_on_ci

.ci/scripts/setup-macos.sh

@@ -136,7 +136,7 @@ install_pytorch_and_domains
 # We build PyTorch from source here instead of using nightly. This allows CI to test against
 # the pinned commit from PyTorch
 install_executorch "use-pt-pinned-commit"
-build_executorch_runner "${BUILD_TOOL}"
+build_executorch_runner "${BUILD_TOOL}" "${2:-Release}"
 
 if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then
   do_not_use_nightly_on_ci

.ci/scripts/test_model.sh

@@ -91,7 +91,17 @@ test_model() {
     # Install requirements for llama vision.
     bash examples/models/llama3_2_vision/install_requirements.sh
   fi
-  # python3 -m examples.portable.scripts.export --model_name="llama2" should works too
+  if [[ "${MODEL_NAME}" == "qwen2_5" ]]; then
+      # Install requirements for export_llama
+      bash examples/models/llama/install_requirements.sh
+      # Test export_llama script: python3 -m examples.models.llama.export_llama.
+      # Use Llama random checkpoint with Qwen 2.5 1.5b model configuration.
+      "${PYTHON_EXECUTABLE}" -m examples.models.llama.export_llama --model "${MODEL_NAME}" -c examples/models/llama/params/demo_rand_params.pth -p examples/models/qwen2_5/1_5b_config.json
+      rm "./${MODEL_NAME}.pte"
+      return  # Skip running with portable executor runnner since portable doesn't support Qwen's biased linears.
+  fi
+
+  # Export a basic .pte and run the model.
   "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export --model_name="${MODEL_NAME}" "${STRICT}"
   run_portable_executor_runner
 }

.ci/scripts/test_qnn_static_llama.sh

@@ -34,11 +34,11 @@ $PYTHON_EXECUTABLE -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o to
 
 set +e
 # Compile only as weight sharing is not applicable on x86
-$PYTHON_EXECUTABLE backends/qualcomm/tests/test_qnn_delegate.py -k TestExampleScript.test_stories_single_llama --model SM8650 --build_folder build-android/ --executorch_root . --artifact_dir . --compile_only
+$PYTHON_EXECUTABLE backends/qualcomm/tests/test_qnn_delegate.py -k TestExampleLLMScript.test_llama_stories_110m --model SM8650 --build_folder build-android/ --executorch_root . --artifact_dir . --llama_artifacts . --compile_only
 exit_code1=$?
 
 # Checks accuracy with weight sharing disabled since x86 does not support weight sharing.
-$PYTHON_EXECUTABLE backends/qualcomm/tests/test_qnn_delegate.py -k TestExampleScript.test_stories_single_llama --model SM8650 --build_folder build-x86/ --executorch_root . --artifact_dir . --enable_x86_64
+$PYTHON_EXECUTABLE backends/qualcomm/tests/test_qnn_delegate.py -k TestExampleLLMScript.test_llama_stories_110m --model SM8650 --build_folder build-x86/ --executorch_root . --artifact_dir . --llama_artifacts . --enable_x86_64
 exit_code2=$?
 
 # Check the exit codes and print messages

.ci/scripts/unittest-buck2.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+# TODO: expand this to //...
+buck2 query //runtime/...
+
+# TODO: expand the covered scope of Buck targets.
+buck2 build //runtime/core/portable_type/...
+buck2 test //runtime/core/portable_type/...

.ci/scripts/unittest-linux-cmake.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+# Run pytest with coverage
+pytest -n auto --cov=./ --cov-report=xml
+# Run gtest
+LLVM_PROFDATA=llvm-profdata-12 LLVM_COV=llvm-cov-12 \
+test/run_oss_cpp_tests.sh

.ci/scripts/unittest-linux.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+BUILD_TOOL=$1
+if [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then
+    echo "Running unittests for ${BUILD_TOOL} ..."
+else
+  echo "Missing build tool (require buck2 or cmake), exiting..."
+  exit 1
+fi
+
+BUILD_MODE=$2
+if [[ "${BUILD_MODE:-}" =~ ^(Debug|Release)$ ]]; then
+    echo "Running tests in build mode ${BUILD_MODE} ..."
+else
+    echo "Unsupported build mode ${BUILD_MODE}, options are Debug or Release."
+    exit 1
+fi
+
+# The generic Linux job chooses to use base env, not the one setup by the image
+eval "$(conda shell.bash hook)"
+CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+conda activate "${CONDA_ENV}"
+
+# Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
+source .ci/scripts/setup-vulkan-linux-deps.sh
+
+PYTHON_EXECUTABLE=python \
+EXECUTORCH_BUILD_PYBIND=ON \
+CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
+.ci/scripts/setup-linux.sh "$BUILD_TOOL" "$BUILD_MODE"
+
+# Install llama3_2_vision dependencies.
+PYTHON_EXECUTABLE=python ./examples/models/llama3_2_vision/install_requirements.sh
+
+if [[ "$BUILD_TOOL" == "cmake" ]]; then
+    .ci/scripts/unittest-linux-cmake.sh
+elif [[ "$BUILD_TOOL" == "buck2" ]]; then
+    .ci/scripts/unittest-buck2.sh
+else
+    echo "Unknown build tool $BUILD_TOOL"
+    exit 1
+fi

.ci/scripts/unittest-macos-cmake.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+# Run pytest with coverage
+${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml
+# Run gtest
+LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \
+${CONDA_RUN} test/run_oss_cpp_tests.sh