Merge branch 'main' into vela-6d-io

Author: zingo

backends/apple/mps/serialization/mps_graph_serialize.py

@@ -1,14 +1,16 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
+# Copyright 2025 Arm Limited and/or its affiliates.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import importlib.resources as _resources
 import json
 import os
 import tempfile
 
-import pkg_resources
+import executorch.backends.apple.mps.serialization as serialization_package
 from executorch.backends.apple.mps.serialization.mps_graph_schema import MPSGraph
 from executorch.exir._serialize._dataclass import _DataclassEncoder
 from executorch.exir._serialize._flatbuffer import _flatc_compile
@@ -19,7 +21,9 @@ def convert_to_flatbuffer(mps_graph: MPSGraph) -> bytes:
     with tempfile.TemporaryDirectory() as d:
         schema_path = os.path.join(d, "schema.fbs")
         with open(schema_path, "wb") as schema_file:
-            schema_file.write(pkg_resources.resource_string(__name__, "schema.fbs"))
+            schema_file.write(
+                _resources.read_binary(serialization_package, "schema.fbs")
+            )
         json_path = os.path.join(d, "schema.json")
         with open(json_path, "wb") as json_file:
             json_file.write(mps_graph_json.encode("ascii"))

backends/arm/test/ops/test_sum.py

@@ -60,6 +60,7 @@ def test_sum_dim_intlist_tosa_INT(test_data: input_t1):
         aten_op,
         exir_op=[],
     )
+    pipeline.dump_artifact("export")
     pipeline.run()
 
 

backends/qualcomm/serialization/qc_schema_serialize.py

@@ -1,14 +1,16 @@
 # Copyright (c) Qualcomm Innovation Center, Inc.
+# Copyright 2025 Arm Limited and/or its affiliates.
 # All rights reserved
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import importlib.resources as _resources
 import json
 import os
 import tempfile
 
-import pkg_resources
+import executorch.backends.qualcomm.serialization as serialization_package
 from executorch.backends.qualcomm.serialization.qc_schema import QnnExecuTorchOptions
 from executorch.exir._serialize._dataclass import _DataclassEncoder, _json_to_dataclass
 from executorch.exir._serialize._flatbuffer import _flatc_compile, _flatc_decompile
@@ -19,7 +21,9 @@ def _convert_to_flatbuffer(obj, schema: str):
     with tempfile.TemporaryDirectory() as d:
         schema_path = os.path.join(d, f"{schema}.fbs")
         with open(schema_path, "wb") as schema_file:
-            schema_file.write(pkg_resources.resource_string(__name__, f"{schema}.fbs"))
+            schema_file.write(
+                _resources.read_binary(serialization_package, f"{schema}.fbs")
+            )
         json_path = os.path.join(d, f"{schema}.json")
         with open(json_path, "wb") as json_file:
             json_file.write(obj_json.encode("ascii"))
@@ -36,7 +40,9 @@ def _convert_to_object(flatbuffers: bytes, obj_type, schema: str):
         schema_path = os.path.join(d, f"{schema}.fbs")
         bin_path = os.path.join(d, f"{schema}.bin")
         with open(schema_path, "wb") as schema_file:
-            schema_file.write(pkg_resources.resource_string(__name__, f"{schema}.fbs"))
+            schema_file.write(
+                _resources.read_binary(serialization_package, f"{schema}.fbs")
+            )
         with open(bin_path, "wb") as bin_file:
             bin_file.write(flatbuffers)
 

backends/vulkan/serialization/vulkan_graph_serialize.py

@@ -1,28 +1,29 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
+# Copyright 2025 Arm Limited and/or its affiliates.
 #
 # pyre-strict
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
 import ctypes
+import importlib.resources as _resources
 import json
 import os
 import tempfile
-
 from dataclasses import dataclass
 from typing import ClassVar, List
 
-import pkg_resources
+import executorch.backends.vulkan.serialization as serialization_package
+
 import torch
 
 from executorch.backends.vulkan.serialization.vulkan_graph_schema import (
     VkBytes,
     VkGraph,
 )
 from executorch.exir._serialize._dataclass import _DataclassEncoder, _json_to_dataclass
-
 from executorch.exir._serialize._flatbuffer import _flatc_compile, _flatc_decompile
 
 
@@ -32,7 +33,9 @@ def convert_to_flatbuffer(vk_graph: VkGraph) -> bytes:
     with tempfile.TemporaryDirectory() as d:
         schema_path = os.path.join(d, "schema.fbs")
         with open(schema_path, "wb") as schema_file:
-            schema_file.write(pkg_resources.resource_string(__name__, "schema.fbs"))
+            schema_file.write(
+                _resources.read_binary(serialization_package, "schema.fbs")
+            )
         json_path = os.path.join(d, "schema.json")
         with open(json_path, "wb") as json_file:
             json_file.write(vk_graph_json.encode("ascii"))
@@ -48,7 +51,9 @@ def flatbuffer_to_vk_graph(flatbuffers: bytes) -> VkGraph:
     with tempfile.TemporaryDirectory() as d:
         schema_path = os.path.join(d, "schema.fbs")
         with open(schema_path, "wb") as schema_file:
-            schema_file.write(pkg_resources.resource_string(__name__, "schema.fbs"))
+            schema_file.write(
+                _resources.read_binary(serialization_package, "schema.fbs")
+            )
 
         bin_path = os.path.join(d, "schema.bin")
         with open(bin_path, "wb") as bin_file:

backends/xnnpack/serialization/xnnpack_graph_serialize.py

@@ -1,22 +1,21 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
+# Copyright 2025 Arm Limited and/or its affiliates.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import importlib.resources as _resources
 import json
-
 import logging
 import os
 import tempfile
-
 from dataclasses import dataclass, fields, is_dataclass
 from typing import ClassVar, Literal, Optional
 
-import pkg_resources
+import executorch.backends.xnnpack.serialization as serialization_package
 from executorch.backends.xnnpack.serialization.xnnpack_graph_schema import XNNGraph
 from executorch.exir._serialize._dataclass import _DataclassEncoder
-
 from executorch.exir._serialize._flatbuffer import _flatc_compile
 
 logger = logging.getLogger(__name__)
@@ -317,7 +316,9 @@ def convert_to_flatbuffer(xnnpack_graph: XNNGraph) -> bytes:
     with tempfile.TemporaryDirectory() as d:
         schema_path = os.path.join(d, "schema.fbs")
         with open(schema_path, "wb") as schema_file:
-            schema_file.write(pkg_resources.resource_string(__name__, "schema.fbs"))
+            schema_file.write(
+                _resources.read_binary(serialization_package, "schema.fbs")
+            )
         json_path = os.path.join(d, "schema.json")
         with open(json_path, "wb") as json_file:
             json_file.write(xnnpack_graph_json.encode("ascii"))

devtools/bundled_program/serialize/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
+# Copyright 2025 Arm Limited and/or its affiliates.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
@@ -8,16 +9,15 @@
 
 # TODO(T138924864): Refactor to unify the serialization for bundled program and executorch program.
 
+import importlib.resources as _resources
 import json
 import os
 import tempfile
 
 import executorch.devtools.bundled_program.schema as bp_schema
 
-# @manual=fbsource//third-party/pypi/setuptools:setuptools
-import pkg_resources
+import executorch.devtools.bundled_program.serialize as serialization_package
 from executorch.devtools.bundled_program.core import BundledProgram
-
 from executorch.exir._serialize._dataclass import _DataclassEncoder, _json_to_dataclass
 from executorch.exir._serialize._flatbuffer import _flatc_compile, _flatc_decompile
 
@@ -30,7 +30,7 @@ def write_schema(d: str, schema_name: str) -> None:
     schema_path = os.path.join(d, "{}.fbs".format(schema_name))
     with open(schema_path, "wb") as schema_file:
         schema_file.write(
-            pkg_resources.resource_string(__name__, "{}.fbs".format(schema_name))
+            _resources.read_binary(serialization_package, f"{schema_name}.fbs")
         )
 
 

devtools/etdump/serialize.py

@@ -1,20 +1,20 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
+# Copyright 2025 Arm Limited and/or its affiliates.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
 # pyre-strict
 
+import importlib.resources as _resources
 import json
 import os
 import tempfile
 
-import pkg_resources
+import executorch.devtools.etdump as etdump_package
 from executorch.devtools.etdump.schema_flatcc import ETDumpFlatCC
-
 from executorch.exir._serialize._dataclass import _DataclassEncoder, _json_to_dataclass
-
 from executorch.exir._serialize._flatbuffer import _flatc_compile, _flatc_decompile
 
 # The prefix of schema files used for etdump
@@ -25,9 +25,7 @@
 def _write_schema(d: str, schema_name: str) -> None:
     schema_path = os.path.join(d, "{}.fbs".format(schema_name))
     with open(schema_path, "wb") as schema_file:
-        schema_file.write(
-            pkg_resources.resource_string(__name__, "{}.fbs".format(schema_name))
-        )
+        schema_file.write(_resources.read_binary(etdump_package, f"{schema_name}.fbs"))
 
 
 def _serialize_from_etdump_to_json(etdump: ETDumpFlatCC) -> str:

examples/arm/aot_arm_compiler.py

@@ -602,9 +602,8 @@ def get_args():
     )
     parser.add_argument(
         "--enable_qdq_fusion_pass",
-        action="store",
-        default=False,
-        help="Skip the QuantizedOpFusionPass fusion step (default: False)",
+        action="store_true",
+        help="Enable the QuantizedOpFusionPass fusion step",
     )
     args = parser.parse_args()
 
@@ -806,7 +805,7 @@ def transform_for_cortex_m_backend(edge, args):
     passes = [ReplaceQuantNodesPass()]
 
     # Conditionally add the QuantizedOpFusionPass
-    if args.enable_qdq_fusion_pass.lower() == "true":
+    if args.enable_qdq_fusion_pass:
         passes.append(QuantizedOpFusionPass())
 
     # Apply the passes

examples/arm/run.sh

@@ -70,7 +70,7 @@ function help() {
     echo "  --pte_placement=<elf|ADDR>             Ethos-U: Control if runtime has PTE baked into the elf or if its placed in memory outside of the elf, defaults to ${pte_placement}"
     echo "  --et_build_root=<FOLDER>               Executorch build output root folder to use, defaults to ${et_build_root}"
     echo "  --scratch-dir=<FOLDER>                 Path to your Ethos-U scrach dir if you not using default ${ethos_u_scratch_dir}"
-    echo "  --qdq_fusion_op=<true/false>           Enable/Disable QDQ fusion op"
+    echo "  --qdq_fusion_op                        Enable QDQ fusion op"
     exit 0
 }
 
@@ -98,7 +98,7 @@ for arg in "$@"; do
       --pte_placement=*) pte_placement="${arg#*=}";;
       --et_build_root=*) et_build_root="${arg#*=}";;
       --scratch-dir=*) ethos_u_scratch_dir="${arg#*=}" ; scratch_dir_set=true ;;
-      --qdq_fusion_op=*) qdq_fusion_op="${arg#*=}";;
+      --qdq_fusion_op) qdq_fusion_op=true;;
       *)
       ;;
     esac
@@ -200,6 +200,7 @@ devtools_flag=""
 bundleio_flag=""
 etrecord_flag=""
 et_dump_flag=""
+qdq_fusion_op_flag=""
 if [ "$build_with_etdump" = true ] ; then
     et_dump_flag="--etdump"
     etrecord_flag="--etrecord"
@@ -210,6 +211,10 @@ if [ "$bundleio" = true ] ; then
     bundleio_flag="--bundleio"
 fi
 
+if [ "$qdq_fusion_op" = true ] ; then
+    qdq_fusion_op_flag="--enable_qdq_fusion_pass"
+fi
+
 backends/arm/scripts/build_executorch.sh --et_build_root="${et_build_root}" --build_type=$build_type $devtools_flag $et_dump_flag --toolchain="${toolchain}"
 
 if [[ -z "$model_name" ]]; then
@@ -278,7 +283,7 @@ for i in "${!test_model[@]}"; do
         model_compiler_flags="${model_compiler_flags} --model_input=${model_input}"
     fi
 
-    ARM_AOT_CMD="python3 -m examples.arm.aot_arm_compiler --model_name=${model} --target=${target} ${model_compiler_flags} --intermediate=${output_folder} --output=${pte_file} --system_config=${system_config} --memory_mode=${memory_mode} $bundleio_flag ${etrecord_flag} --config=${config} --enable_qdq_fusion_pass=${qdq_fusion_op}"
+    ARM_AOT_CMD="python3 -m examples.arm.aot_arm_compiler --model_name=${model} --target=${target} ${model_compiler_flags} --intermediate=${output_folder} --output=${pte_file} --system_config=${system_config} --memory_mode=${memory_mode} $bundleio_flag ${etrecord_flag} --config=${config} $qdq_fusion_op_flag"
     echo "CALL ${ARM_AOT_CMD}" >&2
     ${ARM_AOT_CMD} 1>&2
 

examples/models/checkpoint.py

@@ -1,5 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
+# Copyright 2025 Arm Limited and/or its affiliates.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
@@ -16,7 +17,7 @@ def get_default_model_resource_dir(model_file_path: str) -> Path:
     """
     Get the default path to resouce files (which contain files such as the
     checkpoint and param files), either:
-    1. Uses the path from pkg_resources, only works with buck2
+    1. Uses the path from importlib.resources, only works with buck2
     2. Uses default path located in examples/models/llama/params
 
     Expected to be called from with a `model.py` file located in a
@@ -33,22 +34,22 @@ def get_default_model_resource_dir(model_file_path: str) -> Path:
     """
 
     try:
-        import pkg_resources
+        import importlib.resources as _resources
 
-        # 1st way: If we can import this path, we are running with buck2 and all resources can be accessed with pkg_resources.
+        # 1st way: If we can import this path, we are running with buck2 and all resources can be accessed with importlib.resources.
         # pyre-ignore
         from executorch.examples.models.llama import params  # noqa
 
         # Get the model name from the cwd, assuming that this module is called from a path such as
         # examples/models/<model_name>/model.py.
         model_name = Path(model_file_path).parent.name
-        resource_dir = Path(
-            pkg_resources.resource_filename(
-                f"executorch.examples.models.{model_name}", "params"
-            )
-        )
-    except:
-        # 2nd way.
+        model_dir = _resources.files(f"executorch.examples.models.{model_name}")
+        with _resources.as_file(model_dir) as model_path:
+            resource_dir = model_path / "params"
+        assert resource_dir.exists()
+
+    except Exception:
+        # 2nd way:
         resource_dir = Path(model_file_path).absolute().parent / "params"
 
     return resource_dir