Update

[ghstack-poisoned]

Author: GregoryComer

.ci/scripts/wheel/envvar_windows.sh

@@ -9,9 +9,6 @@ set -euxo pipefail
 
 # This script is run before building ExecuTorch binaries
 
-# Enable long paths (relevant for Windows, but can run on any system).
-git config --system core.longpaths true
-
 # Clone nested submodules for tokenizers - this is a workaround for recursive
 # submodule clone failing due to path length limitations on Windows. Eventually,
 # we should update the core job in test-infra to enable long paths before
@@ -25,4 +22,4 @@ popd
 # which does install them. Though we'd need to disable build isolation to be
 # able to see the installed torch package.
 
-"${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh"  --example
+"${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh" --example

.ci/scripts/wheel/pre_build_script.sh

@@ -0,0 +1,21 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# This script runs the provided command from a symlinked version of the active
+# working directory, in order to minimize path lengths and work around long
+# path limitations on Windows.
+
+set -x
+
+PWSH_SCRIPT = "& {
+    \$symlinkDir = Join-Path -Path \$env:GITHUB_WORKSPACE -ChildPath \"et-build\"
+    New-Item -ItemType SymbolicLink -Path \$symlinkDir -Target $PWD
+    Write-Host \$symlinkDir
+}"
+
+SYMLINK_DIR=`powershell -Command "$PWSH_SCRIPT"`
+cd $SYMLINK_DIR
+$1 ${@:2}

.ci/scripts/wheel/run_with_symlink_windows.sh

@@ -43,6 +43,7 @@ jobs:
       matrix:
         include:
           - repository: pytorch/executorch
+            env-script: .ci/scripts/wheel/run_with_symlink_windows.sh
             pre-script: .ci/scripts/wheel/pre_build_script.sh
             post-script: .ci/scripts/wheel/post_build_script.sh
             smoke-test-script: .ci/scripts/wheel/test_windows.py

.github/workflows/build-wheels-windows.yml

@@ -10,6 +10,7 @@
 # backends. Converts via TOSA as an intermediate form supported by AoT and
 # JIT compiler flows.
 #
+from enum import Enum
 from typing import List, Optional
 
 from executorch.backends.arm.tosa_specification import (  # type: ignore[import-not-found]
@@ -22,12 +23,16 @@
 
 
 class ArmCompileSpecBuilder:
+    class DebugMode(Enum):
+        JSON = 1
+
     def __init__(self):
         self.compile_spec: List[CompileSpec] = []
         self.compiler_flags = []
         self.output_format = None
         self.path_for_intermediates = None
         self.tosa_spec = None
+        self.tosa_debug_mode = None
 
     def vgf_compile_spec(
         self,
@@ -163,6 +168,13 @@ def dump_intermediate_artifacts_to(
         self.path_for_intermediates = output_path
         return self
 
+    def dump_debug_info(self, debug_mode: DebugMode) -> "ArmCompileSpecBuilder":
+        """
+        Dump debugging information into the intermediates path
+        """
+        self.tosa_debug_mode = debug_mode.name
+        return self
+
     def build(self) -> List[CompileSpec]:
         """
         Generate a list of compile spec objects from the builder
@@ -188,6 +200,16 @@ def build(self) -> List[CompileSpec]:
                 CompileSpec("debug_artifact_path", self.path_for_intermediates.encode())
             )
 
+        if self.tosa_debug_mode is not None:
+            if not self.path_for_intermediates:
+                raise ValueError(
+                    "dump_debug_info() must be used in conjunction with dump_intermediate_artifacts_to()"
+                )
+
+            self.compile_spec.append(
+                CompileSpec("dump_debug_info", self.tosa_debug_mode.encode())
+            )
+
         return self.compile_spec
 
 

backends/arm/arm_backend.py

@@ -5,10 +5,11 @@
 
 # pyre-unsafe
 
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 import torch
 
+from executorch.backends.arm.debug.schema import DebugHook
 from executorch.backends.arm.tosa_mapping import TosaArg
 from executorch.backends.arm.tosa_specification import TosaSpecification
 from torch.export import ExportedProgram
@@ -29,9 +30,38 @@ class NodeVisitor:
         TosaSpecification.create_from_string("TOSA-1.0+FP"),
     ]
 
-    def __init__(self, exported_program: ExportedProgram, tosa_spec: TosaSpecification):
+    def __init__(
+        self,
+        exported_program: ExportedProgram,
+        tosa_spec: TosaSpecification,
+        debug_hook: Optional[DebugHook] = None,
+    ):
         self._exported_program = exported_program
         self.tosa_spec = tosa_spec
+        self.debug_hook = debug_hook
+
+    def _serialize_operator(
+        self,
+        node: torch.fx.Node,
+        tosa_graph: Any,
+        tosa_op: Any,
+        inputs: List[str],
+        outputs: List[str],
+        attributes: Optional[Any] = None,
+    ) -> None:
+        tosa_graph.addOperator(
+            tosa_op,
+            inputs=inputs,
+            outputs=outputs,
+            attributes=attributes,
+        )
+
+        if self.debug_hook:
+            self.debug_hook.add(
+                node,
+                tosa_op=outputs[0],
+                tosa_op_id=tosa_op,
+            )
 
     def define_node(
         self,

backends/arm/operators/node_visitor.py

@@ -123,7 +123,9 @@ def define_node(
             )
 
             # MI lowering
-            tosa_graph.addOperator(
+            self._serialize_operator(
+                node,
+                tosa_graph,
                 ts.TosaOp.Op().ABS,
                 [inputs[0].name],
                 [output.name],

backends/arm/operators/op_abs.py

@@ -73,7 +73,9 @@ def define_node(
         input1, input2 = rescaled_inputs
 
         # Do the INT32 Add
-        tosa_graph.addOperator(
+        self._serialize_operator(
+            node,
+            tosa_graph,
             ts.TosaOp.Op().ADD,
             [input1.name, input2.name],
             [add_output.name],
@@ -127,7 +129,9 @@ def define_node(
             input1, input2 = inputs
 
             # FP lowering
-            tosa_graph.addOperator(
+            self._serialize_operator(
+                node,
+                tosa_graph,
                 ts.TosaOp.Op().ADD,
                 [input1.name, input2.name],
                 [output.name],

backends/arm/operators/op_add.py

@@ -61,6 +61,11 @@ def define_node(
 
         attr = ts.TosaSerializerAttribute()
         attr.ReduceMaxAttribute(axis=input.dim_order.index(dim), nan_mode=1)
-        tosa_graph.addOperator(
-            ts.TosaOp.Op().REDUCE_MAX, [input.name], [output.name], attr
+        self._serialize_operator(
+            node,
+            tosa_graph,
+            ts.TosaOp.Op().REDUCE_MAX,
+            [input.name],
+            [output.name],
+            attr,
         )

backends/arm/operators/op_amax.py

@@ -61,6 +61,11 @@ def define_node(
 
         attr = ts.TosaSerializerAttribute()
         attr.ReduceMinAttribute(axis=input.dim_order.index(dim), nan_mode=1)
-        tosa_graph.addOperator(
-            ts.TosaOp.Op().REDUCE_MIN, [input.name], [output.name], attr
+        self._serialize_operator(
+            node,
+            tosa_graph,
+            ts.TosaOp.Op().REDUCE_MIN,
+            [input.name],
+            [output.name],
+            attr,
         )