|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# All rights reserved. |
| 3 | +# |
| 4 | +# This source code is licensed under the BSD-style license found in the |
| 5 | +# LICENSE file in the root directory of this source tree. |
| 6 | + |
| 7 | +import logging |
| 8 | +from copy import deepcopy |
| 9 | +from typing import Set |
| 10 | + |
| 11 | +import executorch.backends.vulkan.utils as utils |
| 12 | + |
| 13 | +import torch |
| 14 | + |
| 15 | +from executorch.backends.vulkan.op_registry import get_op_features, has_impl |
| 16 | + |
| 17 | +from executorch.backends.vulkan.serialization.vulkan_graph_schema import ( |
| 18 | + VkMemoryLayout, |
| 19 | + VkStorageType, |
| 20 | +) |
| 21 | + |
| 22 | +from executorch.exir.dialects._ops import ops as exir_ops |
| 23 | + |
| 24 | +from executorch.exir.pass_base import ExportPass, PassResult |
| 25 | + |
| 26 | +from torch._subclasses.fake_tensor import FakeTensor |
| 27 | + |
| 28 | +from torch.fx.passes.tools_common import NodeList |
| 29 | +from torch.fx.passes.utils.fuser_utils import topo_sort |
| 30 | + |
| 31 | +logger: logging.Logger = logging.getLogger("") |
| 32 | +logger.setLevel(logging.INFO) |
| 33 | + |
| 34 | + |
| 35 | +def set_memory_metadata( |
| 36 | + node: torch.fx.Node, storage: VkStorageType, layout: VkMemoryLayout |
| 37 | +) -> None: |
| 38 | + utils.set_node_spec_attr(node, "vk_storage_type", storage) |
| 39 | + utils.set_node_spec_attr(node, "vk_memory_layout", layout) |
| 40 | + |
| 41 | + |
| 42 | +class TagMemoryMetaPass(ExportPass): |
| 43 | + """ |
| 44 | + There are a variety of ways that tensors can be represented in Vulkan. The two main |
| 45 | + descriptors for how a tensor is laid out in memory is: |
| 46 | +
|
| 47 | + 1. Storage Type (buffer or texture) |
| 48 | + 2. Memory Layout (which dim is packed along a texel / has a stride of 1, etc.) |
| 49 | +
|
| 50 | + Due to the differences between buffers and textures, and the differences between |
| 51 | + different memory layouts, an implementation for an operator may only support a |
| 52 | + specific set of (storage type, memory layout) combinations. |
| 53 | +
|
| 54 | + Furthermore, if an operator implementation supports multiple (storage type, memory |
| 55 | + layout) combinations, there may be a "preferred" setting which results in optimal |
| 56 | + performance. |
| 57 | +
|
| 58 | + This pass is responsible for ensuring that all tensors participating in an operator |
| 59 | + call is has a valid/optimal (storage type, memory layout) setting, and insert |
| 60 | + transition operators to transfer input tensors to the correct memory settings when |
| 61 | + necessary. |
| 62 | + """ |
| 63 | + |
| 64 | + def __init__(self): |
| 65 | + super().__init__() |
| 66 | + self.default_storage: VkStorageType = VkStorageType.DEFAULT_STORAGE |
| 67 | + self.default_layout: VkMemoryLayout = VkMemoryLayout.DEFAULT_LAYOUT |
| 68 | + self.texture_limits = (16384, 16384, 2048) |
| 69 | + |
| 70 | + def propose_node_storage( |
| 71 | + self, |
| 72 | + node: torch.fx.Node, |
| 73 | + ) -> VkStorageType: |
| 74 | + """ |
| 75 | + Uses the operator registry to determine the storage type that should be used for |
| 76 | + a given node. The storage type is determined with the following priorities: |
| 77 | + 1. In some cases, a tensor involved in the computation may be too large to be |
| 78 | + represented as a texture. If this is the case, the node is "opinionated" and |
| 79 | + buffer representation must be used. |
| 80 | + 1. If the operator called by the node indicates a optimal storage type, or only |
| 81 | + supports a single storage type, use that storage type. If either is true, |
| 82 | + then the node is considered to be opinionated as well. If multiple storage |
| 83 | + and no preferred storage type is indicated, then the node is not opinionated; |
| 84 | + go to the next step. |
| 85 | + 2. If the node's arguments already have memory metadata annotations, then |
| 86 | + preserve the settings of the first argument. Otherwise, proceed to the next |
| 87 | + step. |
| 88 | + 3. Recursively search the node's uses to see if any subsequent uses are |
| 89 | + opinionated; inherit the settings of the first opinionated node. If no |
| 90 | + opinionated user can be found, then proceed to the last step. |
| 91 | + 4. Use the default storage type setting. |
| 92 | + """ |
| 93 | + # The node may have an input/output tensor that is too big to be stored in a |
| 94 | + # texture. In this case, buffer storage must be used. Note that the partitioner |
| 95 | + # has already checked for the fact that buffer storage is supported by the |
| 96 | + # operator. |
| 97 | + if len(utils.possible_node_memory_layouts(node, self.texture_limits)) == 0: |
| 98 | + return VkStorageType.BUFFER |
| 99 | + |
| 100 | + valid_storage_types: Set[VkStorageType] = utils.all_storage_types |
| 101 | + |
| 102 | + # pyre-ignore |
| 103 | + if has_impl(node.target): |
| 104 | + # pyre-ignore |
| 105 | + features = get_op_features(node.target) |
| 106 | + valid_storage_types = features.supported_storage_types() |
| 107 | + storage = features.propose_storage_type() |
| 108 | + if storage is not None: |
| 109 | + return storage |
| 110 | + |
| 111 | + for arg in node.args: |
| 112 | + if isinstance(arg, torch.fx.Node) and isinstance( |
| 113 | + arg.meta["val"], FakeTensor |
| 114 | + ): |
| 115 | + storage = utils.get_node_storage_type(arg) |
| 116 | + if storage is not None and storage in valid_storage_types: |
| 117 | + return storage |
| 118 | + |
| 119 | + # If no storage type has been resolved yet, assume the optimal storage type of |
| 120 | + # the first opinionated user. This search is recursive. |
| 121 | + for user in node.users: |
| 122 | + optimal_storage = self.propose_node_storage(user) |
| 123 | + if optimal_storage is not None: |
| 124 | + return optimal_storage |
| 125 | + |
| 126 | + if self.default_storage in valid_storage_types: |
| 127 | + return self.default_storage |
| 128 | + else: |
| 129 | + return next(iter(valid_storage_types)) |
| 130 | + |
| 131 | + def propose_node_layout( |
| 132 | + self, |
| 133 | + node: torch.fx.Node, |
| 134 | + storage: VkStorageType, |
| 135 | + ) -> VkMemoryLayout: |
| 136 | + """ |
| 137 | + Performs the same steps as propose_node_storage, but detects the memory layout |
| 138 | + that should be used for the specific storage type. The same prioritization logic |
| 139 | + is applied. |
| 140 | + """ |
| 141 | + valid_layouts: Set[VkMemoryLayout] = utils.all_memory_layouts |
| 142 | + # pyre-ignore |
| 143 | + if has_impl(node.target): |
| 144 | + # pyre-ignore |
| 145 | + features = get_op_features(node.target) |
| 146 | + valid_layouts = features.supported_memory_layouts(storage) |
| 147 | + layout = features.propose_memory_layout(storage) |
| 148 | + if layout is not None: |
| 149 | + return layout |
| 150 | + |
| 151 | + for arg in node.args: |
| 152 | + if isinstance(arg, torch.fx.Node) and isinstance( |
| 153 | + arg.meta["val"], FakeTensor |
| 154 | + ): |
| 155 | + layout = utils.get_node_memory_layout(arg) |
| 156 | + if layout is not None and layout in valid_layouts: |
| 157 | + return layout |
| 158 | + |
| 159 | + # If no storage type has been resolved yet, assume the optimal storage type of |
| 160 | + # the first opinionated user. This search is recursive. |
| 161 | + for user in node.users: |
| 162 | + optimal_storage = self.propose_node_layout(user, storage) |
| 163 | + if optimal_storage is not None: |
| 164 | + return optimal_storage |
| 165 | + |
| 166 | + # As a last resort, return the default storage type that should be used. |
| 167 | + if self.default_layout in valid_layouts: |
| 168 | + return self.default_layout |
| 169 | + else: |
| 170 | + return next(iter(valid_layouts)) |
| 171 | + |
| 172 | + def call(self, graph_module: torch.fx.GraphModule) -> PassResult: |
| 173 | + sorted_nodes: NodeList = topo_sort(list(graph_module.graph.nodes)) |
| 174 | + |
| 175 | + for node in sorted_nodes: |
| 176 | + if not isinstance(node.meta["val"], FakeTensor): |
| 177 | + continue |
| 178 | + |
| 179 | + if node.target == exir_ops.edge.et_vk.prepack.default: |
| 180 | + continue |
| 181 | + |
| 182 | + storage = self.propose_node_storage(node) |
| 183 | + layout = self.propose_node_layout(node, storage) |
| 184 | + |
| 185 | + set_memory_metadata(node, storage, layout) |
| 186 | + |
| 187 | + inserting_transitions_for_node = False |
| 188 | + for i, arg in enumerate(node.args): |
| 189 | + if not isinstance(arg, torch.fx.Node): |
| 190 | + continue |
| 191 | + if not isinstance(arg.meta["val"], FakeTensor): |
| 192 | + continue |
| 193 | + |
| 194 | + arg_storage = utils.get_node_storage_type(arg) |
| 195 | + arg_layout = utils.get_node_memory_layout(arg) |
| 196 | + |
| 197 | + if arg_storage is None: |
| 198 | + utils.set_node_spec_attr(arg, "vk_storage_type", storage) |
| 199 | + arg_storage = storage |
| 200 | + if arg_layout is None: |
| 201 | + utils.set_node_spec_attr(arg, "vk_memory_layout", layout) |
| 202 | + arg_layout = layout |
| 203 | + |
| 204 | + if arg_storage == storage and arg_layout == layout: |
| 205 | + continue |
| 206 | + |
| 207 | + if not inserting_transitions_for_node: |
| 208 | + inserting_transitions_for_node = True |
| 209 | + logger.info( |
| 210 | + f"[Vulkan Delegate] Inserting transition(s) for {node.format_node()}:" |
| 211 | + ) |
| 212 | + |
| 213 | + logger.info( |
| 214 | + f" args {i} ({arg}): ({arg_storage}, {arg_layout}) -> ({storage}, {layout})" |
| 215 | + ) |
| 216 | + |
| 217 | + # Insert a clone node to copy the original tensor to a tensor with the |
| 218 | + # desired storage type and memory layout. |
| 219 | + with graph_module.graph.inserting_before(node): |
| 220 | + clone_node = graph_module.graph.create_node( |
| 221 | + "call_function", |
| 222 | + exir_ops.edge.aten.clone.default, |
| 223 | + (arg,), |
| 224 | + ) |
| 225 | + clone_node.meta["val"] = arg.meta["val"] |
| 226 | + clone_node.meta["spec"] = deepcopy(arg.meta["spec"]) |
| 227 | + clone_node.meta["spec"].const = False |
| 228 | + set_memory_metadata(clone_node, storage, layout) |
| 229 | + arg.replace_all_uses_with(clone_node, lambda x, y=node: x == y) |
| 230 | + |
| 231 | + return PassResult(graph_module, True) |
0 commit comments