diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb2f9f0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +ingress/Torch-MLIR/examples/**/dumps/*.mlir diff --git a/ingress/Torch-MLIR/README.md b/ingress/Torch-MLIR/README.md new file mode 100644 index 0000000..2b35616 --- /dev/null +++ b/ingress/Torch-MLIR/README.md @@ -0,0 +1,49 @@ +Using scripts in this directory one can convert a Torch Model to a MLIR module. + +The conversion script is written in python and is basically a wrapper around [`torch-mlir` library](https://github.com/llvm/torch-mlir). One need to setup a python virtual environment with torch and torch-mlir libraries +(`./scripts/install-virtualenv.sh`) to use the script. + +In order to convert a model the script has to recieve: +1. An instance of `torch.nn.Model` with proper state (weights). +2. Sample input arguments to the model (e.g. empty tensor with proper shape and dtype). + +There are two options of how this info can be provided to the converter: + +### 1. Instantiate a model in your own script and use a function from the `py_src/export_lib` (recomended) + +In this scenario a user is responsible for instantiating a model with proper state in their +own python script. Then they should import a `generate_mlir` function from `py_src.export_lib` +and call it in order to get a MLIR module: + +```python +model : nn.Model = get_model() +sample_args = (get_sample_tensor(),) + +# PYTHONPATH=$(pwd)/py_src/ +from export_lib import generate_mlir + +mlir_module = generate_mlir(model, sample_args, dialect="linalg") +print(mlir_module) +``` + +### 2. Use `py_src/main.py` or `scripts/generate-mlir.sh` and pass Torch Model parameters via CLI + +In this scenario the `py_src/main.py` script is fully responsible for instantiating a torch model +and converting it to MLIR. User has to pass a proper python entrypoint for model's factory, +its parameters if needed (`--model-args & --model-kwargs`), and sample model arguments (either +as `--sample-shapes` or as an entrypoint to a function returning args and kwargs `--sample-fn`). + +``` +# note that 'my_module' has to be in $PYTHONPATH +python py_src/main.py --model-entrypoint my_module:my_factory \ + --module-state-path path/to/state.pth \ + --sample-shapes '1,2,324,float32' \ + --out-mlir res.mlir + +# note that 'my_module' has to be in $PYTHONPATH +./scripts/generate-mlir.sh --model-entrypoint torchvision.models:resnet18 \ + --sample-fn my_module:generate_resnet18_sample_args \ + --out-mlir res.mlir +``` + +Look into `examples/` folder for more info. diff --git a/ingress/Torch-MLIR/examples/dummy_mlp_cli/dummy_mlp.pth b/ingress/Torch-MLIR/examples/dummy_mlp_cli/dummy_mlp.pth new file mode 100644 index 0000000..28ccd45 Binary files /dev/null and b/ingress/Torch-MLIR/examples/dummy_mlp_cli/dummy_mlp.pth differ diff --git a/ingress/Torch-MLIR/examples/dummy_mlp_cli/dummy_mlp_factory.py b/ingress/Torch-MLIR/examples/dummy_mlp_cli/dummy_mlp_factory.py new file mode 100644 index 0000000..88d5d55 --- /dev/null +++ b/ingress/Torch-MLIR/examples/dummy_mlp_cli/dummy_mlp_factory.py @@ -0,0 +1,23 @@ +import torch +import torch.nn as nn + +import os + +class DummyMLP(nn.Module): + def __init__(self): + super().__init__() + self.net = nn.Sequential( + nn.Linear(10, 32), + nn.ReLU(), + nn.Linear(32, 2) + ) + + def forward(self, x): + return self.net(x) + +def make_dummy_mlp(): + return DummyMLP() + +if __name__ == "__main__": + script_dir = os.path.dirname(os.path.abspath(__file__)) + torch.save(make_dummy_mlp().state_dict(), os.path.join(script_dir, "dummy_mlp.pth")) diff --git a/ingress/Torch-MLIR/examples/dummy_mlp_cli/export_bash.sh b/ingress/Torch-MLIR/examples/dummy_mlp_cli/export_bash.sh new file mode 100755 index 0000000..bf247b2 --- /dev/null +++ b/ingress/Torch-MLIR/examples/dummy_mlp_cli/export_bash.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR=$SCRIPT_DIR/../../scripts/ + +PYTHONPATH=$PYTHONPATH:$SCRIPT_DIR $ROOT_DIR/generate-mlir.sh --model-entrypoint dummy_mlp_factory:make_dummy_mlp \ + --model-state-path $SCRIPT_DIR/dummy_mlp.pth \ + --sample-shapes "1,10,float32" \ + --dialect linalg \ + --out-mlir $SCRIPT_DIR/dummy_mlp_sh.mlir diff --git a/ingress/Torch-MLIR/examples/dummy_mlp_cli/export_py.sh b/ingress/Torch-MLIR/examples/dummy_mlp_cli/export_py.sh new file mode 100755 index 0000000..7d15911 --- /dev/null +++ b/ingress/Torch-MLIR/examples/dummy_mlp_cli/export_py.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR=$SCRIPT_DIR/../../py_src/ + +PYTHONPATH=$PYTHONPATH:$ROOT_DIR:$SCRIPT_DIR python $ROOT_DIR/main.py --model-entrypoint dummy_mlp_factory:make_dummy_mlp \ + --model-state-path $SCRIPT_DIR/dummy_mlp.pth \ + --sample-shapes "1,10,float32" \ + --dialect linalg \ + --out-mlir $SCRIPT_DIR/dummy_mlp.mlir diff --git a/ingress/Torch-MLIR/examples/dummy_mlp_python/export.py b/ingress/Torch-MLIR/examples/dummy_mlp_python/export.py new file mode 100644 index 0000000..523946a --- /dev/null +++ b/ingress/Torch-MLIR/examples/dummy_mlp_python/export.py @@ -0,0 +1,25 @@ +import torch +import torch.nn as nn + +from export_lib.export import generate_mlir + +class DummyMLP(nn.Module): + def __init__(self): + super().__init__() + self.net = nn.Sequential( + nn.Linear(10, 32), + nn.ReLU(), + nn.Linear(32, 2) + ) + + def forward(self, x): + return self.net(x) + +def main(): + model = DummyMLP() + dummy_input = torch.randn(1, 10) + mlir_mod = generate_mlir(model, (dummy_input,), {}) + print(mlir_mod) + +if __name__ == "__main__": + main() diff --git a/ingress/Torch-MLIR/examples/dummy_mlp_python/run.sh b/ingress/Torch-MLIR/examples/dummy_mlp_python/run.sh new file mode 100755 index 0000000..62ff78e --- /dev/null +++ b/ingress/Torch-MLIR/examples/dummy_mlp_python/run.sh @@ -0,0 +1,4 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR=$SCRIPT_DIR/../../py_src/ + +PYTHONPATH=$ROOT_DIR python $SCRIPT_DIR/export.py diff --git a/ingress/Torch-MLIR/examples/resnet_18_cli/export_bash.sh b/ingress/Torch-MLIR/examples/resnet_18_cli/export_bash.sh new file mode 100755 index 0000000..b5e0f5f --- /dev/null +++ b/ingress/Torch-MLIR/examples/resnet_18_cli/export_bash.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR=$SCRIPT_DIR/../../scripts/ + +$ROOT_DIR/generate-mlir.sh --model-entrypoint torchvision.models:resnet18 \ + --sample-shapes "1,3,224,224,float32" \ + --dialect linalg \ + --out-mlir $SCRIPT_DIR/resnet_18_sh.mlir diff --git a/ingress/Torch-MLIR/examples/resnet_18_cli/export_py.sh b/ingress/Torch-MLIR/examples/resnet_18_cli/export_py.sh new file mode 100755 index 0000000..22f3931 --- /dev/null +++ b/ingress/Torch-MLIR/examples/resnet_18_cli/export_py.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR=$SCRIPT_DIR/../../py_src/ + +python $ROOT_DIR/main.py --model-entrypoint torchvision.models:resnet18 \ + --sample-shapes "1,3,224,224,float32" \ + --dialect linalg \ + --out-mlir $SCRIPT_DIR/resnet_18.mlir diff --git a/ingress/Torch-MLIR/generate-mlir.py b/ingress/Torch-MLIR/generate-mlir.py deleted file mode 100644 index 888e6dd..0000000 --- a/ingress/Torch-MLIR/generate-mlir.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import torch -import torch.nn as nn -from torch_mlir import fx -from torch_mlir.fx import OutputType - -# Parse arguments for selecting which model to load and which MLIR dialect to generate -def parse_args(): - parser = argparse.ArgumentParser(description="Generate MLIR for Torch-MLIR models.") - parser.add_argument( - "--model", - type=str, - required=True, - help="Path to the Torch model file.", - ) - parser.add_argument( - "--dialect", - type=str, - choices=["torch", "linalg", "stablehlo", "tosa"], - default="linalg", - help="MLIR dialect to generate.", - ) - return parser.parse_args() - -# Functin to load the Torch model -def load_torch_model(model_path): - - if not os.path.exists(model_path): - raise FileNotFoundError(f"Model file {model_path} does not exist.") - - model = torch.load(model_path) - return model - -# Function to generate MLIR from the Torch model -# See: https://github.com/MrSidims/PytorchExplorer/blob/main/backend/server.py#L237 -def generate_mlir(model, dialect): - - # Convert the Torch model to MLIR - output_type = None - if dialect == "torch": - output_type = OutputType.TORCH - elif dialect == "linalg": - output_type = OutputType.LINALG - elif dialect == "stablehlo": - output_type = OutputType.STABLEHLO - elif dialect == "tosa": - output_type = OutputType.TOSA - else: - raise ValueError(f"Unsupported dialect: {dialect}") - - module = fx.export_and_import(model, "", output_type=output_type) - return module - -# Main function to execute the script -def main(): - args = parse_args() - - # Load the Torch model - model = load_torch_model(args.model) - - # Generate MLIR from the model - mlir_module = generate_mlir(model, args.dialect) - - # Print or save the MLIR module - print(mlir_module) - -# Entry point for the script -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/ingress/Torch-MLIR/generate-mlir.sh b/ingress/Torch-MLIR/generate-mlir.sh deleted file mode 100755 index 0a079c6..0000000 --- a/ingress/Torch-MLIR/generate-mlir.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# Command line argument for model to load and MLIR dialect to generate -while getopts "m:d:" opt; do - case $opt in - m) - MODEL=$OPTARG - ;; - d) - DIALECT=$OPTARG - ;; - *) - echo "Usage: $0 [-m model] [-d dialect]" - exit 1 - ;; - esac -done -if [ -z "$MODEL" ]; then - echo "Model not specified. Please provide a model using -m option." - exit 1 -fi -if [ -z "$DIALECT" ]; then - DIALECT="linalg" -fi - -# Enable local virtualenv created by install-virtualenv.sh -if [ ! -d "torch-mlir-venv" ]; then - echo "Virtual environment not found. Please run install-virtualenv.sh first." - exit 1 -fi -source torch-mlir-venv/bin/activate - -# Find script directory -SCRIPT_DIR=$(dirname "$(readlink -f "$0")") - -# Use the Python script to generate MLIR -echo "Generating MLIR for model '$MODEL' with dialect '$DIALECT'..." -python $SCRIPT_DIR/generate-mlir.py --model "$MODEL" --dialect "$DIALECT" -if [ $? -ne 0 ]; then - echo "Failed to generate MLIR for model '$MODEL'." - exit 1 -fi diff --git a/ingress/Torch-MLIR/py_src/__init__.py b/ingress/Torch-MLIR/py_src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ingress/Torch-MLIR/py_src/export_lib/__init__.py b/ingress/Torch-MLIR/py_src/export_lib/__init__.py new file mode 100644 index 0000000..fce4293 --- /dev/null +++ b/ingress/Torch-MLIR/py_src/export_lib/__init__.py @@ -0,0 +1,8 @@ +# Examples package initialization +""" +Example scripts for Torch-MLIR usage. +""" + +from .export import load_torch_model, generate_sample_args, generate_mlir + +__all__ = ['load_torch_model', 'generate_sample_args', 'generate_mlir'] diff --git a/ingress/Torch-MLIR/py_src/export_lib/export.py b/ingress/Torch-MLIR/py_src/export_lib/export.py new file mode 100644 index 0000000..928ffa6 --- /dev/null +++ b/ingress/Torch-MLIR/py_src/export_lib/export.py @@ -0,0 +1,58 @@ +import os +import torch +from torch_mlir import fx +from torch_mlir.fx import OutputType + +from .utils import parse_shape_str, load_callable_symbol, generate_fake_tensor + +def load_torch_model(entrypoint_path, model_state_path=None, *args, **kwargs): + entrypoint = load_callable_symbol(entrypoint_path) + model = entrypoint(*args, **kwargs) + if model_state_path is not None: + state_dict = load_model_state(model_state_path) + model.load_state_dict(state_dict) + return model + +# Function to load the Torch model +def load_model_state(model_path): + if not os.path.exists(model_path): + raise FileNotFoundError(f"Model file {model_path} does not exist.") + + model = torch.load(model_path) + return model + + +def generate_sample_args(shape_str, sample_fn_path) -> tuple[tuple, dict]: + """ + Generate sample arguments for the model's 'forward' method. + (Required by torch_mlir.fx.export_and_import) + """ + if sample_fn_path is None: + shape, dtype = parse_shape_str(shape_str) + return (generate_fake_tensor(shape, dtype),), {} + + return load_callable_symbol(sample_fn_path)() + + +def generate_mlir(model, sample_args, sample_kwargs=None, dialect="linalg"): + # Convert the Torch model to MLIR + output_type = None + if dialect == "torch": + output_type = OutputType.TORCH + elif dialect == "linalg": + output_type = OutputType.LINALG_ON_TENSORS + elif dialect == "stablehlo": + output_type = OutputType.STABLEHLO + elif dialect == "tosa": + output_type = OutputType.TOSA + else: + raise ValueError(f"Unsupported dialect: {dialect}") + + if sample_kwargs is None: + sample_kwargs = {} + + model.eval() + module = fx.export_and_import( + model, *sample_args, output_type=output_type, **sample_kwargs + ) + return module diff --git a/ingress/Torch-MLIR/py_src/export_lib/utils.py b/ingress/Torch-MLIR/py_src/export_lib/utils.py new file mode 100644 index 0000000..0534e60 --- /dev/null +++ b/ingress/Torch-MLIR/py_src/export_lib/utils.py @@ -0,0 +1,78 @@ +import importlib +import importlib.util +import sys +import os + +import torch +from torch._subclasses.fake_tensor import FakeTensorMode + +from typing import Callable + + +def load_callable_symbol(entry: str) -> Callable: + """ + Load a callable python symbol from a module or a file. + + Parameters + ---------- + entry : str + A string specifying the module or file and the attribute path, + in the format 'module_or_path:attr', e.g. + 'torchvision.models:resnet18' or '/path/to/model.py:build_model'. + + Returns + ------- + Callable + """ + if ":" not in entry: + raise ValueError("Entry must be like 'module_or_path:attr'") + + left, right = entry.split(":", 1) + attr_path = right.split(".") + + if os.path.exists(left) and left.endswith(".py"): + mod_dir = os.path.abspath(os.path.dirname(left)) + mod_name = os.path.splitext(os.path.basename(left))[0] + sys_path_was = list(sys.path) + try: + if mod_dir not in sys.path: + sys.path.insert(0, mod_dir) + spec = importlib.util.spec_from_file_location(mod_name, left) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load spec from {left}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + sys.path = sys_path_was + else: + module = importlib.import_module(left) + + obj = module + for name in attr_path: + obj = getattr(obj, name) + + return obj + + +def parse_shape_str(shape: str) -> tuple[tuple[int], torch.dtype]: + """ + Parse a shape string into a shape tuple and a torch dtype. + + Parameters + ---------- + shape : str + A string representing the shape and dtype, e.g. '1,3,224,224,float32'. + """ + *shapes, dtype = shape.split(",") + tdtype = getattr(torch, dtype) + if tdtype is None: + raise ValueError(f"Unsupported dtype: {dtype}") + if "?" in shapes: + raise ValueError(f"Dynamic shapes are not supported yet: {shape}") + return (tuple(map(int, shapes)), tdtype) + + +def generate_fake_tensor(shape: tuple[int], dtype: torch.dtype) -> torch.Tensor: + """Generate a fake tensor (has no actual buffer) with the given shape and dtype.""" + with FakeTensorMode(): + return torch.empty(shape, dtype=dtype) diff --git a/ingress/Torch-MLIR/py_src/main.py b/ingress/Torch-MLIR/py_src/main.py new file mode 100644 index 0000000..feddf4f --- /dev/null +++ b/ingress/Torch-MLIR/py_src/main.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +import argparse +from export_lib.export import load_torch_model, generate_sample_args, generate_mlir + +# Parse arguments for selecting which model to load and which MLIR dialect to generate +def parse_args(): + parser = argparse.ArgumentParser(description="Generate MLIR for Torch-MLIR models.") + parser.add_argument( + "--model-entrypoint", + type=str, + required=True, + help="Path to the model entrypoint, e.g. 'torchvision.models:resnet18' or '/path/to/model.py:build_model'.", + ) + parser.add_argument( + "--model-state-path", + type=str, + required=False, + help="Path to a state file of the Torch model (usually has .pt or .pth extension).", + ) + parser.add_argument( + "--model-args", + type=str, + required=False, + default="[]", + help="" + "Positional arguments to pass to the model's entrypoint " + "(note that this argument will be passed to an 'eval'," + " so the string should contain a valid python code).", + ) + parser.add_argument( + "--model-kwargs", + type=str, + required=False, + default="{}", + help="" + "Keyword arguments to pass to the model's entrypoint " + "(note that this argument will be passed to an 'eval'," + " so the string should contain a valid python code).", + ) + parser.add_argument( + "--sample-shapes", + type=str, + required=False, + help="Tensor shapes/dtype that the 'forward' method of the model will be called with," + " e.g. '1,3,224,224,float32'. Must be specified if '--sample-fn' is not given.", + ) + parser.add_argument( + "--sample-fn", + type=str, + required=False, + help="Path to a function that generates sample arguments for the model's 'forward' method." + " The function should return a tuple of (args, kwargs). If this is given, '--sample-shapes' is ignored.", + ) + parser.add_argument( + "--dialect", + type=str, + choices=["torch", "linalg", "stablehlo", "tosa"], + default="linalg", + help="MLIR dialect to generate.", + ) + parser.add_argument( + "--out-mlir", + type=str, + required=False, + help="Path to save the generated MLIR module.", + ) + return parser.parse_args() + + +# Main function to execute the script +def main(): + args = parse_args() + + # Load the Torch model + model = load_torch_model( + args.model_entrypoint, + args.model_state_path, + *eval(args.model_args), + **eval(args.model_kwargs) + ) + sample_args, sample_kwargs = generate_sample_args( + args.sample_shapes, args.sample_fn + ) + # Generate MLIR from the model + mlir_module = generate_mlir(model, sample_args, sample_kwargs, args.dialect) + + # Print or save the MLIR module + if args.out_mlir: + with open(args.out_mlir, "w") as f: + f.write(str(mlir_module)) + else: + print(mlir_module) + + +# Entry point for the script +if __name__ == "__main__": + main() diff --git a/ingress/Torch-MLIR/scripts/generate-mlir.sh b/ingress/Torch-MLIR/scripts/generate-mlir.sh new file mode 100755 index 0000000..080cc05 --- /dev/null +++ b/ingress/Torch-MLIR/scripts/generate-mlir.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Find script directory +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") +PY_SCRIPT_DIR=$SCRIPT_DIR/../py_src/ +VENV_DIR="$SCRIPT_DIR/../torch-mlir-venv" + +# Enable local virtualenv created by install-virtualenv.sh +if [ ! -d "$VENV_DIR" ]; then + echo "Virtual environment not found. Please run install-virtualenv.sh first." + exit 1 +fi +source $VENV_DIR/bin/activate + +# Use the Python script to generate MLIR +PYTHONPATH=$PYTHONPATH:$PY_SCRIPT_DIR python "$PY_SCRIPT_DIR/main.py" "$@" diff --git a/ingress/Torch-MLIR/install-virtualenv.sh b/ingress/Torch-MLIR/scripts/install-virtualenv.sh similarity index 94% rename from ingress/Torch-MLIR/install-virtualenv.sh rename to ingress/Torch-MLIR/scripts/install-virtualenv.sh index f48019c..43b9124 100755 --- a/ingress/Torch-MLIR/install-virtualenv.sh +++ b/ingress/Torch-MLIR/scripts/install-virtualenv.sh @@ -9,6 +9,7 @@ else DEVICE_TYPE=$(lspci | grep VGA) fi +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") # Install torch-mlir inside a virtual environment echo "First ensure uv is installed" @@ -16,7 +17,7 @@ echo "First ensure uv is installed" python -m pip install uv --upgrade echo "Preparing the virtual environment" -python -m uv venv torch-mlir-venv --python 3.12 +python -m uv venv $SCRIPT_DIR/../torch-mlir-venv --python 3.12 #echo "Preparing the virtual environment" #python3 -m venv torch-mlir-venv