node_openvino.py

import torch
import openvino as ov
from typing_extensions import override
from typing import Optional
import openvino.frontend.pytorch.torchdynamo.execute as ov_ex
from comfy_api.latest import ComfyExtension, io
from comfy_api.torch_helpers import set_torch_compile_wrapper

TORCH_COMPILE_KWARGS_VAE = "torch_compile_kwargs_vae"


_dynamo_workarounds_applied = False


def _apply_dynamo_workarounds():
    """Work around two PyTorch 2.10 issues with the OpenVINO torch.compile backend:

    1. TorchDynamo bug in nn_module.py wrap_values() references undefined free variable
       'named_children' instead of 'result'. Triggered when tracing self.parameters()
       in ResBlock.forward(). Fix: monkey-patch ResBlock.forward to skip self.parameters()
       when not checkpointing (params are unused in that branch anyway).

    2. OpenVINO backend falls back to inductor (compile_fx) on any exception, but inductor's
       C++ codegen requires omp.h which may be missing on Windows MSVC setups.
       Fix: replace the inductor fallback with eager execution.
    """
    global _dynamo_workarounds_applied
    if _dynamo_workarounds_applied:
        return
    _dynamo_workarounds_applied = True

    # Fix 1: Patch ResBlock.forward to avoid self.parameters() call
    try:
        from comfy.ldm.modules.diffusionmodules.openaimodel import ResBlock
        from comfy.ldm.modules.diffusionmodules.util import checkpoint

        def _patched_forward(self, x, emb):
            if self.use_checkpoint:
                return checkpoint(
                    self._forward, (x, emb), self.parameters(), self.use_checkpoint
                )
            return self._forward(x, emb)

        ResBlock.forward = _patched_forward
    except (ImportError, AttributeError):
        pass

    # Fix 2: In PyTorch 2.10, make_fx(tracing_mode="fake") crashes with
    # "Cannot call numel() on tensor with symbolic sizes/strides" because
    # PyTorch 2.10's fake tracing creates symbolic-shaped tensors that the
    # C++ linear kernel can't handle. Fix: replace make_fx in the openvino
    # backend's namespace so it uses tracing_mode="symbolic" instead.
    # Also set allow_non_fake_inputs_override to prevent FakeTensor assertions.
    try:
        import openvino.frontend.pytorch.torchdynamo.backend as ov_backend
        from torch._subclasses.fake_tensor import fake_tensor_tls

        _original_make_fx = ov_backend.make_fx

        def _patched_make_fx(*args, **kwargs):
            if kwargs.get("tracing_mode") == "fake":
                kwargs["tracing_mode"] = "symbolic"
            return _original_make_fx(*args, **kwargs)

        ov_backend.make_fx = _patched_make_fx

        _original_fx_openvino = ov_backend.fx_openvino

        def _patched_fx_openvino(subgraph, example_inputs, options=None):
            old_override = fake_tensor_tls.allow_non_fake_inputs_override
            fake_tensor_tls.allow_non_fake_inputs_override = True
            try:
                result = _original_fx_openvino(subgraph, example_inputs, options)
                print(f"[OV-DEBUG] fx_openvino SUCCEEDED for subgraph")
                return result
            except Exception as e:
                print(f"[OV-DEBUG] fx_openvino FAILED: {type(e).__name__}: {e}")
                raise
            finally:
                fake_tensor_tls.allow_non_fake_inputs_override = old_override

        ov_backend.fx_openvino = _patched_fx_openvino

        # Fix 3: When fx_openvino fails, the openvino backend falls back to
        # compile_fx (inductor), which requires omp.h missing on Windows MSVC.
        # Replace compile_fx with eager execution so only the failing subgraphs
        # fall back, while successful ones still run on OpenVINO GPU.
        def _eager_fallback(subgraph, example_inputs):
            import sys, traceback as tb
            exc_info = sys.exc_info()
            if exc_info[1]:
                print(f"[OV-DEBUG] compile_fx fallback → eager. Exception: {exc_info[0].__name__}: {exc_info[1]}")
                tb.print_exc()
            else:
                print(f"[OV-DEBUG] compile_fx fallback → eager (no active exception)")
            return subgraph.forward

        ov_backend.compile_fx = _eager_fallback
    except (ImportError, AttributeError):
        pass

    # Fix 4: PyTorch 2.10 bug in symbolic_shapes.py — produce_guards_verbose()
    # crashes with IndexError when symbol_to_source[symbol] is an empty list
    # (triggered by conv shape guards). This happens AFTER successful backend
    # compilation but BEFORE guards are installed, so it would discard the
    # compiled OpenVINO code. Patch to return empty guards on IndexError,
    # which means "always use this compiled version" (fine for fixed-shape inference).
    try:
        from torch.fx.experimental.symbolic_shapes import ShapeEnv, _ShapeGuardsHelper

        _original_produce_guards_verbose = ShapeEnv.produce_guards_verbose

        def _patched_produce_guards_verbose(self, *args, **kwargs):
            try:
                return _original_produce_guards_verbose(self, *args, **kwargs)
            except IndexError:
                # symbol_to_source[symbol] is empty — return empty guards
                # so that compilation can proceed.
                langs = kwargs.get("langs", ("python", "verbose_python"))
                return [_ShapeGuardsHelper(exprs=[]) for _ in langs]

        ShapeEnv.produce_guards_verbose = _patched_produce_guards_verbose
    except (ImportError, AttributeError):
        pass


class VAECompileWrapper:
    """
    VAE compiler wrapper that mirrors set_torch_compile_wrapper
    Compiles high-level encode/decode entry points so pipeline semantics stay intact
    """
    def __init__(self, vae):
        self.vae = vae
        self.first_stage = vae.first_stage_model
        self.compiled_modules = {}
        self.compile_kwargs = {}
        self.is_active = False

        # Store original forward methods
        self.original_encode = None
        self.original_decode = None

    def compile(self, backend: str, options: Optional[dict] = None,
               keys: Optional[list[str]] = None):
        """Compile specified VAE modules"""

        if self.is_active:
            self.remove()

        if keys is None:
            if hasattr(self.first_stage, "taesd_encoder"):
                keys = ["taesd_encoder", "taesd_decoder"]
            else:
                keys = ["encoder", "decoder"]

        compile_kwargs = {
            "backend": backend,
            "options": options,
        }
        compile_kwargs = {k: v for k, v in compile_kwargs.items() if v is not None}

        compiled_any = False

        for key in keys:
            if "encoder" in key and hasattr(self.first_stage, "encode"):
                try:
                    if self.original_encode is None:
                        self.original_encode = self.first_stage.encode
                    compiled_encode = torch.compile(self.original_encode, **compile_kwargs)
                    self.first_stage.encode = compiled_encode
                    self.compiled_modules["encode"] = compiled_encode
                    compiled_any = True
                    print("✅ Successfully compiled VAE.encode")
                except RuntimeError as e:
                    print(f"❌ Failed to compile VAE.encode: {e}")
                    if "encode" not in self.compiled_modules:
                        self.original_encode = None

            if "decoder" in key and hasattr(self.first_stage, "decode"):
                try:
                    if self.original_decode is None:
                        self.original_decode = self.first_stage.decode
                    compiled_decode = torch.compile(self.original_decode, **compile_kwargs)
                    self.first_stage.decode = compiled_decode
                    self.compiled_modules["decode"] = compiled_decode
                    compiled_any = True
                    print("✅ Successfully compiled VAE.decode")
                except RuntimeError as e:
                    print(f"❌ Failed to compile VAE.decode: {e}")
                    if "decode" not in self.compiled_modules:
                        self.original_decode = None

        if compiled_any:
            self.compile_kwargs = compile_kwargs
            self.is_active = True

            # Store into vae_options
            if not hasattr(self.vae, 'vae_options'):
                self.vae.vae_options = {}
            self.vae.vae_options[TORCH_COMPILE_KWARGS_VAE] = compile_kwargs

    def remove(self):
        """Remove compilation wrapper"""
        if not self.is_active:
            return

        if self.original_encode is not None and hasattr(self.first_stage, "encode"):
            self.first_stage.encode = self.original_encode
            self.original_encode = None
        if self.original_decode is not None and hasattr(self.first_stage, "decode"):
            self.first_stage.decode = self.original_decode
            self.original_decode = None

        self.compiled_modules.clear()
        self.compile_kwargs.clear()
        self.is_active = False

        if hasattr(self.vae, 'vae_options') and TORCH_COMPILE_KWARGS_VAE in self.vae.vae_options:
            del self.vae.vae_options[TORCH_COMPILE_KWARGS_VAE]

        print("✅ VAE compilation removed")


class TorchCompileDiffusionOpenVINO(io.ComfyNode):
    @classmethod
    def define_schema(cls) -> io.Schema:
        core = ov.Core()
        available_devices = core.available_devices
        return io.Schema(
            node_id="TorchCompileDiffusionOpenVINO",
            category="OpenVINO",
            inputs=[
                io.Model.Input("model"),
                io.Combo.Input("device", options=available_devices),
            ],
            outputs=[io.Model.Output()],
            is_experimental=True,
        )

    @classmethod
    def execute(cls, model, device) -> io.NodeOutput:
        torch._dynamo.reset()
        _apply_dynamo_workarounds()
        ov_ex.compiled_cache.clear()
        ov_ex.req_cache.clear()
        ov_ex.partitioned_modules.clear()
        ov_ex.max_openvino_partitions = 0

        m = model.clone()
        set_torch_compile_wrapper(
            model=m,
            backend="openvino",
            options={"device": device}
        )
        return io.NodeOutput(m)


class TorchCompileVAEOpenVINO(io.ComfyNode):
    @classmethod
    def define_schema(cls) -> io.Schema:
        core = ov.Core()
        available_devices = core.available_devices
        return io.Schema(
            node_id="TorchCompileVAEOpenVINO",
            category="OpenVINO",
            inputs=[
                io.Vae.Input("vae"),
                io.Combo.Input("device", options=available_devices),
                io.Boolean.Input("compile_encoder", default=True),
                io.Boolean.Input("compile_decoder", default=True),
                io.Boolean.Input("remove_compile", default=False,
                               tooltip="Remove VAE compilation"),
            ],
            outputs=[io.Vae.Output()],
            is_experimental=True,
        )

    @classmethod
    def execute(cls, vae, device, compile_encoder, compile_decoder, remove_compile) -> io.NodeOutput:
        torch._dynamo.reset()
        _apply_dynamo_workarounds()
        ov_ex.compiled_cache.clear()
        ov_ex.req_cache.clear()
        ov_ex.partitioned_modules.clear()
        ov_ex.max_openvino_partitions = 0

        # Get or create wrapper
        if not hasattr(vae, '_compile_wrapper'):
            vae._compile_wrapper = VAECompileWrapper(vae)

        wrapper = vae._compile_wrapper

        # Remove compilation if requested
        if remove_compile:
            wrapper.remove()
            return io.NodeOutput(vae)

        # Otherwise compile as requested
        keys = []
        first_stage = vae.first_stage_model
        has_taesd = hasattr(first_stage, "taesd_encoder")

        if compile_encoder:
            keys.append("taesd_encoder" if has_taesd else "encoder")

        if compile_decoder:
            keys.append("taesd_decoder" if has_taesd else "decoder")

        if keys:
            wrapper.compile(
                backend="openvino",
                options={"device": device},
                keys=keys,
            )

        return io.NodeOutput(vae)


class OpenVINOTorchCompileExtension(ComfyExtension):
    @override
    async def get_node_list(self) -> list[type[io.ComfyNode]]:
        return [
            TorchCompileDiffusionOpenVINO,
            TorchCompileVAEOpenVINO,
        ]


async def comfy_entrypoint() -> OpenVINOTorchCompileExtension:
    return OpenVINOTorchCompileExtension()


NODE_CLASS_MAPPINGS = {
    "TorchCompileVAEOpenVINO": TorchCompileVAEOpenVINO,
    "TorchCompileDiffusionOpenVINO": TorchCompileDiffusionOpenVINO,
}