From e2e7462292c063e335aa2f6fcfc910b5208bacbf Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Thu, 24 Jul 2025 23:23:00 +0000 Subject: [PATCH 01/10] add patches --- optimum/torch_export_patches/__init__.py | 0 optimum/torch_export_patches/_core.py | 527 +++++++ optimum/torch_export_patches/_patch_torch.py | 427 ++++++ .../_patch_transformers.py | 1306 +++++++++++++++++ .../_treenode_registry.py | 276 ++++ pyproject.toml | 4 + 6 files changed, 2540 insertions(+) create mode 100644 optimum/torch_export_patches/__init__.py create mode 100644 optimum/torch_export_patches/_core.py create mode 100644 optimum/torch_export_patches/_patch_torch.py create mode 100644 optimum/torch_export_patches/_patch_transformers.py create mode 100644 optimum/torch_export_patches/_treenode_registry.py diff --git a/optimum/torch_export_patches/__init__.py b/optimum/torch_export_patches/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/optimum/torch_export_patches/_core.py b/optimum/torch_export_patches/_core.py new file mode 100644 index 00000000..064431ad --- /dev/null +++ b/optimum/torch_export_patches/_core.py @@ -0,0 +1,527 @@ +import contextlib +import functools +import importlib +import re +from collections.abc import Generator +from typing import Any, Callable + +import _patch_torch +import _patch_transformers +import _treenode_registry +import torch + + +def patched_function(name: str) -> tuple[type, Callable]: + """Returns the module and the function needed to be patched.""" + spl = name.split(".") + module_name = ".".join(spl[:-1]) + fname = spl[-1] + module = importlib.import_module(module_name) + if not hasattr(module, fname): + return None, None + return module, getattr(module, fname) + + +@functools.lru_cache +def get_patches(module: torch.nn.Module) -> tuple[str, list[Any]]: + """Returns the list of patches to make for a specific nn.module.""" + to_patch = [] + for k in dir(module): + if k.startswith("patched_"): + v = getattr(module, k) + if hasattr(v, "_PATCHED_CLASS_") and hasattr(v, "_PATCHES_"): + to_patch.append(v) + else: + # a function + doc = v.__doc__.lstrip() + if doc.startswith("manual patch"): + continue + reg = re.compile("[\\[]patch:([a-z_A-Z.]+)[\\]]") + fall = reg.findall(doc) + assert len(fall) == 1, f"Unable to find patching information for {v} in \n{doc}" + fmod, f = patched_function(fall[0]) + if fmod is None and f is None: + # The function does not exist in this version of transformers. + # No patch is needed. + continue + to_patch.append({"module": fmod, "function": f, "patch": v}) + + name = module.__name__ + return name, to_patch + + +def patch_module_or_class(mod, verbose: int = 0) -> dict[type, dict[type, Callable]]: + """Applies all patches defined in classes prefixed by ``patched_``. + + ``cls._PATCHED_CLASS_`` defines the class to patch, + ``cls._PATCHES_`` defines the method to patch. + The returns information needs to be sent to :func:`unpatch_module_or_class` + to revert the changes. + + Args: + mod: module of list of clsses to patch + verbose: verbosity + + Returns: + patch info + """ + if isinstance(mod, list): + to_patch = mod + name = "list" + else: + name, to_patch = get_patches(mod, verbose) + + res = {} + for cls in to_patch: + if isinstance(cls, dict): + # a function + keep = {} + original = cls["module"] + f = cls["function"] + res[f] = f + if verbose: + print(f"[patch_module_or_class] function: {original.__name__}.{f.__name__}") + setattr(original, f.__name__, cls["patch"]) + continue + + original = cls._PATCHED_CLASS_ + methods = cls._PATCHES_ + if verbose: + print(f"[patch_module_or_class] {name}.{cls.__name__}: {', '.join(methods)}") + + keep = {n: getattr(original, n, None) for n in methods} + for n in methods: + setattr(original, n, getattr(cls, n)) + res[cls] = keep + + return res + + +def unpatch_module_or_class(mod, info: dict[type, dict[type, Callable]], verbose: int = 0): + """Reverts modification made by :func:`patch_module_or_class`. + + Args: + mod: module of list of clsses to patch + info: patch information returned by patch_module_or_class + verbose: verbosity + """ + if isinstance(mod, list): + to_patch = mod + name = "list" + else: + name, to_patch = get_patches(mod, verbose) + + set_patch_cls = {i for i in to_patch if not isinstance(i, dict)} + dict_patch_fct = {i["function"]: i for i in to_patch if isinstance(i, dict)} + + for cls, methods in info.items(): + if cls in set_patch_cls: + if verbose: + print(f"[unpatch_module_or_class] {name}.{cls.__name__}: {', '.join(methods)}") + original = cls._PATCHED_CLASS_ + for n, v in methods.items(): + if v is None: + # The method did not exist. We remove it. + delattr(original, n) + else: + setattr(original, n, v) + continue + assert cls in dict_patch_fct, ( + f"No patch registered for {cls} in {mod} (found {set_patch_cls} and {set(dict_patch_fct)})" + ) + patch = dict_patch_fct[cls] + if verbose: + print(f"[unpatch_module_or_class] function {patch['module'].__name__}.{cls.__name__}") + setattr(patch["module"], cls.__name__, patch["function"]) + + +@contextlib.contextmanager +def torch_export_patches( + patch_sympy: bool = True, + patch_torch: bool = True, + patch_transformers: bool = False, + patch_diffusers: bool = False, + catch_constraints: bool = True, + stop_if_static: int = 0, + verbose: int = 0, + patch: bool = True, + custom_patches: type["torch.nn.Module"] | None = None, + rewrite: list[Callable] | None = None, + dump_rewriting: str | None = None, +) -> Generator[Any, Any, Any]: + """Bypass the situations :func:`torch.export.export` does not support. + + See also :ref:`l-patches-explained` and :ref:`l-patch-coverage`. + + Args: + patch_sympy: fix missing method ``name`` for IntegerConstant + patch_torch: patches :epkg:`torch` with supported implementation + patch_transformers: patches :epkg:`transformers` with supported implementation + patch_diffusers: patches :epkg:`diffusers` with supported implementation + catch_constraints: catch constraints related to dynamic shapes, + as a result, some dynamic dimension may turn into static ones, + the environment variable ``SKIP_SOLVE_CONSTRAINTS=0`` + can be put to stop at that stage. + stop_if_static: see example :ref:`l-plot-export-locale-issue`, + to stop the export as soon as an issue is detected with dynamic shapes + and show a stack trace indicating the exact location of the issue, + ``if stop_if_static > 1``, more methods are replace to catch more + issues + patch: if False, disable all patches but keeps the registration of + serialization functions if other patch functions are enabled + custom_patches: to apply custom patches, + every patched class must define static attributes + ``_PATCHES_``, ``_PATCHED_CLASS_`` + rewrite: list of methods to automatically rewrite + before exporting, methods with control flow need to be rewritten + before being exported if the execution path depends on the inputs, + this is done by function :func:`transform_method + `, + its documentation provides possible values + dump_rewriting: dumps rewriting information in file beginning with that prefix + verbose: to show which patches is applied + + The list of available patches: + + * ``torch.jit.isinstance`` + * ``torch._dynamo.mark_static_address`` + * ``torch._subclasses.fake_impls.infer_size`` + * ``torch.vmap`` + * fix missing method ``name`` for ``sympy.S.IntegerConstant`` + * ``AttentionMaskConverter._make_causal_mask`` + * Serialization of ``MambaCache`` (in :epkg:`transformers`) + * Serialization of ``DynamicCache`` (in :epkg:`transformers`) + * reduce errors due to shape inference + * fixes some transformers classes, + see :mod:`onnx_diagnostic.torch_export_patches.patches.patch_transformers` + + Serialization issues happen when a module takes one input or output + has a type :func:`torch.export.export` cannot serialize. + + Examples: + .. code-block:: python + + with torch_export_patches(patch_transformers=True) as modificator: + inputs = modificator(inputs) + onx = to_onnx(..., inputs, ...) + + .. code-block:: python + + with torch_export_patches(patch_transformers=True) as modificator: + inputs = modificator(inputs) + onx = torch.onnx.export(..., inputs, ...) + + It can be used as well to fix the torch export: + + .. code-block:: python + + with torch_export_patches(patch_transformers=True) as modificator: + inputs = modificator(inputs) + ep = torch.export.export(..., inputs, ...) + + When running the model through the exported program, only the + serialization functions need to be restored: + + .. code-block:: python + + with register_additional_serialization_functions() as modificator: + inputs = modificator(inputs) + ep = torch.export.export(..., inputs, ...) + + When exporting a model with a cache, the following error message + may appear ``AssertionError: Mutating module attribute _seen_tokens during export.``. + It can be avoided by setting ``strict=False`` when call :func:`torch.export.export`. + """ + if rewrite: + raise ValueError( + "The argument `rewrite` is not supported at the moment, as the automatic rewriting is not implemented yet." + ) + # from .patch_module import torch_export_rewrite + + # with torch_export_rewrite( + # rewrite=rewrite, dump_rewriting=dump_rewriting, verbose=verbose + # ), torch_export_patches( # type: ignore[var-annotated] + # patch_sympy=patch_sympy, + # patch_torch=patch_torch, + # patch_transformers=patch_transformers, + # patch_diffusers=patch_diffusers, + # catch_constraints=catch_constraints, + # stop_if_static=stop_if_static, + # verbose=verbose, + # patch=patch, + # custom_patches=custom_patches, + # ) as f: + # try: + # yield f + # finally: + # pass + # elif not patch: + if not patch: + fct_callable = lambda x: x # noqa: E731 + done = _treenode_registry.register_cache_serialization( + patch_transformers=patch_transformers, + patch_diffusers=patch_diffusers, + verbose=verbose, + ) + try: + yield fct_callable + finally: + _treenode_registry.unregister_cache_serialization(done, verbose=verbose) + else: + import torch + import torch._export.non_strict_utils # produce_guards_and_solve_constraints + import torch.jit + + if verbose: + print("[torch_export_patches] replace torch.jit.isinstance, torch._dynamo.mark_static_address") + + ######## + # caches + ######## + + cache_done = _treenode_registry.register_cache_serialization( + patch_transformers=patch_transformers, + patch_diffusers=patch_diffusers, + verbose=verbose, + ) + + ############# + # patch sympy + ############# + + if patch_sympy: + import sympy + + f_sympy_name = getattr(sympy.core.numbers.IntegerConstant, "name", None) + + if verbose: + print(f"[torch_export_patches] sympy.__version__={sympy.__version__!r}") + print("[torch_export_patches] patch sympy") + + sympy.core.numbers.IntegerConstant.name = lambda self: f"IntCst{self!s}" + + ############### + # patch pytorch + ############### + + if patch_torch: + if verbose: + print(f"[torch_export_patches] torch.__version__={torch.__version__!r}") + print(f"[torch_export_patches] stop_if_static={stop_if_static!r}") + print("[torch_export_patches] patch pytorch") + + # torch.vmap + f_vmap = torch.vmap + torch.vmap = _patch_torch.patched_vmap + + # torch.jit.isinstance + f_jit_isinstance = torch.jit.isinstance + torch.jit.isinstance = isinstance + + # torch._dynamo.mark_static_address + f_mark_static_address = torch._dynamo.mark_static_address + torch._dynamo.mark_static_address = lambda *_, **y_: None + + # torch._subclasses.fake_impls.infer_size + f_infer_size = torch._subclasses.fake_impls.infer_size + torch._subclasses.fake_impls.infer_size = _patch_torch.patched_infer_size + + # torch._refs._broadcast_shapes + f__broadcast_shapes = torch._refs._broadcast_shapes + torch._refs._broadcast_shapes = _patch_torch.patched__broadcast_shapes + torch._meta_registrations._broadcast_shapes = _patch_torch.patched__broadcast_shapes + + # torch._export.non_strict_utils.produce_guards_and_solve_constraints + if catch_constraints: + if verbose: + print("[torch_export_patches] modifies shape constraints") + f_produce_guards_and_solve_constraints = ( + torch._export.non_strict_utils.produce_guards_and_solve_constraints + ) + f__check_input_constraints_for_graph = torch._export.utils._check_input_constraints_for_graph + torch._export.non_strict_utils.produce_guards_and_solve_constraints = ( + lambda *args, **kwargs: _patch_torch.catch_produce_guards_and_solve_constraints( + f_produce_guards_and_solve_constraints, *args, verbose=verbose, **kwargs + ) + ) + torch._export.utils._check_input_constraints_for_graph = ( + lambda *args, **kwargs: _patch_torch.patch__check_input_constraints_for_graph( + f__check_input_constraints_for_graph, *args, verbose=verbose, **kwargs + ) + ) + + if stop_if_static: + from torch.fx.experimental.symbolic_shapes import ShapeEnv + + ShapeEnv._log_guard_remember = ShapeEnv._log_guard + + if verbose: + print("[torch_export_patches] assert when a dynamic dimension turns static") + print("[torch_export_patches] replaces ShapeEnv._set_replacement") + + f_shape_env__set_replacement = ShapeEnv._set_replacement + ShapeEnv._set_replacement = _patch_torch.patched_ShapeEnv._set_replacement + + if verbose: + print("[torch_export_patches] replaces ShapeEnv._log_guard") + f_shape_env__log_guard = ShapeEnv._log_guard + ShapeEnv._log_guard = _patch_torch.patched_ShapeEnv._log_guard + + if stop_if_static > 1: + if verbose: + print("[torch_export_patches] replaces ShapeEnv._check_frozen") + f_shape_env__check_frozen = ShapeEnv._check_frozen + ShapeEnv._check_frozen = _patch_torch.patched_ShapeEnv._check_frozen + + #################### + # patch transformers + #################### + + if patch_transformers: + try: + import transformers.masking_utils as masking_utils + except ImportError: + masking_utils = None + + if verbose: + import transformers + + print(f"[torch_export_patches] transformers.__version__={transformers.__version__!r}") + revert_patches_info = patch_module_or_class(_patch_transformers, verbose=verbose) + + if masking_utils and _patch_transformers.patch_masking_utils and hasattr(masking_utils, "_vmap_for_bhqkv"): + if verbose: + print("[torch_export_patches] patches transformers.masking_utils._vmap_for_bhqkv") + f_transformers__vmap_for_bhqkv = masking_utils._vmap_for_bhqkv + masking_utils._vmap_for_bhqkv = _patch_transformers.patched__vmap_for_bhqkv + + if masking_utils and _patch_transformers.patch_masking_utils and hasattr(masking_utils, "eager_mask"): + if verbose: + print("[torch_export_patches] patches transformers.masking_utils.eager_mask") + f_transformers_eager_mask = masking_utils.eager_mask + masking_utils.eager_mask = _patch_transformers.patched_eager_mask + if ( + "eager" in masking_utils.ALL_MASK_ATTENTION_FUNCTIONS + and masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["eager"] == f_transformers_eager_mask + ): + masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["eager"] = _patch_transformers.patched_eager_mask + + if custom_patches: + if verbose: + print("[torch_export_patches] applies custom patches") + revert_custom_patches_info = patch_module_or_class(custom_patches, verbose=verbose) + + ######## + # export + ######## + + if verbose: + print("[torch_export_patches] done patching") + + try: + yield fct_callable + finally: + ####### + # sympy + ####### + + if verbose: + print("[torch_export_patches] remove patches") + + if patch_sympy: + # tracked by https://github.com/pytorch/pytorch/issues/143494 + if f_sympy_name: + sympy.core.numbers.IntegerConstant.name = f_sympy_name + else: + delattr(sympy.core.numbers.IntegerConstant, "name") + + if verbose: + print("[torch_export_patches] restored sympy functions") + + ####### + # torch + ####### + + if patch_torch: + # this should disappear when torch.jit is removed + torch.vmap = f_vmap + torch.jit.isinstance = f_jit_isinstance + torch._dynamo.mark_static_address = f_mark_static_address + # tracked by https://github.com/pytorch/pytorch/issues/143495 + torch._subclasses.fake_impls.infer_size = f_infer_size + torch._refs._broadcast_shapes = f__broadcast_shapes + torch._meta_registrations._broadcast_shapes = f__broadcast_shapes + + if verbose: + print("[torch_export_patches] restored pytorch functions") + + if stop_if_static: + if verbose: + print("[torch_export_patches] restored ShapeEnv._set_replacement") + + ShapeEnv._set_replacement = f_shape_env__set_replacement + + if verbose: + print("[torch_export_patches] restored ShapeEnv._log_guard") + + ShapeEnv._log_guard = f_shape_env__log_guard + + if stop_if_static > 1: + if verbose: + print("[torch_export_patches] restored ShapeEnv._check_frozen") + ShapeEnv._check_frozen = f_shape_env__check_frozen + + if catch_constraints: + # to catch or skip dynamic_shapes issues + torch._export.non_strict_utils.produce_guards_and_solve_constraints = ( + f_produce_guards_and_solve_constraints + ) + torch._export.utils._check_input_constraints_for_graph = f__check_input_constraints_for_graph + if verbose: + print("[torch_export_patches] restored shape constraints") + + if custom_patches: + if verbose: + print("[torch_export_patches] unpatches custom patches") + unpatch_module_or_class(custom_patches, revert_custom_patches_info, verbose=verbose) + + ############## + # transformers + ############## + + if patch_transformers: + try: + # masking_utils is available only if transformers==4.53.0 or later + import transformers.masking_utils as masking_utils + except ImportError: + masking_utils = None + if verbose: + print("[torch_export_patches] unpatches transformers") + unpatch_module_or_class(_patch_transformers, revert_patches_info, verbose=verbose) + + if ( + masking_utils + and _patch_transformers.patch_masking_utils + and hasattr(masking_utils, "_vmap_for_bhqkv") + ): + masking_utils._vmap_for_bhqkv = f_transformers__vmap_for_bhqkv + if verbose: + print("[torch_export_patches] restored transformers.masking_utils._vmap_for_bhqkv") + + if masking_utils and _patch_transformers.patch_masking_utils and hasattr(masking_utils, "eager_mask"): + f_transformers_eager_mask = masking_utils.eager_mask + masking_utils.eager_mask = f_transformers_eager_mask + if ( + "eager" in masking_utils.ALL_MASK_ATTENTION_FUNCTIONS + and masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["eager"] + == _patch_transformers.patched_eager_mask + ): + masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["eager"] = f_transformers_eager_mask + if verbose: + print("[torch_export_patches] restored transformers.masking_utils.eager_mask") + + ######## + # caches + ######## + + _treenode_registry.unregister_cache_serialization(cache_done, verbose=verbose) diff --git a/optimum/torch_export_patches/_patch_torch.py b/optimum/torch_export_patches/_patch_torch.py new file mode 100644 index 00000000..3327964f --- /dev/null +++ b/optimum/torch_export_patches/_patch_torch.py @@ -0,0 +1,427 @@ +import inspect +import os +from typing import Any, Callable +import torch + + +def catch_produce_guards_and_solve_constraints( + previous_function: Callable, + fake_mode: "FakeTensorMode", + gm: "torch.fx.GraphModule", + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None, + equalities_inputs: "EqualityConstraint", + original_signature: inspect.Signature, + _is_torch_jit_trace: bool = False, + verbose: int = 0, +): + try: + return previous_function( + fake_mode=fake_mode, + gm=gm, + dynamic_shapes=dynamic_shapes, + equalities_inputs=equalities_inputs, + original_signature=original_signature, + _is_torch_jit_trace=_is_torch_jit_trace, + ) + except Exception as e: + if not int(os.environ.get("SKIP_SOLVE_CONSTRAINTS", "1")): + raise + if verbose: + print( + f"[_catch_produce_guards_and_solve_constraints] ERROR: " + f"produce_guards_and_solve_constraints failed, " + f"use SKIP_SOLVE_CONSTRAINTS=0 to avoid skipping\n" + f"fake_mode={fake_mode}\n" + f"dynamic_shapes={dynamic_shapes}\n" + f"equalities_inputs={equalities_inputs}\n" + f"original_signature={original_signature}\n" + f"_is_torch_jit_trace={_is_torch_jit_trace}\n" + f"exc={e}\ngm={gm}" + ) + torch._dynamo.reset() + + +def patch__check_input_constraints_for_graph( + previous_function: Callable, + input_placeholders: list[torch.fx.Node], + flat_args_with_path, + range_constraints, + verbose: int = 0, +) -> None: + try: + return previous_function(input_placeholders, flat_args_with_path, range_constraints) + except Exception as e: + if not int(os.environ.get("SKIP_SOLVE_CONSTRAINTS", "1")): + raise + if verbose: + print( + f"[_check_input_constraints_for_graph] ERROR: " + f"_check_input_constraints_for_graph failed, " + f"use SKIP_SOLVE_CONSTRAINTS=0 to avoid skipping\n" + f"input_placeholders={input_placeholders}\n" + f"range_constraints={range_constraints}\n" + f"exc={e}" + ) + torch._dynamo.reset() + + +def patched_infer_size(a, b): + """Patches ``torch._subclasses.fake_impls.infer_size``.""" + from torch.fx.experimental.symbolic_shapes import guard_size_oblivious + + dimsa = len(a) + dimsb = len(b) + ndim = max(dimsa, dimsb) + expandedSizes = [0] * ndim + for i in range(ndim - 1, -1, -1): + offset = ndim - 1 - i + dima = dimsa - 1 - offset + dimb = dimsb - 1 - offset + sizeA = a[dima] if dima >= 0 else 1 + sizeB = b[dimb] if dimb >= 0 else 1 + + # NB: It is very important to test for broadcasting, before testing + # sizeA == sizeB. This is because the broadcasting tests are likely + # to be statically known (in particular, if sizeA/sizeB is unbacked + # but size-like, we will unsoundly assume they never equal 1), but + # the sizeA == sizeB test may not be statically known. However, once + # we have established that no broadcasting is happening, the + # sizeA == sizeB is now expect_true and we can defer it as a runtime + # assert (this works because Python will return the terminal + # expression of an or statement as-is, without bool()'ing it; if this + # were not the case, we'd need to write this using torch.sym_or() or + # something like that). + try: + b1 = guard_size_oblivious(sizeA == 1) + except torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode: + b1 = False + try: + b2 = guard_size_oblivious(sizeB == 1) + except torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode: + b2 = False + try: + b3 = guard_size_oblivious(sizeA == sizeB) + except torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode: + b3 = False + if b1 or b2 or b3: + expandedSizes[i] = sizeB if guard_size_oblivious(sizeA == 1) else sizeA + else: + # In this case, the current implementation of torch fails (17/12/2024). + # Try model SmolLM. + expandedSizes[i] = torch.sym_max(sizeA, sizeB) + return tuple(expandedSizes) + + +def patched__broadcast_shapes(*_shapes): + """Patches ``torch._refs._broadcast_shapes``.""" + from functools import reduce + from torch._prims_common import IntLike + from torch.fx.experimental.symbolic_shapes import guard_size_oblivious + + shapes = tuple( + (x,) if isinstance(x, IntLike) else x for x in filter(lambda x: x is not None, _shapes) + ) + + # Short-circuits on no input + if len(shapes) == 0: + return None + + # Type checking + # TODO: make common validations available as utils + for shape in shapes: + assert isinstance(shape, Sequence) + + # Computes common shape + common_shape = [ # List[Union[int, torch.SymInt]] + 1, + ] * reduce(max, (len(shape) for shape in shapes)) + for _arg_idx, shape in enumerate(shapes): + for idx in range(-1, -1 - len(shape), -1): + if guard_size_oblivious(common_shape[idx] == 1): + if shape[idx] < 0: + raise ValueError( + "Attempting to broadcast a dimension with negative length!" + ) + common_shape[idx] = shape[idx] + elif guard_size_oblivious(shape[idx] != 1): + common_shape[idx] = torch.sym_max(common_shape[idx], shape[idx]) + + return common_shape + + +class patched_ShapeEnv: + + def _check_frozen( + self, expr: "sympy.Basic", concrete_val: "sympy.Basic" # noqa: F821 + ) -> None: + if self.frozen: + self.counter["ignored_backward_guard"] += 1 + raise AssertionError( + f"[patched_ShapeEnv] Ignored guard {expr} == {concrete_val}, " + f"this could result in accuracy problems." + ) + + def _set_replacement( + self, a: "sympy.Symbol", tgt: "sympy.Expr", msg: str # noqa: F821 + ) -> None: + """ + Adds or updates a replacement for a symbol. + Use this instead of `self.replacements[a] = tgt`. + """ + if tgt == self.replacements.get(a, None): + return + + if a in tgt.free_symbols: + return + + import sympy + from torch._logging import structured + from torch.utils._traceback import CapturedTraceback + from torch._logging import trace_structured + from torch._guards import TracingContext + from torch.utils._sympy.functions import FloorToInt, CeilToInt + from torch.utils._sympy.solve import try_solve + from torch.fx.experimental.symbolic_shapes import ( + _is_supported_equivalence, + ValueRanges, + ) + + # Precondition: a == tgt + assert isinstance(a, sympy.Symbol) + + if self.allow_complex_guards_as_runtime_asserts and not _is_supported_equivalence(tgt): + # continuing leads to placeholder shapes + # having complex expressions that we can't resolve + return + + # Handles nested tensor symbolic variables which don't have + # var_to_range bounds + tgt_bound = None + if a in self.var_to_range: + src_bound = self.var_to_range[a] + + # First, refine the value range of a based on the computed value range + # of tgt. This is always OK to do, even if we decide not to do the + # substitution in the end. This might be a no-op, if a already has + # a tighter bound + tgt_bound = self.bound_sympy(tgt) + self._update_var_to_range(a, tgt_bound) + + # Next, check if we can update the range of free symbols in tgt + # based on the range in a. But only do it if: + # - the source bound non-trivially improves over what we get out of + # the existing bounds. + # - the replacement is univariate and we can invert the tgt expression + if not tgt_bound.issubset(src_bound) and len(tgt.free_symbols) == 1: + b = next(iter(tgt.free_symbols)) + # Try to invert the equality + r = try_solve(sympy.Eq(a, tgt), b, floordiv_inequality=False) + if r is not None: + self.log.debug( + "set_replacement: solve for %s in %s == %s gives %s", + b, + a, + tgt, + r, + ) + # The solution here can be non-integral, for example, if + # we have s0 = 2*s1, then s1 = s0/2. What we would like + # to do is calculated the bounds in arbitrary precision, + # and then requantize the bound to integers when we are + # done. + rat_b_bound = self.bound_sympy(r[1]) + b_bound = ValueRanges( + CeilToInt(rat_b_bound.lower), FloorToInt(rat_b_bound.upper) + ) + self._update_var_to_range(b, b_bound, self.var_to_range_sloc[a]) + tgt_bound = self.bound_sympy(tgt) + assert tgt_bound.issubset( + src_bound + ), f"{tgt_bound=} not a subset of {src_bound=}" + + # TODO: Should we propagate size-like-ness? + # + # Pros: if u0 is size-like, intuitively u0 == u1 should cause u1 + # to become size-like. + # + # Cons: if u0 is size-like, what about u0 - 1 == u1? You CAN'T + # propagate in this case, because what if u0 == 0, then u1 is negative + # and clearly isn't a size. So, at minimum, any f(x) whose value + # range isn't [0, inf] given x in [0, inf] cannot propagate + # size-like-ness. But there are many situations where you could + # imagine u1 is going to be size-like and actually you just didn't + # have a refined enough value range on u0. Since even innocuous + # looking arithmetic operations can destroy size-like-ness, it's + # best to not propagate it at all and force the user to annotate it + # as necessary. + # + # Compromise: we preserve size-like-ness only for exact equality + # and nothing else. + if a in self.size_like and isinstance(tgt, sympy.Symbol): + self.size_like.add(tgt) + elif isinstance(tgt, sympy.Symbol) and tgt in self.size_like: + self.size_like.add(a) + + # Now, decide if we will do the substitution. + # + # - If the source has a non-trivial range, only substitute if + # we preserve this range. Note that we may have propagated + # the src_range to free variables in tgt when tgt is univariate + # and we could find an inverse, which helps us achieve this. + # This ensures we never "forget" about user defined ranges, + # even if they end up being defined on composite formulas + # like s0 + s1. + # + # - If the variable is unbacked, only substitute if the substitution + # would preserve the bounds also under size-like-ness conditions. + + if not tgt_bound.issubset(src_bound): + self.log.debug( + "skipped set_replacement %s = %s (%s) [%s not subset of %s]", + a, + tgt, + msg, + tgt_bound, + src_bound, + ) + return + elif a in self.size_like: + tgt_bound_so = self.bound_sympy(tgt, size_oblivious=True) + src_bound_so = self.bound_sympy(a, size_oblivious=True) + if not tgt_bound_so.issubset(src_bound_so): + self.log.debug( + "skipped set_replacement %s = %s (%s) " + "[%s not subset of %s (size-oblivious conditions)]", + a, + tgt, + msg, + tgt_bound_so, + src_bound_so, + ) + return + + if isinstance(tgt, (sympy.Integer, sympy.Float)): + # specializing to a constant, which is likely unexpected (unless + # you specified dynamic=True) + + user_tb = TracingContext.extract_stack() + trace_structured( + "symbolic_shape_specialization", + metadata_fn=lambda: { + "symbol": repr(a), + "sources": [s.name() for s in self.var_to_sources.get(a, [])], + "value": repr(tgt), + "reason": msg, + "stack": structured.from_traceback( + CapturedTraceback.extract(skip=1).summary() + ), + "user_stack": (structured.from_traceback(user_tb) if user_tb else None), + }, + ) + + # if config.print_specializations: + # self.log.warning( + # "Specializing %s to %s", self.var_to_sources[a][0].name(), tgt + # ) + # self.log.debug("SPECIALIZATION", stack_info=True) + assert msg != "range_refined_to_singleton", ( + f"patched_ShapeEnv: A dynamic dimension becomes static! " + f"a={a!r}, tgt={tgt!r}, msg={msg!r}, tgt_bound={tgt_bound}" + ) + # log.info("set_replacement %s = %s (%s) %s", a, tgt, msg, tgt_bound) + self.replacements[a] = tgt + # NB: the replacement may get refined, but the user will find the + # FIRST one most useful (TODO: Maybe we could consider tracking all of + # them) + if a not in self.replacements_slocs: + self.replacements_slocs[a] = self._get_sloc() + self._update_version_counter() + + # When specializing 'a == tgt', the equality should be also conveyed to + # Z3, in case an expression uses 'a'. + self._add_target_expr(sympy.Eq(a, tgt, evaluate=False)) + + def _log_guard( + self, prefix: str, g: "SympyBoolean", forcing_spec: bool # noqa: F821 + ) -> None: + self._log_guard_remember(prefix=prefix, g=g, forcing_spec=forcing_spec) + + +def patched_vmap(func, in_dims=0, out_dims=0): + """Python implementation of :func:`torch.vmap`. + + The implementation raises an issue when it is being exported with + :func:`torch.export.export` when the function is called with + non tensors arguments and the batch size is dynamic. + """ + + def wrapped(*args): + in_dims_ = ( + ([in_dims] * len(args)) + if not isinstance(in_dims, (list, tuple)) + else list(in_dims) + ) + batch_size = None + batched_args = [] + for arg, in_dim in zip(args, in_dims_): + if in_dim is None: + batched_args.append(arg) + continue + + assert batch_size is None or batch_size == arg.size(in_dim), ( + f"Unable to continue, batch_size={batch_size}, in_dim={in_dim}, " + f"arg.size(in_dim)={arg.size(in_dim)}" + ) + if batch_size is None: + batch_size = arg.size(in_dim) + arg = arg.movedim(in_dim, 0) + batched_args.append(arg) + + if all(isinstance(a, torch.Tensor) for a in args) and isinstance( + batch_size, torch.SymInt + ): + batched_tensors = [ + ( + arg + if (isinstance(arg, torch.Tensor) and in_dim is not None) + else arg.unsqueeze(0).expand((batch_size, *arg.shape)) + ) + for arg, in_dim in zip(batched_args, in_dims_) + ] + results = torch.ops.higher_order.scan(func, [], batched_tensors, []) + stacked = results[0] + if out_dims != 0: + return stacked.movedim(0, out_dims) + return stacked + + else: + torch._check( + not isinstance(batch_size, torch.SymInt), + lambda: ( + f"patched_vmap supports dynamic batch_size only if all argument " + f"are tensors but types are {[type(a) for a in args]}" + ), + ) + batched_tensors = [ + ( + (None, arg) + if (isinstance(arg, torch.Tensor) and in_dim is not None) + else (arg, arg) + ) + for arg, in_dim in zip(batched_args, in_dims_) + ] + + results = [] + for i in range(batch_size): + input_slice = [v if v is not None else arg[i] for v, arg in batched_tensors] + result = func(*input_slice) + results.append(result) + + if isinstance(results[0], torch.Tensor): + stacked = torch.stack(results) + if out_dims != 0: + return stacked.movedim(0, out_dims) + return stacked + return results + + return wrapped diff --git a/optimum/torch_export_patches/_patch_transformers.py b/optimum/torch_export_patches/_patch_transformers.py new file mode 100644 index 00000000..50711283 --- /dev/null +++ b/optimum/torch_export_patches/_patch_transformers.py @@ -0,0 +1,1306 @@ +import inspect +from dataclasses import dataclass +from functools import wraps +from typing import Any, Callable, Dict, List, Optional, Tuple +import packaging.version as pv +import torch +import transformers +from transformers.modeling_attn_mask_utils import AttentionMaskConverter +from transformers.cache_utils import StaticCache, Cache, DynamicCache + +try: + import transformers.masking_utils + + patch_masking_utils = True +except ImportError: + patch_masking_utils = False + +from ...ext_test_case import has_transformers +from ...helpers.torch_helper import is_torchdynamo_exporting + + +if patch_masking_utils: + # Introduced in 4.52 + from transformers.masking_utils import causal_mask_function, sdpa_mask + + def patched__vmap_for_bhqkv(mask_function: Callable, bh_indices: bool = True) -> Callable: + """manual patch for function ``transformers.masking_utils._vmap_for_bhqkv``.""" + from ...helpers import string_type + + dimensions: List[Tuple[Optional[int], ...]] = [ + (None, None, None, 0), + (None, None, 0, None), + ] + if bh_indices: + dimensions.extend([(None, 0, None, None), (0, None, None, None)]) + # reshape + dimensions = [tuple(1 if d is None else -1 for d in shape) for shape in dimensions] + dimensions = tuple(reversed(dimensions)) + indices = tuple(shape.index(-1) for shape in dimensions) + + # unsqueeze + udimensions = [ + tuple(di for di, d in enumerate(shape) if d == 1) for shape in dimensions + ] + + def vector_mask_function( + *args, mask_function=mask_function, dimensions=dimensions, indices=indices + ): + assert len(args) == len(dimensions) == len(udimensions), ( + f"Mismatch between args={string_type(args)} and dimensions={dimensions} " + f"and udimensions={udimensions}." + ) + assert len(indices) == len(args), ( + f"Mismatch between args={string_type(args)} and indices={indices}, " + f"they should have the same length." + ) + for a in args: + assert ( + a.ndim == 1 + ), f"Expected a tensor with 1 dimension not {string_type(a, with_shape=True)}" + torch._check(a.shape[0] > 0) + + new_args = [a.reshape(shape) for a, shape in zip(args, dimensions)] + # new_args = [ + # a.unsqueeze(dims[0]).unsqueeze(dims[1]).unsqueeze(dims[2]) + # for a, dims in zip(args, udimensions) + # ] + max_shape = tuple(args[i].shape[0] for i in indices) + # if is_torchdynamo_exporting(): + # for a in args: + # # The exporter should export with a dimension > 1 + # # to make sure it is dynamic. + # torch._check(a.shape[0] > 1) + expanded_args = [a.expand(max_shape) for a in new_args] + return mask_function(*expanded_args) + + return vector_mask_function + + def patched_eager_mask( + batch_size: int, + cache_position: torch.Tensor, + kv_length: int, + kv_offset: int = 0, + mask_function: Callable = causal_mask_function, + attention_mask: Optional[torch.Tensor] = None, + dtype: torch.dtype = torch.float32, + **kwargs, + ) -> torch.Tensor: + """manual patch for function ``transformers.masking_utils.eager_mask``.""" + # The masks for eager attention are simply boolean mask from sdpa, casted to 0 and -inf + _ = kwargs.pop("allow_is_causal_skip", None) + mask = sdpa_mask( + batch_size=batch_size, + cache_position=cache_position, + kv_length=kv_length, + kv_offset=kv_offset, + mask_function=mask_function, + attention_mask=attention_mask, + allow_is_causal_skip=False, + allow_torch_fix=False, + **kwargs, + ) + min_dtype = torch.finfo(dtype).min + # The patched line. + # we need 0s where the tokens should be taken into account, + # and -inf otherwise (mask is already of boolean type) + # mask = + # torch.where(mask, torch.tensor(0.0, device=mask.device, dtype=dtype), min_dtype) + mask = (~mask).to(dtype) * min_dtype + return mask + + +def _patch_make_causal_mask( + input_ids_shape: torch.Size, + dtype: torch.dtype, + device: torch.device, + past_key_values_length: int = 0, + sliding_window: Optional[int] = None, +): + """Patched method.""" + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat( + [ + torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), + mask, + ], + dim=-1, + ) + + if sliding_window is not None: + diagonal = past_key_values_length - sliding_window - 1 + + context_mask = torch.tril(torch.ones_like(mask, dtype=torch.bool), diagonal=diagonal) + # In this case, the current implementation of torch fails (17/12/2024). + # Try model Phi-3.5-Mini-Instruct. + mask = mask.masked_fill(context_mask, torch.finfo(dtype).min) + + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +@dataclass +class patched_AttentionMaskConverter: + """ + Patches + ``transformers.modeling_attn_mask_utils.AttentionMaskConverter._make_causal_mask``. + """ + + # This method was fixed in 4.51 at least. + _PATCHES_ = ["_make_causal_mask"] if not has_transformers("4.48.3") else [] + _PATCHED_CLASS_ = AttentionMaskConverter + + @staticmethod + def _make_causal_mask( + *args, + **kwargs, + # input_ids_shape: torch.Size, + # dtype: torch.dtype, + # device: torch.device, + # past_key_values_length: int = 0, + # sliding_window: Optional[int] = None, + ): + """ + Patched method. + + This static method may be called with ``AttentionMaskConverter._make_causal_mask`` + or ``self._make_causal_mask``. That changes this argument is receives. + That should not matter but... + The patch should be implemented in another way. static methods do not play well + with a simple replacement. + Fortunately, this patch does not seem to be needed anymore with transformers>=4.48.3. + """ + if args: + index = 0 if isinstance(args[0], (tuple, torch.Size)) else 1 + names = [ + "input_ids_shape", + "dtype", + "device", + "past_key_values_length", + "sliding_window", + ] + for i, a in enumerate(args): + if i < index: + continue + kwargs[names[i - index]] = a + return _patch_make_causal_mask(**kwargs) + + +class patched_DynamicCache: + """ + Applies modifications implemented in PR + `transformers/#36652 `_. + """ + + _PATCHES_ = ["reorder_cache", "update", "crop", "from_batch_splits", "get_seq_length"] + _PATCHED_CLASS_ = transformers.cache_utils.DynamicCache + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. + A layer index can be optionally passed.""" + # TODO: deprecate this function in favor of `cache_position` + is_empty_layer = ( + len(self.key_cache) == 0 # no cache in any layer + or len(self.key_cache) + <= layer_idx # skipped `layer_idx` and hasn't run a layer with cache after it + or self.key_cache[layer_idx].numel() == 0 # the layer has no cache + ) + layer_seq_length = self.key_cache[layer_idx].shape[-2] if not is_empty_layer else 0 + return layer_seq_length + + def reorder_cache(self, beam_idx: torch.LongTensor): + """Reorders the cache for beam search, given the selected beam indices.""" + for layer_idx in range(len(self.key_cache)): + if self.key_cache[layer_idx].numel(): + device = self.key_cache[layer_idx].device + self.key_cache[layer_idx] = self.key_cache[layer_idx].index_select( + 0, beam_idx.to(device) + ) + if self.value_cache[layer_idx].numel(): + device = self.value_cache[layer_idx].device + self.value_cache[layer_idx] = self.value_cache[layer_idx].index_select( + 0, beam_idx.to(device) + ) + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` + and `value_states` for the layer `layer_idx`. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. + No additional arguments are used in `DynamicCache`. + + Return: + A tuple containing the updated key and value states. + """ + # Update the number of seen tokens + if layer_idx == 0: + if hasattr(self, "_seen_tokens"): + self._seen_tokens += key_states.shape[-2] + + # Update the cache + if key_states is not None: + if len(self.key_cache) <= layer_idx: + # There may be skipped layers, fill them with empty lists + for _ in range(len(self.key_cache), layer_idx): + self.key_cache.append(torch.tensor([], dtype=key_states.dtype)) + self.value_cache.append(torch.tensor([], dtype=key_states.dtype)) + self.key_cache.append(key_states) + self.value_cache.append(value_states) + elif not self.key_cache[ + layer_idx + ].numel(): # prefers not t.numel() to len(t) == 0 to export the model + # fills previously skipped layers; checking for tensor causes errors + self.key_cache[layer_idx] = key_states + self.value_cache[layer_idx] = value_states + else: + self.key_cache[layer_idx] = torch.cat( + [self.key_cache[layer_idx], key_states], dim=-2 + ) + self.value_cache[layer_idx] = torch.cat( + [self.value_cache[layer_idx], value_states], dim=-2 + ) + return self.key_cache[layer_idx], self.value_cache[layer_idx] + + def crop(self, max_length: int): + """Crop the past key values up to a new `max_length` + in terms of tokens. `max_length` can also be + negative to remove `max_length` tokens. + This is used in assisted decoding and contrastive search. + """ + # In case it is negative + if max_length < 0: + max_length = self.get_seq_length() - abs(max_length) + + if self.get_seq_length() <= max_length: + return + + if hasattr(self, "_seen_tokens"): + self._seen_tokens = max_length + for idx in range(len(self.key_cache)): + if self.key_cache[idx].numel(): + self.key_cache[idx] = self.key_cache[idx][..., :max_length, :] + self.value_cache[idx] = self.value_cache[idx][..., :max_length, :] + + @classmethod + def from_batch_splits(cls, splits: List[DynamicCache]) -> DynamicCache: + """This is the opposite of the above `batch_split()` method. + This will be used by `stack_model_outputs` in + `generation.utils`""" + cache = cls() + for idx in range(len(splits[0])): + key_cache = [ + current.key_cache[idx] for current in splits if current.key_cache[idx].numel() + ] + value_cache = [ + current.value_cache[idx] + for current in splits + if current.value_cache[idx].numel() + ] + if key_cache != []: + layer_keys = torch.cat(key_cache, dim=0) + layer_values = torch.cat(value_cache, dim=0) + cache.update(layer_keys, layer_values, idx) + return cache + + +class patched_GenerationMixin: + """ + Applies modifications implemented in PR + `transformers/#36652 `_. + """ + + _PATCHES_ = [ + "_cache_dependant_input_preparation", + "_cache_dependant_input_preparation_exporting", + "prepare_inputs_for_generation", + ] + _PATCHED_CLASS_ = transformers.generation.utils.GenerationMixin + + def _cache_dependant_input_preparation( + self, + input_ids: torch.LongTensor, + inputs_embeds: Optional[torch.FloatTensor], + cache_position: Optional[torch.LongTensor], + ) -> Tuple[torch.FloatTensor, torch.LongTensor]: + """ + Generic cache-dependent input preparation + The code is put in a separate function to allow granular unit testing + as it needs a different implementation to be exportable. + + If we have cache: let's slice `input_ids` through `cache_position`, + to keep only the unprocessed tokens + - Exception 1: when passing input_embeds, + input_ids may be missing entries + - Exception 2: some generation methods do special slicing of input_ids, + so we don't need to do it here + - Exception 3: with synced GPUs cache_position may go out of bounds, + but we only want dummy token in that case. + - Exception 4: If input_embeds are passed then slice it through + `cache_position`, to keep only the unprocessed tokens and + generate the first token for each sequence. + Later use the generated Input ids for continuation. + + The current implementation does not rely on ``self`` and could be + a class method. It is left as a standard method to be easily rewritten. + """ + if is_torchdynamo_exporting(): + return self._cache_dependant_input_preparation_exporting( + input_ids, inputs_embeds, cache_position + ) + if inputs_embeds is not None and input_ids.shape[1] == 0: # Exception 4 + inputs_embeds = inputs_embeds[:, -cache_position.shape[0] :] + elif inputs_embeds is not None or ( # Exception 1 + cache_position[-1] >= input_ids.shape[1] + ): # Exception 3 + input_ids = input_ids[:, -cache_position.shape[0] :] + elif ( + input_ids.shape[1] != cache_position.shape[0] + ): # Default case (the "else", a no op, is Exception 2) + input_ids = input_ids[:, cache_position] + return inputs_embeds, input_ids + + def _cache_dependant_input_preparation_exporting( + self, + input_ids: torch.LongTensor, + inputs_embeds: Optional[torch.FloatTensor], + cache_position: Optional[torch.LongTensor], + ) -> Tuple[torch.FloatTensor, torch.LongTensor]: + """ + This method implements method ``_cache_dependant_input_preparation`` + with :func:`torch.cond` to make it exportable with :func:`torch.export.export`. + The code is put in a separate function to allow granular unit testing. + """ + if inputs_embeds is None: + input_ids = input_ids[:, cache_position] + else: + # This is the code we need to implemented with torch.cond. + # if input_ids.shape[1] == 0: + # inputs_embeds = inputs_embeds[:, -cache_position.shape[0] :] + # else: + # if cache_position[-1] >= input_ids.shape[1]: + # input_ids = input_ids[:, -cache_position.shape[0] :] + # else: + # if input_ids.shape[1] != cache_position.shape[0]: + # input_ids = input_ids[:, cache_position] + def branch_1(inputs_embeds, cache_position): + return inputs_embeds[:, -cache_position.shape[0] :] + + def branch_2(input_ids, cache_position): + return input_ids[:, -cache_position.shape[0] :] + + def branch_3(input_ids, cache_position): + return input_ids[:, cache_position] + + inputs_embeds, input_ids = torch.cond( + input_ids.shape[1] == 0, + ( + lambda input_ids, inputs_embeds, cache_position: ( + branch_1(inputs_embeds, cache_position), + input_ids, + ) + ), + ( + lambda input_ids, inputs_embeds, cache_position: ( + inputs_embeds, + torch.cond( + cache_position[-1] >= input_ids.shape[1], + branch_2, + lambda input_ids, cache_position: ( + torch.cond( + input_ids.shape[1] != cache_position.shape[0], + branch_3, + (lambda input_ids, cache_position: input_ids), + [input_ids, cache_position], + ) + ), + [input_ids, cache_position], + ), + ) + ), + [input_ids, inputs_embeds, cache_position], + ) + return inputs_embeds, input_ids + + def prepare_inputs_for_generation( + self, + input_ids: torch.LongTensor, + past_key_values: Optional[Cache] = None, + attention_mask: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ): + """ + Prepare the model inputs for generation. + In includes operations like computing the 4D attention mask or + slicing inputs given the existing cache. + + See the forward pass in the model documentation + for expected arguments (different models might have different + requirements for e.g. `past_key_values`). + This function should work as is for most LLMs. + """ + + # 1. Handle BC: + model_inputs = {} + # - some models don't have `Cache` support + # (which implies they don't expect `cache_position` in `forward`) + if self._supports_cache_class: + model_inputs["cache_position"] = cache_position + # - `cache_position` was not a mandatory input in + # `prepare_inputs_for_generation` for those models, and this + # function may be called outside of `generate`. + # Handle most use cases by creating `cache_position` on the fly + # (this alternative is not as robust as calling + # `generate` and letting it create `cache_position`) + elif cache_position is None: + past_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 + cache_position = torch.arange( + past_length, input_ids.shape[1], dtype=torch.long, device=input_ids.device + ) + + # 2. Generic cache-dependent input preparation + if past_key_values is not None: + model_inputs["past_key_values"] = past_key_values + inputs_embeds, input_ids = self._cache_dependant_input_preparation( + input_ids, inputs_embeds, cache_position + ) + + # 3. Prepare base model inputs + input_ids_key = "decoder_input_ids" if self.config.is_encoder_decoder else "input_ids" + # if `inputs_embeds` are passed, we only want + # to use them in the 1st generation step for every prompt. + if not self.config.is_encoder_decoder: + if inputs_embeds is not None and len(cache_position) == inputs_embeds.shape[1]: + model_inputs[input_ids_key] = None + model_inputs["inputs_embeds"] = inputs_embeds + else: + # `clone` calls in this function ensure a consistent stride. See #32227 + model_inputs[input_ids_key] = input_ids.clone( + memory_format=torch.contiguous_format + ) + model_inputs["inputs_embeds"] = None + else: + model_inputs[input_ids_key] = input_ids.clone( + memory_format=torch.contiguous_format + ) + + # 4. Create missing `position_ids` on the fly + encoder_attention_mask = attention_mask if self.config.is_encoder_decoder else None + attention_mask = ( + kwargs.pop("decoder_attention_mask", None) + if self.config.is_encoder_decoder + else attention_mask + ) + attention_mask_key = ( + "decoder_attention_mask" if self.config.is_encoder_decoder else "attention_mask" + ) + position_ids_key = ( + "decoder_position_ids" if self.config.is_encoder_decoder else "position_ids" + ) + if ( + attention_mask is not None + and kwargs.get(position_ids_key) is None + and position_ids_key in set(inspect.signature(self.forward).parameters.keys()) + ): + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + kwargs[position_ids_key] = ( + position_ids # placed in kwargs for further processing (see below) + ) + + # 5. Slice model inputs if it's an input + # that should have the same length as `input_ids` + for model_input_name in ["position_ids", "token_type_ids", "decoder_position_ids"]: + model_input = kwargs.get(model_input_name) + if model_input is not None: + if past_key_values is not None: + current_input_length = ( + model_inputs["inputs_embeds"].shape[1] + if model_inputs.get("inputs_embeds") is not None + else model_inputs[input_ids_key].shape[1] + ) + model_input = model_input[:, -current_input_length:] + model_input = model_input.clone(memory_format=torch.contiguous_format) + model_inputs[model_input_name] = model_input + + # 6. Create 4D attention mask is we are using a + # `StaticCache` (important for performant compiled forward pass) + if isinstance(past_key_values, StaticCache) and attention_mask.ndim == 2: + if model_inputs["inputs_embeds"] is not None: + batch_size, sequence_length, _ = model_inputs["inputs_embeds"].shape + device = model_inputs["inputs_embeds"].device + else: + batch_size, sequence_length = model_inputs[input_ids_key].shape + device = model_inputs[input_ids_key].device + + # Create the causal mask with fixed shape in advance, + # to reduce recompilations. If the function to create + # the 4D causal mask exists, + # it should be present in the base model (XXXModel class). + base_model = getattr(self, self.base_model_prefix, None) + if base_model is None: + causal_mask_creation_function = getattr( + self, "_prepare_4d_causal_attention_mask_with_cache_position", None + ) + else: + causal_mask_creation_function = getattr( + base_model, "_prepare_4d_causal_attention_mask_with_cache_position", None + ) + if causal_mask_creation_function is None: + pass + # logger.warning_once( + # f"{self.__class__.__name__} has no " + # "`_prepare_4d_causal_attention_mask_with_cache_position` method " + # "defined in its base modeling class. " + # "Compiled forward passes will be sub-optimal. If you're " + # "writing code, see Llama for an example implementation. " + # "If you're a user, please report this " + # "issue on GitHub." + # ) + else: + attention_mask = causal_mask_creation_function( + attention_mask, + sequence_length=sequence_length, + target_length=past_key_values.get_max_cache_shape(), + dtype=self.dtype, + device=device, + cache_position=cache_position, + batch_size=batch_size, + config=self.config, + past_key_values=past_key_values, + ) + if attention_mask is not None: + model_inputs[attention_mask_key] = attention_mask + + if encoder_attention_mask is not None: + model_inputs["attention_mask"] = encoder_attention_mask + + # 7. Forward ALL kwargs that are uninitialized (e.g. `use_cache`). + for key, value in kwargs.items(): + if key not in model_inputs: + model_inputs[key] = value + + # 8. Remove unexpected `generate` inputs (TODO @joao: fix trainer and examples) + model_inputs.pop("labels", None) + return model_inputs + + +def patched__compute_dynamic_ntk_parameters( + config: Optional[transformers.PretrainedConfig] = None, + device: Optional["torch.device"] = None, + seq_len: Optional[int] = None, + **rope_kwargs, +) -> Tuple["torch.Tensor", float]: + """ + manual patch: + ``[patch:transformers.modeling_rope_utils._compute_dynamic_ntk_parameters]`` + + Computes the inverse frequencies with NTK scaling. + Credits to the Reddit users /u/bloc97 and /u/emozilla + + Args: + config ([`~transformers.PretrainedConfig`]): + The model configuration. + device (`torch.device`): + The device to use for initialization of the inverse frequencies. + seq_len (`int`, *optional*): + The current sequence length, + used to update the dynamic RoPE at inference time. + rope_kwargs (`Dict`, *optional*): + BC compatibility with the previous + RoPE class instantiation, will be removed in v4.45. + + Returns: + Tuple of (`torch.Tensor`, `float`), + containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the + omputed cos/sin (unused in this type of RoPE). + """ + if config is not None and len(rope_kwargs) > 0: + raise ValueError( + "Unexpected arguments: `**rope_kwargs` and `config` are mutually exclusive in " + f"`_compute_dynamic_ntk_parameters`, got " + f"`rope_kwargs`={rope_kwargs} and `config`={config}" + ) + if len(rope_kwargs) > 0: + base = rope_kwargs["base"] + dim = rope_kwargs["dim"] + max_position_embeddings = rope_kwargs["max_position_embeddings"] + factor = rope_kwargs["factor"] + elif config is not None: + base = config.rope_theta + partial_rotary_factor = ( + config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0 + ) + head_dim = getattr( + config, "head_dim", config.hidden_size // config.num_attention_heads + ) + dim = int(head_dim * partial_rotary_factor) + max_position_embeddings = config.max_position_embeddings + factor = config.rope_scaling["factor"] + + attention_factor = 1.0 # Unused in this type of RoPE + + # seq_len: default to max_position_embeddings, e.g. at init time + # seq_len = seq_len if seq_len is not None and + # seq_len > max_position_embeddings else max_position_embeddings + if seq_len is None: + seq_len = max_position_embeddings + else: + torch._check(isinstance(seq_len, torch.Tensor)) + seq_len = torch.maximum( + seq_len, + torch.tensor(max_position_embeddings, dtype=seq_len.dtype, device=seq_len.device), + ) + + # Compute the inverse frequencies + base = base * ((factor * seq_len / max_position_embeddings) - (factor - 1)) ** ( + dim / (dim - 2) + ) + inv_freq = 1.0 / ( + base + ** ( + torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) + / dim + ) + ) + return inv_freq, attention_factor + + +def patched_dynamic_rope_update(rope_forward): + """manual patch: ``[patch:transformers.modeling_rope_utils.dynamic_rope_update]`` + + ``rope_type`` is determined in the constructor of class + :class:`transformers.models.phi3.modeling_phi3.Phi3RotaryEmbedding`. + + .. code-block:: python + + if hasattr(config, "rope_scaling") and config.rope_scaling is not None: + self.rope_type = config.rope_scaling.get( + "rope_type", config.rope_scaling.get("type")) + else: + self.rope_type = "default" + + The original code of the patched function: + + .. code-block:: python + + def dynamic_rope_update(rope_forward): + def longrope_frequency_update(self, position_ids, device): + seq_len = torch.max(position_ids) + 1 + if hasattr(self.config, "original_max_position_embeddings"): + original_max_position_embeddings = + self.config.original_max_position_embeddings + else: + original_max_position_embeddings = + self.config.max_position_embeddings + if seq_len > original_max_position_embeddings: + if not hasattr(self, "long_inv_freq"): + self.long_inv_freq, _ = self.rope_init_fn( + self.config, device, seq_len=original_max_position_embeddings + 1 + ) + self.register_buffer("inv_freq", self.long_inv_freq, persistent=False) + else: + self.original_inv_freq = self.original_inv_freq.to(device) + self.register_buffer("inv_freq", self.original_inv_freq, persistent=False) + + def dynamic_frequency_update(self, position_ids, device): + seq_len = torch.max(position_ids) + 1 + if seq_len > self.max_seq_len_cached: # growth + inv_freq, self.attention_scaling = self.rope_init_fn( + self.config, device, seq_len=seq_len) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.max_seq_len_cached = seq_len + + if seq_len < self.original_max_seq_len and + self.max_seq_len_cached > self.original_max_seq_len: + self.original_inv_freq = self.original_inv_freq.to(device) + self.register_buffer("inv_freq", self.original_inv_freq, persistent=False) + self.max_seq_len_cached = self.original_max_seq_len + + @wraps(rope_forward) + def wrapper(self, x, position_ids): + if "dynamic" in self.rope_type: + dynamic_frequency_update(self, position_ids, device=x.device) + elif self.rope_type == "longrope": + longrope_frequency_update(self, position_ids, device=x.device) + return rope_forward(self, x, position_ids) + + return wrapper + + """ + + def longrope_frequency_update(self, position_ids, device): + # It is no use to patch the function after the model is created + # as rope_init_fn is an attribute set to one function when the model + # is created and when no patch is applied yet. + # So we select the patched version here. + rope_init_fn = ( + patched__compute_dynamic_ntk_parameters + if self.rope_init_fn + is transformers.modeling_rope_utils._compute_dynamic_ntk_parameters + else self.rope_init_fn + ) + seq_len = torch.max(position_ids) + 1 + if hasattr(self.config, "original_max_position_embeddings"): + original_max_position_embeddings = self.config.original_max_position_embeddings + else: + original_max_position_embeddings = self.config.max_position_embeddings + # At export time, seq_len is unknown. + long_inv_freq, _ = rope_init_fn( + self.config, device, seq_len=original_max_position_embeddings + 1 + ) + original_inv_freq = self.original_inv_freq.to(device) + + cond = (seq_len > original_max_position_embeddings).item() + inv_freq = torch.cond( + cond, + (lambda x, y: x.clone()), + (lambda x, y: y.clone()), + [long_inv_freq, original_inv_freq], + ) + self.inv_freq = inv_freq + # if seq_len > original_max_position_embeddings: + # self.inv_freq = self.long_inv_freq + # else: + # self.inv_freq = self.original_inv_freq + + def dynamic_frequency_update(self, position_ids, device): + # constructor: + # - self.max_seq_len_cached = config.max_position_embeddings + # - self.original_max_seq_len = config.max_position_embeddings + # - inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device) + + # It is no use to patch the function after the model is created + # as rope_init_fn is an attribute set to one function when the model + # is created and when no patch is applied yet. + # So we select the patched version here. + rope_init_fn = ( + patched__compute_dynamic_ntk_parameters + if self.rope_init_fn + is transformers.modeling_rope_utils._compute_dynamic_ntk_parameters + else self.rope_init_fn + ) + + # This behaviour is difficult to translate. + # The sequence always grows. + # The test should always True. + # So: self.max_seq_len_cached = max(self.max_seq_len_cached, seq_len) --> seq_len + # + # if seq_len > self.max_seq_len_cached: # growth + # inv_freq, self.attention_scaling = self.rope_init_fn( + # self.config, device, seq_len=seq_len + # ) + # self.register_buffer("inv_freq", inv_freq, persistent=False) + # self.max_seq_len_cached = seq_len + # + # So we should not need what follows. + # + # cond = (seq_len > self.max_seq_len_cached).item() + # self.attention_scaling = torch.cond( + # cond, + # (lambda x, y: x.clone()), + # (lambda x, y: y.clone()), + # [attention_scaling, self.attention_scaling], + # ) + + seq_len = torch.max(position_ids) + 1 + long_inv_freq, self.attention_scaling = rope_init_fn( + self.config, device, seq_len=seq_len + ) + + # Second test to translate. + # Let's keep in mind, self.max_seq_len_cached = seq_len is likely to be True. + # But in that case the following condition is a way to restore the original cache. + + # if ( + # seq_len < self.original_max_seq_len + # and self.max_seq_len_cached > self.original_max_seq_len + # ): + # self.original_inv_freq = self.original_inv_freq.to(device) + # self.register_buffer("inv_freq", self.original_inv_freq, persistent=False) + # self.max_seq_len_cached = self.original_max_seq_len + + original_inv_freq = self.original_inv_freq.to(device) + cond = (seq_len >= self.original_max_seq_len).item() + inv_freq = torch.cond( + cond, + (lambda x, y: x.clone()), + (lambda x, y: y.clone()), + [long_inv_freq, original_inv_freq], + ) + self.inv_freq = inv_freq + + @wraps(rope_forward) + def wrapper(self, x, position_ids): + if "dynamic" in self.rope_type: + dynamic_frequency_update(self, position_ids, device=x.device) + elif self.rope_type == "longrope": + longrope_frequency_update(self, position_ids, device=x.device) + return rope_forward(self, x, position_ids) + + return wrapper + + +def common_eager_attention_forward( + module: torch.nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: Optional[float] = None, + dropout: float = 0.0, + head_mask: Optional[torch.Tensor] = None, + **kwargs, +): + if scaling is None: + scaling = query.size(-1) ** -0.5 + + attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling + if attention_mask is not None: + # The two following lines were added. + if attention_mask is not None and attention_mask.ndim == 4: + attention_mask = attention_mask[:, :, :, : key.shape[-2]] + attn_weights = attn_weights + attention_mask + + attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1) + + if head_mask is not None: + attn_weights = attn_weights * head_mask.view(1, -1, 1, 1) + + attn_weights = torch.nn.functional.dropout( + attn_weights, p=dropout, training=module.training + ) + attn_output = torch.matmul(attn_weights, value) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +def patched_model_bart_eager_attention_forward( + module: torch.nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: Optional[float] = None, + dropout: float = 0.0, + head_mask: Optional[torch.Tensor] = None, + **kwargs, +): + """[patch:transformers.models.bart.modeling_bart.eager_attention_forward]""" + return common_eager_attention_forward( + module, + query, + key, + value, + attention_mask=attention_mask, + scaling=scaling, + dropout=dropout, + head_mask=head_mask, + **kwargs, + ) + + +def patched_modeling_marian_eager_attention_forward( + module: torch.nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: Optional[float] = None, + dropout: float = 0.0, + head_mask: Optional[torch.Tensor] = None, + **kwargs, +): + """[patch:transformers.models.marian.modeling_marian.eager_attention_forward]""" + return common_eager_attention_forward( + module, + query, + key, + value, + attention_mask=attention_mask, + scaling=scaling, + dropout=dropout, + head_mask=head_mask, + **kwargs, + ) + + +class common_RotaryEmbedding(torch.nn.Module): + @torch.no_grad() + @patched_dynamic_rope_update + def forward(self, x, position_ids): + inv_freq_expanded = ( + self.inv_freq[None, :, None] + .float() + .expand(position_ids.shape[0], -1, 1) + .to(x.device) + ) + position_ids_expanded = position_ids[:, None, :].float() + + device_type = ( + x.device.type + if isinstance(x.device.type, str) and x.device.type != "mps" + else "cpu" + ) + with torch.autocast(device_type=device_type, enabled=False): # Force float32 + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class patched_GemmaRotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding + + +if pv.Version(transformers.__version__) >= pv.Version("4.52"): + + class patched_Gemma2RotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.gemma2.modeling_gemma2.Gemma2RotaryEmbedding + + class patched_Gemma3RotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.gemma3.modeling_gemma3.Gemma3RotaryEmbedding + + +class patched_LlamaRotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.llama.modeling_llama.LlamaRotaryEmbedding + + +class patched_MistralRotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding + + +class patched_MixtralRotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.mixtral.modeling_mixtral.MixtralRotaryEmbedding + + +class patched_PhiRotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.phi.modeling_phi.PhiRotaryEmbedding + + +if pv.Version(transformers.__version__) >= pv.Version("4.51"): + + class patched_Phi3RotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.phi3.modeling_phi3.Phi3RotaryEmbedding + + +if pv.Version(transformers.__version__) >= pv.Version("4.52"): + + class patched_Phi4MultimodalRotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = ( + transformers.models.phi4_multimodal.modeling_phi4_multimodal.Phi4MultimodalRotaryEmbedding + ) + + +if pv.Version(transformers.__version__) >= pv.Version("4.53"): + + class patched_SmolLM3RotaryEmbedding(common_RotaryEmbedding): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.smollm3.modeling_smollm3.SmolLM3RotaryEmbedding + + +class patched_IdeficsEmbedding(torch.nn.Module): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.idefics.modeling_idefics.IdeficsEmbedding + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + # if seq_len > self.max_seq_len_cached: + # self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype) + + def _set_cos_sin_cache_then(x, inv_freq, seq_len, _cos_cached, _sin_cached): + t = torch.arange(seq_len, device=x.device, dtype=torch.int64).type_as(inv_freq) + freqs = torch.einsum("i,j->ij", t, inv_freq) + emb = torch.cat((freqs, freqs), dim=-1) + return emb.cos().to(x.dtype), emb.sin().to(x.dtype) + + def _set_cos_sin_cache_else(_x, _inv_freq, _seq_len, cos_cached, sin_cached): + torch._check(seq_len.item() <= cos_cached.shape[0]) + co = cos_cached[: seq_len.item()].detach().clone() + torch._check(seq_len.item() <= sin_cached.shape[0]) + si = sin_cached[: seq_len.item()].detach().clone() + return co.to(dtype=x.dtype), si.to(dtype=x.dtype) + + cos_cached, sin_cached = torch.cond( + (seq_len > self.max_seq_len_cached).item(), + _set_cos_sin_cache_then, + _set_cos_sin_cache_else, + [x, self.inv_freq, seq_len, self.cos_cached, self.sin_cached], + ) + return cos_cached, sin_cached + + +class patched_IdeficsAttention(torch.nn.Module): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.idefics.modeling_idefics.IdeficsAttention + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # if key_value_states are provided this layer is used as a cross-attention layer + is_cross_attention = self.is_cross_attention or key_value_states is not None + + bsz, q_len, _ = hidden_states.size() + + query_states = ( + self.q_proj(hidden_states) + .view(bsz, q_len, self.num_heads, self.head_dim) + .transpose(1, 2) + ) + if not is_cross_attention: + key_states = ( + self.k_proj(hidden_states) + .view(bsz, q_len, self.num_heads, self.head_dim) + .transpose(1, 2) + ) + value_states = ( + self.v_proj(hidden_states) + .view(bsz, q_len, self.num_heads, self.head_dim) + .transpose(1, 2) + ) + else: + _, kv_len, _ = ( + key_value_states.size() + ) # Note that, in this case, `kv_len` == `kv_seq_len` + key_states = ( + self.k_proj(key_value_states) + .view(bsz, kv_len, self.num_heads, self.head_dim) + .transpose(1, 2) + ) + value_states = ( + self.v_proj(key_value_states) + .view(bsz, kv_len, self.num_heads, self.head_dim) + .transpose(1, 2) + ) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += cache_position[0] + + if not is_cross_attention: + rotary_length = torch.maximum( + torch.tensor(kv_seq_len, dtype=torch.int64), + torch.tensor(q_len, dtype=torch.int64), + ) + cos, sin = self.rotary_emb(value_states, seq_len=rotary_length) + query_states, key_states = ( + transformers.models.idefics.modeling_idefics.apply_rotary_pos_emb( + query_states, key_states, cos, sin, position_ids + ) + ) + # [bsz, nh, t, hd] + + if past_key_value is not None: + # sin and cos are specific to RoPE models; + # cache_position needed for the static cache + cache_kwargs = {"cache_position": cache_position} + key_states, value_states = past_key_value.update( + key_states, value_states, self.layer_idx, cache_kwargs + ) + + if self.qk_layer_norms: + query_states = self.q_layer_norm(query_states) + key_states = self.k_layer_norm(key_states) + + attention_interface: Callable = ( + transformers.models.idefics.modeling_idefics.eager_attention_forward + ) + + if self.config._attn_implementation != "eager": + if self.config._attn_implementation == "sdpa" and output_attentions: + transformers.models.idefics.modeling_idefics.logger.warning_once( + "`torch.nn.functional.scaled_dot_product_attention` does not support " + "`output_attentions=True`. Falling back to " + "eager attention. This warning can be removed using the argument " + '`attn_implementation="eager"` when loading the model.' + ) + else: + attention_interface = transformers.modeling_utils.ALL_ATTENTION_FUNCTIONS[ + self.config._attn_implementation + ] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + + if output_attentions: + attn_weights = None + + if pv.Version(transformers.__version__) < pv.Version("4.53.99"): + return attn_output, attn_weights, past_key_value + return attn_output, attn_weights + + +class patched_SamMaskDecoder(torch.nn.Module): + _PATCHES_ = ["forward"] + _PATCHED_CLASS_ = transformers.models.sam.modeling_sam.SamMaskDecoder + + def forward( + self, + image_embeddings: torch.Tensor, + image_positional_embeddings: torch.Tensor, + sparse_prompt_embeddings: torch.Tensor, + dense_prompt_embeddings: torch.Tensor, + multimask_output: bool, + output_attentions: Optional[bool] = None, + attention_similarity: Optional[torch.Tensor] = None, + target_embedding: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Predict masks given image and prompt embeddings. + + Args: + image_embeddings (`torch.Tensor`): + the embeddings from the image encoder + image_positional_embedding (`torch.Tensor`): + positional encoding with the shape of image_embeddings + sparse_prompt_embeddings (`torch.Tensor`): + The embeddings of the points and boxes + dense_prompt_embeddings (`torch.Tensor`): + the embeddings of the mask inputs + multimask_output (bool): + Whether to return multiple masks or a single mask. + output_attentions (bool, *optional*): + Whether or not to return the attentions tensors of all attention layers. + """ + batch_size, num_channels, height, width = image_embeddings.shape + point_batch_size = sparse_prompt_embeddings.shape[1] + # Concatenate output tokens + output_tokens = torch.cat([self.iou_token.weight, self.mask_tokens.weight], dim=0) + output_tokens = output_tokens.repeat(batch_size, point_batch_size, 1, 1) + + # torch.cond rewrites the if-else logic to handle empty sparse_prompt_embeddings + # torch.any is needed to avoid data-dependent control flow + # with sparse_prompt_embeddings.sum().item() != 0 + def sparse_prompt_embeddings_is_not_empty(output_tokens, sparse_prompt_embeddings): + return torch.cat((output_tokens, sparse_prompt_embeddings), dim=2) + + def sparse_prompt_embeddings_is_empty(output_tokens, sparse_prompt_embeddings): + return output_tokens.clone() + + tokens = torch.cond( + torch.any(sparse_prompt_embeddings != 0), + sparse_prompt_embeddings_is_not_empty, + sparse_prompt_embeddings_is_empty, + [output_tokens, sparse_prompt_embeddings], + ) + + point_embeddings = tokens.to(self.iou_token.weight.dtype) + + # Expand per-image data in batch direction to be per-point + image_embeddings = image_embeddings + dense_prompt_embeddings + image_embeddings = image_embeddings.repeat_interleave(point_batch_size, 0) + image_positional_embeddings = image_positional_embeddings.repeat_interleave( + point_batch_size, 0 + ) + + # Run the transformer, image_positional_embedding are consumed + point_embedding, image_embeddings, attentions = self.transformer( + point_embeddings=point_embeddings, + image_embeddings=image_embeddings, + image_positional_embeddings=image_positional_embeddings, + attention_similarity=attention_similarity, + target_embedding=target_embedding, + output_attentions=output_attentions, + ) + iou_token_out = torch.select(point_embedding, dim=2, index=0) + mask_tokens_out = torch.narrow( + point_embedding, dim=2, start=1, length=self.num_mask_tokens + ) + + # Upscale mask embeddings and predict masks using the mask tokens + image_embeddings = image_embeddings.transpose(2, 3).reshape( + batch_size * point_batch_size, num_channels, height, width + ) + + upscaled_embedding = self.upscale_conv1(image_embeddings) + upscaled_embedding = self.activation(self.upscale_layer_norm(upscaled_embedding)) + upscaled_embedding = self.activation(self.upscale_conv2(upscaled_embedding)) + + hyper_in_list = [] + for i in range(self.num_mask_tokens): + current_mlp = self.output_hypernetworks_mlps[i] + hyper_in_list += [current_mlp(mask_tokens_out[:, :, i, :])] + hyper_in = torch.stack(hyper_in_list, dim=2) + + _, num_channels, height, width = upscaled_embedding.shape + upscaled_embedding = upscaled_embedding.reshape( + batch_size, point_batch_size, num_channels, height * width + ) + masks = (hyper_in @ upscaled_embedding).reshape( + batch_size, point_batch_size, -1, height, width + ) + + # Generate mask quality predictions + iou_pred = self.iou_prediction_head(iou_token_out) + + # Select the correct mask or masks for output + if multimask_output: + mask_slice = slice(1, None) + else: + mask_slice = slice(0, 1) + masks = masks[:, :, mask_slice, :, :] + iou_pred = iou_pred[:, :, mask_slice] + + outputs = (masks, iou_pred) + + if output_attentions: + outputs = outputs + (attentions,) # noqa: RUF005 + else: + outputs = outputs + (None,) # noqa: RUF005 + + return outputs diff --git a/optimum/torch_export_patches/_treenode_registry.py b/optimum/torch_export_patches/_treenode_registry.py new file mode 100644 index 00000000..50128f06 --- /dev/null +++ b/optimum/torch_export_patches/_treenode_registry.py @@ -0,0 +1,276 @@ +import pprint +from typing import Any, Callable + +import optree +import torch +import transformers +from packaging import version as pv +from transformers import cache_utils + + +try: + from transformers.models.mamba.modeling_mamba import MambaCache +except ImportError: + from transformers.cache_utils import MambaCache + +from .serialization import _lower_name_with_ + + +PATCH_OF_PATCHES: set[Any] = set() + + +def register_class_serialization( + cls, + f_flatten: Callable, + f_unflatten: Callable, + f_flatten_with_keys: Callable, + f_check: Callable | None = None, + verbose: int = 0, +) -> bool: + """Registers a class. + + It can be undone with + :func:`onnx_diagnostic.torch_export_patches.onnx_export_serialization.unregister_class_serialization`. + + Args: + cls: class to register + f_flatten: see ``torch.utils._pytree.register_pytree_node`` + f_unflatten: see ``torch.utils._pytree.register_pytree_node`` + f_flatten_with_keys: see ``torch.utils._pytree.register_pytree_node`` + f_check: called to check the registration was successful + verbose: verbosity + + Returns: + registered or not + """ + if cls is not None and cls in torch.utils._pytree.SUPPORTED_NODES: + if verbose and cls is not None: + print(f"[register_class_serialization] already registered {cls.__name__}") + return False + + if verbose: + print(f"[register_class_serialization] ---------- register {cls.__name__}") + torch.utils._pytree.register_pytree_node( + cls, + f_flatten, + f_unflatten, + serialized_type_name=f"{cls.__module__}.{cls.__name__}", + flatten_with_keys_fn=f_flatten_with_keys, + ) + if pv.Version(torch.__version__) < pv.Version("2.7"): + if verbose: + print(f"[register_class_serialization] ---------- register {cls.__name__} for torch=={torch.__version__}") + torch.fx._pytree.register_pytree_flatten_spec(cls, lambda x, _: f_flatten(x)[0]) + + # check + if f_check: + raise NotImplementedError( + "The check function is not implemented. Please provide a valid check function to verify the registration." + ) + # inst = f_check() + # values, spec = torch.utils._pytree.tree_flatten(inst) + # restored = torch.utils._pytree.tree_unflatten(values, spec) + # assert string_type(inst, with_shape=True) == string_type(restored, with_shape=True), ( + # f"Issue with registration of class {cls} " + # f"inst={string_type(inst, with_shape=True)}, " + # f"restored={string_type(restored, with_shape=True)}" + # ) + return True + + +def register_cache_serialization( + patch_transformers: bool = False, patch_diffusers: bool = True, verbose: int = 0 +) -> dict[str, bool]: + """Registers many classes with register_class_serialization. + + Returns information needed to undo the registration. + + Args: + patch_transformers: Add serialization function for transformers package. + patch_diffusers: Add serialization function for diffusers package. + verbose: Verbosity level. + + Returns: + Information to unpatch. + """ + wrong: dict[type, str | None] = {} + if patch_transformers: + from .serialization.transformers_impl import WRONG_REGISTRATIONS + + wrong |= WRONG_REGISTRATIONS + if patch_diffusers: + from .serialization.diffusers_impl import WRONG_REGISTRATIONS + + wrong |= WRONG_REGISTRATIONS + + registration_functions = serialization_functions( + patch_transformers=patch_transformers, patch_diffusers=patch_diffusers, verbose=verbose + ) + + # DynamicCache serialization is different in transformers and does not + # play way with torch.export.export. + # see test test_export_dynamic_cache_cat with NOBYPASS=1 + # :: NOBYBASS=1 python _unittests/ut_torch_export_patches/test_dynamic_class.py -k e_c + # This is caused by this line: + # torch.fx._pytree.register_pytree_flatten_spec( + # DynamicCache, _flatten_dynamic_cache_for_fx) + # so we remove it anyway + # BaseModelOutput serialization is incomplete. + # It does not include dynamic shapes mapping. + for cls, version in wrong.items(): + if ( + cls in torch.utils._pytree.SUPPORTED_NODES + and cls not in PATCH_OF_PATCHES + # and pv.Version(torch.__version__) < pv.Version("2.7") + and (version is None or pv.Version(transformers.__version__) >= pv.Version(version)) + ): + assert cls in registration_functions, ( + f"{cls} has no registration functions mapped to it, " + f"available options are {list(registration_functions)}" + ) + if verbose: + print(f"[_fix_registration] {cls.__name__} is unregistered and registered first") + unregister_class_serialization(cls, verbose=verbose) + registration_functions[cls](verbose=verbose) # type: ignore[arg-type, call-arg] + if verbose: + print(f"[_fix_registration] {cls.__name__} done.") + # To avoid doing it multiple times. + PATCH_OF_PATCHES.add(cls) + + # classes with no registration at all. + done = {} + for k, v in registration_functions.items(): + done[k] = v(verbose=verbose) # type: ignore[arg-type, call-arg] + return done + + +def serialization_functions( + patch_transformers: bool = False, patch_diffusers: bool = False, verbose: int = 0 +) -> dict[type, Callable[[int], bool]]: + """Returns the list of serialization functions.""" + supported_classes: set[type] = set() + classes: dict[type, Callable[[int], bool]] = {} + all_functions: dict[type, str | None] = {} + + if patch_transformers: + from .serialization.transformers_impl import ( + SUPPORTED_DATACLASSES, + flatten_dynamic_cache, + flatten_encoder_decoder_cache, + flatten_mamba_cache, + flatten_sliding_window_cache, + flatten_static_cache, + flatten_with_keys_dynamic_cache, + flatten_with_keys_encoder_decoder_cache, + flatten_with_keys_mamba_cache, + flatten_with_keys_sliding_window_cache, + flatten_with_keys_static_cache, + unflatten_dynamic_cache, + unflatten_encoder_decoder_cache, + unflatten_mamba_cache, + unflatten_sliding_window_cache, + unflatten_static_cache, + ) + from .serialization.transformers_impl import ( + __dict__ as dtr, + ) + + all_functions.update(dtr) + supported_classes |= SUPPORTED_DATACLASSES + + transformers_classes = { + cache_utils.DynamicCache: lambda verbose=verbose: register_class_serialization( + cache_utils.DynamicCache, + flatten_dynamic_cache, + unflatten_dynamic_cache, + flatten_with_keys_dynamic_cache, + # f_check=make_dynamic_cache([(torch.rand((4, 4, 4)), torch.rand((4, 4, 4)))]), + verbose=verbose, + ), + MambaCache: lambda verbose=verbose: register_class_serialization( + MambaCache, + flatten_mamba_cache, + unflatten_mamba_cache, + flatten_with_keys_mamba_cache, + verbose=verbose, + ), + cache_utils.EncoderDecoderCache: lambda verbose=verbose: register_class_serialization( + cache_utils.EncoderDecoderCache, + flatten_encoder_decoder_cache, + unflatten_encoder_decoder_cache, + flatten_with_keys_encoder_decoder_cache, + verbose=verbose, + ), + cache_utils.SlidingWindowCache: lambda verbose=verbose: register_class_serialization( + cache_utils.SlidingWindowCache, + flatten_sliding_window_cache, + unflatten_sliding_window_cache, + flatten_with_keys_sliding_window_cache, + verbose=verbose, + ), + cache_utils.StaticCache: lambda verbose=verbose: register_class_serialization( + cache_utils.StaticCache, + flatten_static_cache, + unflatten_static_cache, + flatten_with_keys_static_cache, + verbose=verbose, + ), + } + classes.update(transformers_classes) + + if patch_diffusers: + from .serialization.diffusers_impl import SUPPORTED_DATACLASSES + from .serialization.diffusers_impl import __dict__ as dfu + + all_functions.update(dfu) + supported_classes |= SUPPORTED_DATACLASSES + + for cls in supported_classes: + lname = _lower_name_with_(cls.__name__) + assert f"flatten_{lname}" in all_functions, ( + f"Unable to find function 'flatten_{lname}' in {list(all_functions)}" + ) + classes[cls] = lambda verbose=verbose, _ln=lname, cls=cls, _al=all_functions: register_class_serialization( + cls, + _al[f"flatten_{_ln}"], + _al[f"unflatten_{_ln}"], + _al[f"flatten_with_keys_{_ln}"], + verbose=verbose, + ) + return classes + + +def unregister_class_serialization(cls: type, verbose: int = 0): + """Undo the registration.""" + # torch.utils._pytree._deregister_pytree_flatten_spec(cls) + if cls in torch.fx._pytree.SUPPORTED_NODES: + del torch.fx._pytree.SUPPORTED_NODES[cls] + if cls in torch.fx._pytree.SUPPORTED_NODES_EXACT_MATCH: + del torch.fx._pytree.SUPPORTED_NODES_EXACT_MATCH[cls] + if hasattr(torch.utils._pytree, "_deregister_pytree_node"): + # torch >= 2.7 + torch.utils._pytree._deregister_pytree_node(cls) + else: + if cls in torch.utils._pytree.SUPPORTED_NODES: + del torch.utils._pytree.SUPPORTED_NODES[cls] + optree.unregister_pytree_node(cls, namespace="torch") + if cls in torch.utils._pytree.SUPPORTED_NODES: + import packaging.version as pv + + if pv.Version(torch.__version__) < pv.Version("2.7.0"): + del torch.utils._pytree.SUPPORTED_NODES[cls] + assert cls not in torch.utils._pytree.SUPPORTED_NODES, ( + f"{cls} was not successful unregistered " + f"from torch.utils._pytree.SUPPORTED_NODES=" + f"{pprint.pformat(list(torch.utils._pytree.SUPPORTED_NODES))}" + ) + if verbose: + print(f"[unregister_cache_serialization] unregistered {cls.__name__}") + + +def unregister_cache_serialization(undo: dict[str, bool], verbose: int = 0): + """Undo all registrations.""" + cls_ensemble = {MambaCache, cache_utils.DynamicCache, cache_utils.EncoderDecoderCache} | set(undo) + for cls in cls_ensemble: + if undo.get(cls.__name__, False): + unregister_class_serialization(cls, verbose) diff --git a/pyproject.toml b/pyproject.toml index 8ad8da8c..71ae2bb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,10 @@ version = {attr = "optimum.onnx.version.__version__"} [tool.ruff] line-length = 119 target-version = "py39" +exclude = [ + "optimum/torch_export_patches/_patch_transformers.py", + "optimum/torch_export_patches/_patch_torch.py", +] [tool.ruff.lint] ignore = [ From 70db82cca3b8c2250b3b382cb30d6e0a35b19401 Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Fri, 25 Jul 2025 23:25:01 +0000 Subject: [PATCH 02/10] add tests and organize serialization files --- .github/workflows/test_patch.yml | 43 +++ optimum/torch_export_patches/_core.py | 27 +- .../_patch_transformers.py | 26 +- optimum/torch_export_patches/_treenode.py | 305 ++++++++++++++++++ .../_treenode_registry.py | 27 +- tests/patch/test_torch_export_patches.py | 138 ++++++++ 6 files changed, 551 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/test_patch.yml create mode 100644 optimum/torch_export_patches/_treenode.py create mode 100644 tests/patch/test_torch_export_patches.py diff --git a/.github/workflows/test_patch.yml b/.github/workflows/test_patch.yml new file mode 100644 index 00000000..2d8391ee --- /dev/null +++ b/.github/workflows/test_patch.yml @@ -0,0 +1,43 @@ +name: ONNX / Python - Test + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + TRANSFORMERS_IS_CI: true + +jobs: + build: + strategy: + fail-fast: false + matrix: + python-version: [3.9] + runs-on: [ubuntu-22.04] + + runs-on: ${{ matrix.runs-on }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + pip install .[tests,onnxruntime] diffusers + + - name: Test with pytest + run: | + pytest tests/patch/test_torch_export_patches.py -n auto -vvvv --durations=0 diff --git a/optimum/torch_export_patches/_core.py b/optimum/torch_export_patches/_core.py index 064431ad..ca16a61d 100644 --- a/optimum/torch_export_patches/_core.py +++ b/optimum/torch_export_patches/_core.py @@ -5,11 +5,10 @@ from collections.abc import Generator from typing import Any, Callable -import _patch_torch -import _patch_transformers -import _treenode_registry import torch +from optimum.torch_export_patches import _patch_torch, _patch_transformers, _treenode_registry + def patched_function(name: str) -> tuple[type, Callable]: """Returns the module and the function needed to be patched.""" @@ -289,6 +288,7 @@ def torch_export_patches( # patch sympy ############# + # TODO (anyone): Is this still needed? if patch_sympy: import sympy @@ -415,6 +415,8 @@ def torch_export_patches( # export ######## + fct_callable = _replacement_before_exporting if patch_transformers else (lambda x: x) + if verbose: print("[torch_export_patches] done patching") @@ -525,3 +527,22 @@ def torch_export_patches( ######## _treenode_registry.unregister_cache_serialization(cache_done, verbose=verbose) + + +def _replacement_before_exporting(args: Any) -> Any: + """Does replacements on the given inputs if needed.""" + if args is None: + return None + if isinstance(args, (int, float)): + return args + if type(args) not in {dict, tuple, list}: + # BaseModelOutput is a dict + return args + if isinstance(args, dict): + return {k: _replacement_before_exporting(v) for k, v in args.items()} + if isinstance(args, tuple): + return tuple(_replacement_before_exporting(v) for v in args) + if isinstance(args, list): + return [_replacement_before_exporting(v) for v in args] + + return args diff --git a/optimum/torch_export_patches/_patch_transformers.py b/optimum/torch_export_patches/_patch_transformers.py index 50711283..6220dfa9 100644 --- a/optimum/torch_export_patches/_patch_transformers.py +++ b/optimum/torch_export_patches/_patch_transformers.py @@ -15,9 +15,27 @@ except ImportError: patch_masking_utils = False -from ...ext_test_case import has_transformers -from ...helpers.torch_helper import is_torchdynamo_exporting +def _has_transformers(version: str) -> bool: + "Returns True if transformers version is higher." + return pv.Version(transformers.__version__) >= pv.Version(version) + +def _is_torchdynamo_exporting() -> bool: + """Tells if :epkg:`torch` is exporting a model. Relies on ``torch.compiler.is_exporting()``.""" + + if not hasattr(torch.compiler, "is_exporting"): + # torch.compiler.is_exporting requires torch>=2.7 + return False + + try: + return torch.compiler.is_exporting() + except Exception: + try: + import torch._dynamo as dynamo + + return dynamo.is_exporting() # type: ignore + except Exception: + return False if patch_masking_utils: # Introduced in 4.52 @@ -153,7 +171,7 @@ class patched_AttentionMaskConverter: """ # This method was fixed in 4.51 at least. - _PATCHES_ = ["_make_causal_mask"] if not has_transformers("4.48.3") else [] + _PATCHES_ = ["_make_causal_mask"] if not _has_transformers("4.48.3") else [] _PATCHED_CLASS_ = AttentionMaskConverter @staticmethod @@ -364,7 +382,7 @@ def _cache_dependant_input_preparation( The current implementation does not rely on ``self`` and could be a class method. It is left as a standard method to be easily rewritten. """ - if is_torchdynamo_exporting(): + if _is_torchdynamo_exporting(): return self._cache_dependant_input_preparation_exporting( input_ids, inputs_embeds, cache_position ) diff --git a/optimum/torch_export_patches/_treenode.py b/optimum/torch_export_patches/_treenode.py new file mode 100644 index 00000000..6d5ae4ac --- /dev/null +++ b/optimum/torch_export_patches/_treenode.py @@ -0,0 +1,305 @@ +import re +from typing import Any, Callable + +import torch +from transformers.cache_utils import ( + DynamicCache, + EncoderDecoderCache, + HybridCache, + SlidingWindowCache, + StaticCache, +) + + +try: + from transformers.models.mamba.modeling_mamba import MambaCache +except ImportError: + from transformers.cache_utils import MambaCache +from transformers.modeling_outputs import BaseModelOutput + +from ...helpers.cache_helper import ( + CacheKeyValue, + make_dynamic_cache, + make_hybrid_cache, + make_sliding_window_cache, + make_static_cache, +) + + +SUPPORTED_DATACLASSES: set[type] = set() +WRONG_REGISTRATIONS = { + DynamicCache: "4.50", + BaseModelOutput: None, +} + + +############ +# MambaCache +############ + + +def flatten_mamba_cache( + mamba_cache: MambaCache, +) -> tuple[list[Any], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.MambaCache` with python objects.""" + assert isinstance(mamba_cache.conv_states, list) and isinstance(mamba_cache.ssm_states, list), ( + f"Unexpected types for conv_states and ssm_states {type(mamba_cache.conv_states)}, " + f"{type(mamba_cache.ssm_states)}" + ) + flat = [ + ("conv_states", mamba_cache.conv_states), + ("ssm_states", mamba_cache.ssm_states), + ] + return [f[1] for f in flat], [f[0] for f in flat] + + +def unflatten_mamba_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> MambaCache: + """Restores a :class:`transformers.cache_utils.MambaCache` from python objects.""" + conv_states, ssm_states = values + + class Config: + def __init__(self): + if isinstance(conv_states, list): + self.intermediate_size = conv_states[0].shape[1] + self.state_size = ssm_states[0].shape[2] + self.conv_kernel = conv_states[0].shape[2] + self.num_hidden_layers = len(conv_states) + else: + self.intermediate_size = conv_states.shape[2] + self.state_size = ssm_states.shape[3] + self.conv_kernel = conv_states.shape[3] + self.num_hidden_layers = conv_states.shape[0] + + cache = MambaCache( + Config(), + max_batch_size=1, + dtype=values[-1][0].dtype, + device="cpu" if values[-1][0].get_device() < 0 else "cuda", + ) + values = dict(zip(context, values)) + for k, v in values.items(): + setattr(cache, k, v) + return cache + + +def flatten_with_keys_mamba_cache( + cache: MambaCache, +) -> tuple[ + list[tuple[torch.utils._pytree.KeyEntry, Any]], + torch.utils._pytree.Context, +]: + """Serializes a :class:`transformers.cache_utils.MambaCache` with python objects.""" + values, context = flatten_mamba_cache(cache) + return [(torch.utils._pytree.MappingKey(k), v) for k, v in zip(context, values)], context + + +############## +# DynamicCache +############## + + +def flatten_dynamic_cache( + dynamic_cache: DynamicCache, +) -> tuple[list[Any], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.DynamicCache` with python objects.""" + ca = CacheKeyValue(dynamic_cache) + flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] + return [f[1] for f in flat], [f[0] for f in flat] + + +def flatten_with_keys_dynamic_cache( + dynamic_cache: DynamicCache, +) -> tuple[list[tuple[torch.utils._pytree.KeyEntry, Any]], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.DynamicCache` with python objects.""" + values, context = flatten_dynamic_cache(dynamic_cache) + return [(torch.utils._pytree.MappingKey(k), v) for k, v in zip(context, values)], context + + +def unflatten_dynamic_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> DynamicCache: + """Restores a :class:`transformers.cache_utils.DynamicCache` from python objects.""" + return make_dynamic_cache(list(zip(values[0], values[1]))) + + +############# +# HybridCache +############# + + +def flatten_hybrid_cache( + cache: HybridCache, +) -> tuple[list[Any], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.HybridCache` with python objects.""" + ca = CacheKeyValue(cache) + flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] + return [f[1] for f in flat], [f[0] for f in flat] + + +def flatten_with_keys_hybrid_cache( + cache: HybridCache, +) -> tuple[list[tuple[torch.utils._pytree.KeyEntry, Any]], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.HybridCache` with python objects.""" + values, context = flatten_hybrid_cache(cache) + return [(torch.utils._pytree.MappingKey(k), v) for k, v in zip(context, values)], context + + +def unflatten_hybrid_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> HybridCache: + """Restores a :class:`transformers.cache_utils.HybridCache` from python objects.""" + return make_hybrid_cache(list(zip(values[0], values[1]))) + + +############# +# StaticCache +############# + + +def flatten_static_cache( + cache: StaticCache, +) -> tuple[list[Any], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.StaticCache` with python objects.""" + ca = CacheKeyValue(cache) + assert not ca.key_cache or cache.max_cache_len == ca.key_cache[0].shape[2], ( + f"Serialization doet not work when " + f"cache.max_cache_len={cache.max_cache_len} != " + f"cache.key_cache[0].shape[2]={ca.keu_cache[0].shape[2]}" + ) + flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] + return [f[1] for f in flat], [f[0] for f in flat] + + +def flatten_with_keys_static_cache( + cache: StaticCache, +) -> tuple[list[tuple[torch.utils._pytree.KeyEntry, Any]], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.StaticCache` with python objects.""" + values, context = flatten_static_cache(cache) + return [(torch.utils._pytree.MappingKey(k), v) for k, v in zip(context, values)], context + + +def unflatten_static_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> StaticCache: + """Restores a :class:`transformers.cache_utils.StaticCache` from python objects.""" + return make_static_cache(list(zip(values[0], values[1])), max_cache_len=values[0][0].shape[2]) + + +#################### +# SlidingWindowCache +#################### + + +def flatten_sliding_window_cache( + cache: SlidingWindowCache, +) -> tuple[list[Any], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.SlidingWindowCache` + with python objects. + """ + ca = CacheKeyValue(cache) + flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] + return [f[1] for f in flat], [f[0] for f in flat] + + +def flatten_with_keys_sliding_window_cache( + cache: SlidingWindowCache, +) -> tuple[list[tuple[torch.utils._pytree.KeyEntry, Any]], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.SlidingWindowCache` + with python objects. + """ + values, context = flatten_sliding_window_cache(cache) + return [(torch.utils._pytree.MappingKey(k), v) for k, v in zip(context, values)], context + + +def unflatten_sliding_window_cache( + values: list[Any], context: torch.utils._pytree.Context, output_type=None +) -> SlidingWindowCache: + """Restores a :class:`transformers.cache_utils.SlidingWindowCache` from python objects.""" + key_cache, value_cache = values + return make_sliding_window_cache(list(zip(values[0], values[1]))) + + +##################### +# EncoderDecoderCache +##################### + + +def flatten_encoder_decoder_cache( + ec_cache: EncoderDecoderCache, +) -> tuple[list[Any], torch.utils._pytree.Context]: + """Serializes a :class:`transformers.cache_utils.EncoderDecoderCache` + with python objects. + """ + dictionary = { + "self_attention_cache": ec_cache.self_attention_cache, + "cross_attention_cache": ec_cache.cross_attention_cache, + } + return torch.utils._pytree._dict_flatten(dictionary) + + +def flatten_with_keys_encoder_decoder_cache( + ec_cache: EncoderDecoderCache, +) -> tuple[ + list[tuple[torch.utils._pytree.KeyEntry, Any]], + torch.utils._pytree.Context, +]: + """Serializes a :class:`transformers.cache_utils.EncoderDecoderCache` + with python objects. + """ + dictionary = { + "self_attention_cache": ec_cache.self_attention_cache, + "cross_attention_cache": ec_cache.cross_attention_cache, + } + return torch.utils._pytree._dict_flatten_with_keys(dictionary) + + +def unflatten_encoder_decoder_cache( + values: list[Any], context: torch.utils._pytree.Context, output_type=None +) -> EncoderDecoderCache: + """Restores a :class:`transformers.cache_utils.EncoderDecoderCache` from python objects.""" + dictionary = torch.utils._pytree._dict_unflatten(values, context) + return EncoderDecoderCache(**dictionary) + + +############# +# dataclasses +############# + + +( + flatten_base_model_output, + flatten_with_keys_base_model_output, + unflatten_base_model_output, +) = _make_serialization_function_for_dataclass(BaseModelOutput, SUPPORTED_DATACLASSES) + + +def _make_serialization_function_for_dataclass( + cls: type, supported_classes: set[type] +) -> tuple[Callable, Callable, Callable]: + """Automatically creates serialization function for a class decorated with + ``dataclasses.dataclass``. + """ + + def flatten_cls(obj: cls) -> tuple[list[Any], torch.utils._pytree.Context]: # type: ignore[valid-type] + """Serializes a ``%s`` with python objects.""" + return list(obj.values()), list(obj.keys()) + + def flatten_with_keys_cls( + obj: cls, # type: ignore[valid-type] + ) -> tuple[list[tuple[torch.utils._pytree.KeyEntry, Any]], torch.utils._pytree.Context]: + """Serializes a ``%s`` with python objects with keys.""" + values, context = list(obj.values()), list(obj.keys()) + return [(torch.utils._pytree.MappingKey(k), v) for k, v in zip(context, values)], context + + def unflatten_cls(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> cls: # type: ignore[valid-type] + """Restores an instance of ``%s`` from python objects.""" + return cls(**dict(zip(context, values))) + + name = _lower_name_with_(cls.__name__) + flatten_cls.__name__ = f"flatten_{name}" + flatten_with_keys_cls.__name__ = f"flatten_with_keys_{name}" + unflatten_cls.__name__ = f"unflatten_{name}" + flatten_cls.__doc__ = flatten_cls.__doc__ % cls.__name__ + flatten_with_keys_cls.__doc__ = flatten_with_keys_cls.__doc__ % cls.__name__ + unflatten_cls.__doc__ = unflatten_cls.__doc__ % cls.__name__ + supported_classes.add(cls) + return flatten_cls, flatten_with_keys_cls, unflatten_cls + + +def _lower_name_with_(name): + s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() diff --git a/optimum/torch_export_patches/_treenode_registry.py b/optimum/torch_export_patches/_treenode_registry.py index 50128f06..24224bcf 100644 --- a/optimum/torch_export_patches/_treenode_registry.py +++ b/optimum/torch_export_patches/_treenode_registry.py @@ -4,8 +4,11 @@ import optree import torch import transformers +from diffusers.models.unets import unet_2d_condition from packaging import version as pv -from transformers import cache_utils +from transformers import cache_utils, modeling_outputs + +from optimum.torch_export_patches import _treenode try: @@ -13,8 +16,6 @@ except ImportError: from transformers.cache_utils import MambaCache -from .serialization import _lower_name_with_ - PATCH_OF_PATCHES: set[Any] = set() @@ -95,13 +96,22 @@ def register_cache_serialization( """ wrong: dict[type, str | None] = {} if patch_transformers: - from .serialization.transformers_impl import WRONG_REGISTRATIONS + WRONG_REGISTRATIONS = { + cache_utils.DynamicCache: "4.50", + modeling_outputs.BaseModelOutput: None, + } wrong |= WRONG_REGISTRATIONS if patch_diffusers: - from .serialization.diffusers_impl import WRONG_REGISTRATIONS - wrong |= WRONG_REGISTRATIONS + def _make_wrong_registrations() -> dict[type, str | None]: + res: dict[type, str | None] = {} + for c in [unet_2d_condition.UNet2DConditionOutput]: + if c is not None: + res[c] = None + return res + + wrong |= _make_wrong_registrations() registration_functions = serialization_functions( patch_transformers=patch_transformers, patch_diffusers=patch_diffusers, verbose=verbose @@ -153,7 +163,7 @@ def serialization_functions( all_functions: dict[type, str | None] = {} if patch_transformers: - from .serialization.transformers_impl import ( + from optimum.torch_export_patches._treenode import ( SUPPORTED_DATACLASSES, flatten_dynamic_cache, flatten_encoder_decoder_cache, @@ -171,6 +181,7 @@ def serialization_functions( unflatten_sliding_window_cache, unflatten_static_cache, ) + from .serialization.transformers_impl import ( __dict__ as dtr, ) @@ -226,7 +237,7 @@ def serialization_functions( supported_classes |= SUPPORTED_DATACLASSES for cls in supported_classes: - lname = _lower_name_with_(cls.__name__) + lname = _treenode._lower_name_with_(cls.__name__) assert f"flatten_{lname}" in all_functions, ( f"Unable to find function 'flatten_{lname}' in {list(all_functions)}" ) diff --git a/tests/patch/test_torch_export_patches.py b/tests/patch/test_torch_export_patches.py new file mode 100644 index 00000000..1a198252 --- /dev/null +++ b/tests/patch/test_torch_export_patches.py @@ -0,0 +1,138 @@ +import unittest + +from optimum.torch_export_patches._core import torch_export_patches + + +class TestOnnxExportErrors(unittest.TestCase): + # @skipif_ci_windows("not working on Windows") + def test_pytree_flatten_mamba_cache(self): + import torch + import torch.utils._pytree as py_pytree + + try: + from transformers.models.mamba.modeling_mamba import MambaCache + except ImportError: + from transformers.cache_utils import MambaCache + + class Config: + def __init__(self): + self.intermediate_size = 8 + self.state_size = 16 + self.conv_kernel = 32 + self.num_hidden_layers = 64 + self.dtype = torch.float16 + + cache = MambaCache(Config(), max_batch_size=1, device="cpu") + + with torch_export_patches(verbose=1): + values, spec = py_pytree.tree_flatten(cache) + cache2 = py_pytree.tree_unflatten(values, spec) + self.assertEqual(cache.max_batch_size, cache2.max_batch_size) + self.assertEqual(cache.intermediate_size, cache2.intermediate_size) + self.assertEqual(cache.ssm_state_size, cache2.ssm_state_size) + self.assertEqual(cache.conv_kernel_size, cache2.conv_kernel_size) + self.assertEqualArrayAny(cache.conv_states, cache2.conv_states) + self.assertEqualArrayAny(cache.ssm_states, cache2.ssm_states) + + # @skipif_ci_windows("not working on Windows") + def test_exportable_mamba_cache(self): + import torch + from transformers.models.mamba.modeling_mamba import MambaCache + + class Config: + def __init__(self): + self.intermediate_size = 8 + self.state_size = 16 + self.conv_kernel = 32 + self.num_hidden_layers = 64 + self.dtype = torch.float16 + + class Model(torch.nn.Module): + def forward(self, x: torch.Tensor, cache: MambaCache): + x1 = cache.ssm_states[0] + x + x2 = cache.conv_states[0][:, :, ::2] + x1 + return x2 + + cache = MambaCache(Config(), max_batch_size=1, device="cpu") + x = torch.ones(2, 8, 16).to(torch.float16) + model = Model() + model(x, cache) + + with torch_export_patches(verbose=1, patch_transformers=True): + cache = MambaCache(Config(), max_batch_size=1, device="cpu") + torch.export.export(Model(), (x, cache)) + + # @skipif_ci_windows("not working on Windows") + def test_exportable_mamba_cache_dynamic(self): + import torch + from transformers.models.mamba.modeling_mamba import MambaCache + + class Config: + def __init__(self): + self.intermediate_size = 8 + self.state_size = 16 + self.conv_kernel = 32 + self.num_hidden_layers = 2 + self.dtype = torch.float16 + + class Model(torch.nn.Module): + def forward(self, x: torch.Tensor, cache: MambaCache): + x1 = cache.ssm_states[0] + x + x2 = cache.conv_states[0][:, :, ::2] + x1 + return x2 + + cache = MambaCache(Config(), max_batch_size=1, device="cpu") + x = torch.ones(2, 8, 16).to(torch.float16) + model = Model() + model(x, cache) + dynamic = torch.export.Dim.DYNAMIC + + with torch_export_patches(): + cache = MambaCache(Config(), max_batch_size=2, device="cpu") + torch.export.export( + Model(), + (x, cache), + dynamic_shapes=({0: dynamic}, [[{0: dynamic}, {0: dynamic}], [{0: dynamic}, {0: dynamic}]]), + ) + + def test_exportable_dynamic_shapes_constraints(self): + import torch + + class CustomCache: + def __init__(self, shape=None): + self.cache = [torch.zeros(shape), torch.zeros(shape)] if shape else [] + + def flatten_cache(cache): + return [cache.cache], ["cache"] + + def unflatten_cache(values, context, output_type=None): + cache = CustomCache() + cache.cache = values[0] + return cache + + def flatten_with_keys_cache(d): + values, context = flatten_cache(d) + return [(torch.utils._pytree.MappingKey(k), v) for k, v in zip(context, values)], context + + torch.utils._pytree.register_pytree_node( + CustomCache, + flatten_cache, + unflatten_cache, + serialized_type_name=f"{CustomCache.__module__}.{CustomCache.__name__}", + flatten_with_keys_fn=flatten_with_keys_cache, + ) + + class Model(torch.nn.Module): + def forward(self, x, cache): + return cache.cache[0][0, :] + x + + model = Model() + model.eval() + x, cache = torch.rand((2, 4)), CustomCache((2, 4)) + model(x, cache) + dynamic = torch.export.Dim.DYNAMIC + torch.export.export(model, (x, cache), dynamic_shapes=({0: dynamic}, [[{0: dynamic}, {0: dynamic}]])) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 75f2b3d71b981b9d0ff0410b959ce515e18d1982 Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Mon, 28 Jul 2025 21:13:48 +0000 Subject: [PATCH 03/10] deleted debugging parameters --- optimum/torch_export_patches/_core.py | 115 ++------------------------ 1 file changed, 8 insertions(+), 107 deletions(-) diff --git a/optimum/torch_export_patches/_core.py b/optimum/torch_export_patches/_core.py index ca16a61d..2baa3667 100644 --- a/optimum/torch_export_patches/_core.py +++ b/optimum/torch_export_patches/_core.py @@ -140,8 +140,6 @@ def torch_export_patches( patch_torch: bool = True, patch_transformers: bool = False, patch_diffusers: bool = False, - catch_constraints: bool = True, - stop_if_static: int = 0, verbose: int = 0, patch: bool = True, custom_patches: type["torch.nn.Module"] | None = None, @@ -157,15 +155,6 @@ def torch_export_patches( patch_torch: patches :epkg:`torch` with supported implementation patch_transformers: patches :epkg:`transformers` with supported implementation patch_diffusers: patches :epkg:`diffusers` with supported implementation - catch_constraints: catch constraints related to dynamic shapes, - as a result, some dynamic dimension may turn into static ones, - the environment variable ``SKIP_SOLVE_CONSTRAINTS=0`` - can be put to stop at that stage. - stop_if_static: see example :ref:`l-plot-export-locale-issue`, - to stop the export as soon as an issue is detected with dynamic shapes - and show a stack trace indicating the exact location of the issue, - ``if stop_if_static > 1``, more methods are replace to catch more - issues patch: if False, disable all patches but keeps the registration of serialization functions if other patch functions are enabled custom_patches: to apply custom patches, @@ -231,7 +220,7 @@ def torch_export_patches( may appear ``AssertionError: Mutating module attribute _seen_tokens during export.``. It can be avoided by setting ``strict=False`` when call :func:`torch.export.export`. """ - if rewrite: + if rewrite is not None or dump_rewriting is not None: raise ValueError( "The argument `rewrite` is not supported at the moment, as the automatic rewriting is not implemented yet." ) @@ -256,6 +245,10 @@ def torch_export_patches( # pass # elif not patch: if not patch: + # NOTE: registering caches serialization is not recognized as patches + # It's needed to successfully export the models through torch.export, + # as the serialization functions are not registered by default. + fct_callable = lambda x: x # noqa: E731 done = _treenode_registry.register_cache_serialization( patch_transformers=patch_transformers, @@ -307,72 +300,24 @@ def torch_export_patches( if patch_torch: if verbose: print(f"[torch_export_patches] torch.__version__={torch.__version__!r}") - print(f"[torch_export_patches] stop_if_static={stop_if_static!r}") print("[torch_export_patches] patch pytorch") # torch.vmap f_vmap = torch.vmap torch.vmap = _patch_torch.patched_vmap - # torch.jit.isinstance - f_jit_isinstance = torch.jit.isinstance - torch.jit.isinstance = isinstance - + # TODO(anyone): Test these to see if they are still needed # torch._dynamo.mark_static_address f_mark_static_address = torch._dynamo.mark_static_address torch._dynamo.mark_static_address = lambda *_, **y_: None - # torch._subclasses.fake_impls.infer_size f_infer_size = torch._subclasses.fake_impls.infer_size torch._subclasses.fake_impls.infer_size = _patch_torch.patched_infer_size - # torch._refs._broadcast_shapes f__broadcast_shapes = torch._refs._broadcast_shapes torch._refs._broadcast_shapes = _patch_torch.patched__broadcast_shapes torch._meta_registrations._broadcast_shapes = _patch_torch.patched__broadcast_shapes - # torch._export.non_strict_utils.produce_guards_and_solve_constraints - if catch_constraints: - if verbose: - print("[torch_export_patches] modifies shape constraints") - f_produce_guards_and_solve_constraints = ( - torch._export.non_strict_utils.produce_guards_and_solve_constraints - ) - f__check_input_constraints_for_graph = torch._export.utils._check_input_constraints_for_graph - torch._export.non_strict_utils.produce_guards_and_solve_constraints = ( - lambda *args, **kwargs: _patch_torch.catch_produce_guards_and_solve_constraints( - f_produce_guards_and_solve_constraints, *args, verbose=verbose, **kwargs - ) - ) - torch._export.utils._check_input_constraints_for_graph = ( - lambda *args, **kwargs: _patch_torch.patch__check_input_constraints_for_graph( - f__check_input_constraints_for_graph, *args, verbose=verbose, **kwargs - ) - ) - - if stop_if_static: - from torch.fx.experimental.symbolic_shapes import ShapeEnv - - ShapeEnv._log_guard_remember = ShapeEnv._log_guard - - if verbose: - print("[torch_export_patches] assert when a dynamic dimension turns static") - print("[torch_export_patches] replaces ShapeEnv._set_replacement") - - f_shape_env__set_replacement = ShapeEnv._set_replacement - ShapeEnv._set_replacement = _patch_torch.patched_ShapeEnv._set_replacement - - if verbose: - print("[torch_export_patches] replaces ShapeEnv._log_guard") - f_shape_env__log_guard = ShapeEnv._log_guard - ShapeEnv._log_guard = _patch_torch.patched_ShapeEnv._log_guard - - if stop_if_static > 1: - if verbose: - print("[torch_export_patches] replaces ShapeEnv._check_frozen") - f_shape_env__check_frozen = ShapeEnv._check_frozen - ShapeEnv._check_frozen = _patch_torch.patched_ShapeEnv._check_frozen - #################### # patch transformers #################### @@ -415,7 +360,8 @@ def torch_export_patches( # export ######## - fct_callable = _replacement_before_exporting if patch_transformers else (lambda x: x) + def fct_callable(x): + return x if verbose: print("[torch_export_patches] done patching") @@ -447,7 +393,6 @@ def torch_export_patches( if patch_torch: # this should disappear when torch.jit is removed torch.vmap = f_vmap - torch.jit.isinstance = f_jit_isinstance torch._dynamo.mark_static_address = f_mark_static_address # tracked by https://github.com/pytorch/pytorch/issues/143495 torch._subclasses.fake_impls.infer_size = f_infer_size @@ -457,31 +402,6 @@ def torch_export_patches( if verbose: print("[torch_export_patches] restored pytorch functions") - if stop_if_static: - if verbose: - print("[torch_export_patches] restored ShapeEnv._set_replacement") - - ShapeEnv._set_replacement = f_shape_env__set_replacement - - if verbose: - print("[torch_export_patches] restored ShapeEnv._log_guard") - - ShapeEnv._log_guard = f_shape_env__log_guard - - if stop_if_static > 1: - if verbose: - print("[torch_export_patches] restored ShapeEnv._check_frozen") - ShapeEnv._check_frozen = f_shape_env__check_frozen - - if catch_constraints: - # to catch or skip dynamic_shapes issues - torch._export.non_strict_utils.produce_guards_and_solve_constraints = ( - f_produce_guards_and_solve_constraints - ) - torch._export.utils._check_input_constraints_for_graph = f__check_input_constraints_for_graph - if verbose: - print("[torch_export_patches] restored shape constraints") - if custom_patches: if verbose: print("[torch_export_patches] unpatches custom patches") @@ -527,22 +447,3 @@ def torch_export_patches( ######## _treenode_registry.unregister_cache_serialization(cache_done, verbose=verbose) - - -def _replacement_before_exporting(args: Any) -> Any: - """Does replacements on the given inputs if needed.""" - if args is None: - return None - if isinstance(args, (int, float)): - return args - if type(args) not in {dict, tuple, list}: - # BaseModelOutput is a dict - return args - if isinstance(args, dict): - return {k: _replacement_before_exporting(v) for k, v in args.items()} - if isinstance(args, tuple): - return tuple(_replacement_before_exporting(v) for v in args) - if isinstance(args, list): - return [_replacement_before_exporting(v) for v in args] - - return args From a5ea693074170fe4df0c2bf7c55c5621832c373d Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Mon, 28 Jul 2025 23:51:43 +0000 Subject: [PATCH 04/10] fix CI tests --- .../torch_export_patches/_cache_creater.py | 549 ++++++++++++++ optimum/torch_export_patches/_core.py | 21 +- optimum/torch_export_patches/_patch_torch.py | 2 +- .../_treenode_diffusers.py | 23 + .../_treenode_registry.py | 28 +- ..._treenode.py => _treenode_transformers.py} | 48 +- optimum/torch_export_patches/_type_utils.py | 117 +++ optimum/torch_export_patches/_verbose.py | 686 ++++++++++++++++++ tests/patch/test_torch_export_patches.py | 109 +++ 9 files changed, 1528 insertions(+), 55 deletions(-) create mode 100644 optimum/torch_export_patches/_cache_creater.py create mode 100644 optimum/torch_export_patches/_treenode_diffusers.py rename optimum/torch_export_patches/{_treenode.py => _treenode_transformers.py} (92%) create mode 100644 optimum/torch_export_patches/_type_utils.py create mode 100644 optimum/torch_export_patches/_verbose.py diff --git a/optimum/torch_export_patches/_cache_creater.py b/optimum/torch_export_patches/_cache_creater.py new file mode 100644 index 00000000..4e0f46b7 --- /dev/null +++ b/optimum/torch_export_patches/_cache_creater.py @@ -0,0 +1,549 @@ +from typing import Any, Callable + +import packaging.version as pv +import torch +import transformers +import transformers.cache_utils + + +try: + from transformers.models.mamba.modeling_mamba import MambaCache +except ImportError: + from transformers.cache_utils import MambaCache + + +class CacheKeyValue: + """Cache wrapper for compatibility across transformers versions. + + Starting transformers>=4.54, the cache API has deprecated + ``cache.key_cache`` and ``cache.value_cache``. + This class wraps a cache independently from transformers version and enables + attributes ``key_cache`` and ``value_cache``. + + Example: + .. code-block:: python + + capi = CacheKeyValue(cache) + capi.key_cache + capi.value_cache + """ + + def __init__(self, cache=None): + if hasattr(cache, "layers"): + layers = [ + layer + for layer in cache.layers + if layer is not None and layer.keys is not None and layer.values is not None + ] + self.key_cache = [layer.keys for layer in layers] + self.value_cache = [layer.values for layer in layers] + if None in self.key_cache or None in self.value_cache: + from .helper import string_type + + raise AssertionError( + f"issue with key_cache={string_type(self.key_cache)}, " + f"or value_cache={string_type(self.value_cache)}, " + f"cache.layers={string_type(cache.layers)}" + ) + elif cache is not None: + self.key_cache = cache.key_cache + self.value_cache = cache.value_cache + + def make_dynamic_cache(self): + """Create a dynamic cache from the wrapped cache. + + Returns: + transformers.cache_utils.DynamicCache: The dynamic cache created from key-value pairs. + """ + return make_dynamic_cache(list(zip(self.key_cache, self.value_cache))) + + +if pv.Version(transformers.__version__) > pv.Version("4.49.99999"): + + def make_dynamic_cache( + key_value_pairs: list[tuple[torch.Tensor, torch.Tensor]], + ) -> transformers.cache_utils.DynamicCache: + """Create an instance of transformers.cache_utils.DynamicCache. + + This version is valid for transformers >= 4.50. + + Args: + key_value_pairs: List of pairs of (key, values). + + Returns: + transformers.cache_utils.DynamicCache: The created dynamic cache. + + Example: + .. runpython:: + :showcode: + + import torch + from onnx_diagnostic.helpers import string_type + from onnx_diagnostic.helpers.cache_helper import make_dynamic_cache + + n_layers = 2 + bsize, nheads, slen, dim = 2, 4, 3, 7 + + past_key_values = make_dynamic_cache( + [ + ( + torch.randn(bsize, nheads, slen, dim), + torch.randn(bsize, nheads, slen, dim), + ) + for i in range(n_layers) + ] + ) + print(string_type(past_key_values, with_shape=True)) + """ + cache = transformers.cache_utils.DynamicCache(key_value_pairs) + if hasattr(cache, "layers") and len(key_value_pairs) < len(cache.layers): + # The cache constructor contains the two following lines + # (in cache_utils.py) which append empty layers when the cache is + # initialized. We need to remove them. + # self.num_hidden_layers = getattr(config, "num_hidden_layers", 1) + # self.append_new_layers(self.num_hidden_layers - 1) + cache.layers[:] = cache.layers[-len(key_value_pairs) :] + assert not hasattr(cache, "layers") or len(key_value_pairs) == len(cache.layers), ( + f"Unexpected number of layers in the cache ({len(cache.layers)}), {len(key_value_pairs)} expected." + ) + return cache + +else: + + def make_dynamic_cache( + key_value_pairs: list[tuple[torch.Tensor, torch.Tensor]], + ) -> transformers.cache_utils.DynamicCache: + """Create an instance of transformers.cache_utils.DynamicCache. + + This version is valid for transformers < 4.50. + + Args: + key_value_pairs: List of pairs of (key, values). + + Returns: + transformers.cache_utils.DynamicCache: The created dynamic cache. + + Example: + .. runpython:: + :showcode: + + import torch + from onnx_diagnostic.helpers import string_type + from onnx_diagnostic.helpers.cache_helper import make_dynamic_cache + + n_layers = 2 + bsize, nheads, slen, dim = 2, 4, 3, 7 + + past_key_values = make_dynamic_cache( + [ + ( + torch.randn(bsize, nheads, slen, dim), + torch.randn(bsize, nheads, slen, dim), + ) + for i in range(n_layers) + ] + ) + print(string_type(past_key_values, with_shape=True)) + """ + cache = transformers.cache_utils.DynamicCache(len(key_value_pairs)) # type: ignore + for i, (key, value) in enumerate(key_value_pairs): + cache.update(key, value, i) + return cache + + +def make_static_cache( + key_value_pairs: list[tuple[torch.Tensor, torch.Tensor]], + max_cache_len: int | None = None, +) -> transformers.cache_utils.DynamicCache: + """Create an instance of transformers.cache_utils.StaticCache. + + Args: + key_value_pairs: List of pairs of (key, values). + max_cache_len: Max cache length or something inferred from the vector. + + Returns: + transformers.cache_utils.StaticCache: The created static cache. + + Example: + .. runpython:: + :showcode: + + import torch + from onnx_diagnostic.helpers import string_type + from onnx_diagnostic.helpers.cache_helper import make_static_cache + + n_layers = 2 + bsize, nheads, slen, dim = 2, 4, 3, 7 + + past_key_values = make_static_cache( + [ + ( + torch.randn(bsize, nheads, slen, dim), + torch.randn(bsize, nheads, slen, dim), + ) + for i in range(n_layers) + ], + max_cache_len=10, + ) + print(string_type(past_key_values, with_shape=True)) + """ + + class Config: + def __init__(self): + self.head_dim = key_value_pairs[0][0].shape[-1] + self.num_attention_heads = key_value_pairs[0][0].shape[1] + self.num_hidden_layers = len(key_value_pairs) + + assert max_cache_len is not None, ( + f"max_cache_len={max_cache_len} cannot be setup automatically yet from shape {key_value_pairs[0][0].shape}" + ) + torch._check( + max_cache_len >= key_value_pairs[0][0].shape[2], + ( + f"max_cache_len={max_cache_len} cannot be smaller " + f"shape[2]={key_value_pairs[0][0].shape[2]} in shape " + f"{key_value_pairs[0][0].shape}" + ), + ) + cache = transformers.cache_utils.StaticCache( + config=Config(), + max_batch_size=key_value_pairs[0][0].shape[0], + device=key_value_pairs[0][0].device, + dtype=key_value_pairs[0][0].dtype, + max_cache_len=max_cache_len, + ) + ca = CacheKeyValue(cache) + for i in range(len(key_value_pairs)): + assert key_value_pairs[i][0].shape == key_value_pairs[i][1].shape, ( + f"Shape mismatch {key_value_pairs[i][0].shape} != {key_value_pairs[i][1].shape}" + ) + d = key_value_pairs[i][1].shape[2] + ca.key_cache[i][:, :, :d, :] = key_value_pairs[i][0] + ca.value_cache[i][:, :, :d, :] = key_value_pairs[i][1] + if hasattr(cache, "layers") and len(key_value_pairs) < len(cache.layers): + # The cache constructor contains the two following lines + # (in cache_utils.py) which append empty layers when the cache is + # initialized. We need to remove them. + # self.num_hidden_layers = getattr(config, "num_hidden_layers", 1) + # self.append_new_layers(self.num_hidden_layers - 1) + cache.layers[:] = cache.layers[-len(key_value_pairs) :] + assert not hasattr(cache, "layers") or len(key_value_pairs) == len(cache.layers), ( + f"Unexpected number of layers in the cache ({len(cache.layers)}), {len(key_value_pairs)} expected." + ) + return cache + + +def make_encoder_decoder_cache( + self_attention_cache: transformers.cache_utils.DynamicCache, + cross_attention_cache: transformers.cache_utils.DynamicCache, +) -> transformers.cache_utils.EncoderDecoderCache: + """Create an EncoderDecoderCache. + + Args: + self_attention_cache: The self-attention cache. + cross_attention_cache: The cross-attention cache. + + Returns: + transformers.cache_utils.EncoderDecoderCache: The created encoder-decoder cache. + """ + return transformers.cache_utils.EncoderDecoderCache( + self_attention_cache=self_attention_cache, cross_attention_cache=cross_attention_cache + ) + + +def make_mamba_cache(key_value_pairs: list[tuple[torch.Tensor, torch.Tensor]]) -> MambaCache: + """Create a MambaCache. + + Args: + key_value_pairs: List of pairs of (key, values). + + Returns: + MambaCache: The created Mamba cache. + """ + dtype = key_value_pairs[0][0].dtype + + class Config: + def __init__(self): + self.intermediate_size = key_value_pairs[0][0].shape[1] + self.conv_kernel = key_value_pairs[0][0].shape[-1] + self.state_size = key_value_pairs[0][1].shape[-1] + self.num_hidden_layers = len(key_value_pairs) + self.dtype = dtype + + cache = MambaCache( + Config(), + max_batch_size=key_value_pairs[0][0].shape[0], + device=key_value_pairs[0][0].device, + dtype=dtype, + ) + for i in range(len(key_value_pairs)): + assert cache.conv_states[i].dtype == dtype, ( + f"Type mismatch for cache.conv_states[{i}].dtype={cache.conv_states[i].dtype} != {dtype}" + ) + assert cache.ssm_states[i].dtype == dtype, ( + f"Type mismatch for cache.ssm_states[{i}].dtype={cache.ssm_states[i].dtype} != {dtype}" + ) + assert cache.conv_states[i].shape == key_value_pairs[i][0].shape, ( + f"Shape mismatch, expected {cache.conv_states[i].shape}, got {key_value_pairs[i][0].shape}" + ) + cache.conv_states[i][:, :, :] = key_value_pairs[i][0] + assert cache.ssm_states[i].shape == key_value_pairs[i][1].shape, ( + f"Shape mismatch, expected {cache.ssm_states[i].shape}, got {key_value_pairs[i][1].shape}" + ) + cache.ssm_states[i][:, :, :] = key_value_pairs[i][1] + return cache + + +def make_sliding_window_cache( + key_value_pairs: list[tuple[torch.Tensor, torch.Tensor]], +) -> transformers.cache_utils.SlidingWindowCache: + """Create a transformers.cache_utils.SlidingWindowCache. + + Args: + key_value_pairs: List of pairs of (key, values). + + Returns: + transformers.cache_utils.SlidingWindowCache: The created sliding window cache. + """ + + class Config: + def __init__(self): + self.head_dim = key_value_pairs[0][0].shape[-1] + self.num_attention_heads = key_value_pairs[0][0].shape[1] + self.num_hidden_layers = len(key_value_pairs) + self.sliding_window = key_value_pairs[0][0].shape[2] + + cache = transformers.cache_utils.SlidingWindowCache( + config=Config(), + max_batch_size=key_value_pairs[0][0].shape[0], + max_cache_len=key_value_pairs[0][0].shape[2], # same as sliding_window + device=key_value_pairs[0][0].device, + dtype=key_value_pairs[0][0].dtype, + ) + ca = CacheKeyValue(cache) + for i in range(len(key_value_pairs)): + assert ca.key_cache[i].shape == key_value_pairs[i][0].shape, ( + f"Shape mismatch, expected {cache.key_cache[i].shape}, got {key_value_pairs[i][0].shape}" + ) + ca.key_cache[i][:, :, :, :] = key_value_pairs[i][0] + assert ca.value_cache[i].shape == key_value_pairs[i][1].shape, ( + f"Shape mismatch, expected {cache.value_cache[i].shape}, got {key_value_pairs[i][1].shape}" + ) + ca.value_cache[i][:, :, :, :] = key_value_pairs[i][1] + if hasattr(cache, "layers") and len(key_value_pairs) < len(cache.layers): + # The cache constructor contains the two following lines + # (in cache_utils.py) which append empty layers when the cache is + # initialized. We need to remove them. + # self.num_hidden_layers = getattr(config, "num_hidden_layers", 1) + # self.append_new_layers(self.num_hidden_layers - 1) + cache.layers[:] = cache.layers[-len(key_value_pairs) :] + assert not hasattr(cache, "layers") or len(key_value_pairs) == len(cache.layers), ( + f"Unexpected number of layers in the cache ({len(cache.layers)}), {len(key_value_pairs)} expected." + ) + return cache + + +def make_hybrid_cache( + key_value_pairs: list[tuple[torch.Tensor, torch.Tensor]], + max_cache_len: int | None = None, + max_batch_size: int | None = None, + sliding_window: int | None = None, +) -> transformers.cache_utils.HybridCache: + """Create an instance of transformers.cache_utils.HybridCache. + + This version is valid for transformers < 4.50. + + Args: + key_value_pairs: List of pairs of (key, values). + max_cache_len: Maximum cache length. + max_batch_size: Maximum batch size. + sliding_window: Sliding window size. + + Returns: + transformers.cache_utils.HybridCache: The created hybrid cache. + + Example: + .. runpython:: + :showcode: + + import torch + from onnx_diagnostic.helpers import string_type + from onnx_diagnostic.helpers.cache_helper import make_hybrid_cache + + n_layers = 2 + bsize, nheads, slen, dim = 2, 4, 3, 7 + + past_key_values = make_hybrid_cache( + [ + ( + torch.randn(bsize, nheads, slen, dim), + torch.randn(bsize, nheads, slen, dim), + ) + for i in range(n_layers) + ] + ) + print(string_type(past_key_values, with_shape=True)) + + Note: + This part defines how the shapes are working in one HybridCache: + + .. code-block:: python + + self.max_cache_len = ( + max_cache_len if max_cache_len is not None else config.max_position_embeddings) + + # Sliding layers can't be larger than the overall max cache len + self.sliding_window_len = min(config.sliding_window, self.max_cache_len) + self.max_batch_size = max_batch_size + + self.head_dim = ( + config.head_dim if hasattr(config, "head_dim") + else config.hidden_size // config.num_attention_heads + ) + + self._dtype = dtype + self.num_key_value_heads = ( + config.num_attention_heads + if getattr(config, "num_key_value_heads", None) is None + else config.num_key_value_heads + ) + + # If the attribute does not exist in the config, fallback to a simple StaticCache + if hasattr(config, "layer_types"): + self.is_sliding = [ + layer_type != "full_attention" for layer_type in config.layer_types] + else: + self.is_sliding = [False] * config.num_hidden_layers + + self.key_cache: list[torch.Tensor] = [] + self.value_cache: list[torch.Tensor] = [] + global_cache_shape = (self.max_batch_size, self.num_key_value_heads, + self.max_cache_len, self.head_dim) + sliding_cache_shape = (self.max_batch_size, self.num_key_value_heads, + self.sliding_window_len, self.head_dim) + self.sliding_window = min(config.sliding_window, max_cache_len) + device = torch.device(device) if device is not None else None + for i in range(config.num_hidden_layers): + layer_device = layer_device_map[i] if layer_device_map is not None else device + cache_shape = sliding_cache_shape if self.is_sliding[i] else global_cache_shape + new_layer_key_cache = torch.zeros( + cache_shape, dtype=self._dtype, device=layer_device) + new_layer_value_cache = torch.zeros( + cache_shape, dtype=self._dtype, device=layer_device) + torch._dynamo.mark_static_address(new_layer_key_cache) + torch._dynamo.mark_static_address(new_layer_value_cache) + self.key_cache.append(new_layer_key_cache) + self.value_cache.append(new_layer_value_cache) + """ + layer_types = None + if key_value_pairs: + assert not max_batch_size and not max_cache_len, ( + "key_value_pairs is not empty, do not specify max_cache_len and max_batch_size" + ) + max_batch_size = key_value_pairs[0][0].shape[0] + sets_of_dim = {kv[0].shape[2] for kv in key_value_pairs} + if len(sets_of_dim) == 1: + max_cache_len = sets_of_dim.pop() + sliding_window = max_cache_len + else: + assert len(sets_of_dim) == 2, f"Not implemented for more than 2 dimensions {sets_of_dim}" + max_cache_len = max(sets_of_dim) + sliding_window = min(sets_of_dim) + layer_types = [ + "full_attention" if i == max_cache_len else "sliding_attention" + for i in [kv[0].shape[2] for kv in key_value_pairs] + ] + else: + assert max_batch_size and max_cache_len, ( + "key_value_pairs is empty, max_batch_size and max_cache_len are required" + ) + if sliding_window is None: + sliding_window = max_cache_len + _max_cache_len = max_cache_len + _sliding_window = sliding_window + + class Config: + max_cache_len = _max_cache_len + batch_size = max_batch_size + num_heads = key_value_pairs[0][0].shape[1] if key_value_pairs else None + head_dim = key_value_pairs[0][0].shape[-1] if key_value_pairs else None + num_attention_heads = key_value_pairs[0][1].shape[1] if key_value_pairs else None + num_hidden_layers = len(key_value_pairs) + sliding_window = _sliding_window + num_key_value_heads = key_value_pairs[0][1].shape[1] # transformers 4.48.3 + + if layer_types: + Config.layer_types = layer_types # type: ignore[attr-defined] + + cache = transformers.cache_utils.HybridCache( + config=Config(), max_cache_len=max_cache_len, max_batch_size=max_batch_size + ) + for i, (key, value) in enumerate(key_value_pairs): + cache.update( + key, + value, + i, + cache_kwargs={"cache_position": torch.arange(0, key.shape[2], dtype=torch.int64).to(key.device)}, + ) + if hasattr(cache, "layers") and len(key_value_pairs) < len(cache.layers): + # The cache constructor contains the two following lines + # (in cache_utils.py) which append empty layers when the cache is + # initialized. We need to remove them. + # self.num_hidden_layers = getattr(config, "num_hidden_layers", 1) + # self.append_new_layers(self.num_hidden_layers - 1) + cache.layers[:] = cache.layers[-len(key_value_pairs) :] + assert not hasattr(cache, "layers") or len(key_value_pairs) == len(cache.layers), ( + f"Unexpected number of layers in the cache ({len(cache.layers)}), {len(key_value_pairs)} expected." + ) + return cache + + +def flatten_unflatten_for_dynamic_shapes( + obj: Any, + use_dict: bool = False, + change_function: Callable[[torch.Tensor], Any] | None = None, +) -> Any: + """Returns the object in a different structure similar to what the definition of the dynamic shapes should use. + + Args: + obj: object from a custom class + use_dict: closer to the original result but + :func:`torch.export.export` only considers the values, + the context gives the dictionary keys but it is not expressed + in the dynamic shapes, these specifications seems to be different + for the strict and non strict mode. It also preserves tuple. + change_function: to modifies the tensor in the structure itself, + like replace them by a shape + + Returns: + The serialized object + """ + if isinstance(obj, torch.Tensor): + return change_function(obj) if change_function else obj + flat, spec = torch.utils._pytree.tree_flatten(obj) + start = 0 + end = 0 + subtrees = [] + for subspec in spec.children_specs: + end += subspec.num_leaves + value = subspec.unflatten(flat[start:end]) + value = flatten_unflatten_for_dynamic_shapes(value, use_dict=use_dict, change_function=change_function) + subtrees.append(value) + start = end + if use_dict: + if spec.type is dict: + # This a dictionary. + return dict(zip(spec.context, subtrees)) + if spec.type is tuple: + return tuple(subtrees) + if spec.type is list: + return list(subtrees) + if spec.context: + # This is a custom class with attributes. + # It is returned as a list. + return list(subtrees) + raise ValueError( + f"Unable to interpret spec type {spec.type} (type is {type(spec.type)}, context is {spec.context})." + ) + # This is a list. + return subtrees diff --git a/optimum/torch_export_patches/_core.py b/optimum/torch_export_patches/_core.py index 2baa3667..08de49d2 100644 --- a/optimum/torch_export_patches/_core.py +++ b/optimum/torch_export_patches/_core.py @@ -68,7 +68,7 @@ def patch_module_or_class(mod, verbose: int = 0) -> dict[type, dict[type, Callab to_patch = mod name = "list" else: - name, to_patch = get_patches(mod, verbose) + name, to_patch = get_patches(mod) res = {} for cls in to_patch: @@ -108,7 +108,7 @@ def unpatch_module_or_class(mod, info: dict[type, dict[type, Callable]], verbose to_patch = mod name = "list" else: - name, to_patch = get_patches(mod, verbose) + name, to_patch = get_patches(mod) set_patch_cls = {i for i in to_patch if not isinstance(i, dict)} dict_patch_fct = {i["function"]: i for i in to_patch if isinstance(i, dict)} @@ -171,7 +171,6 @@ def torch_export_patches( The list of available patches: - * ``torch.jit.isinstance`` * ``torch._dynamo.mark_static_address`` * ``torch._subclasses.fake_impls.infer_size`` * ``torch.vmap`` @@ -180,8 +179,7 @@ def torch_export_patches( * Serialization of ``MambaCache`` (in :epkg:`transformers`) * Serialization of ``DynamicCache`` (in :epkg:`transformers`) * reduce errors due to shape inference - * fixes some transformers classes, - see :mod:`onnx_diagnostic.torch_export_patches.patches.patch_transformers` + * fixes some transformers classes Serialization issues happen when a module takes one input or output has a type :func:`torch.export.export` cannot serialize. @@ -207,15 +205,6 @@ def torch_export_patches( inputs = modificator(inputs) ep = torch.export.export(..., inputs, ...) - When running the model through the exported program, only the - serialization functions need to be restored: - - .. code-block:: python - - with register_additional_serialization_functions() as modificator: - inputs = modificator(inputs) - ep = torch.export.export(..., inputs, ...) - When exporting a model with a cache, the following error message may appear ``AssertionError: Mutating module attribute _seen_tokens during export.``. It can be avoided by setting ``strict=False`` when call :func:`torch.export.export`. @@ -324,7 +313,7 @@ def torch_export_patches( if patch_transformers: try: - import transformers.masking_utils as masking_utils + from transformers import masking_utils except ImportError: masking_utils = None @@ -414,7 +403,7 @@ def fct_callable(x): if patch_transformers: try: # masking_utils is available only if transformers==4.53.0 or later - import transformers.masking_utils as masking_utils + from transformers import masking_utils except ImportError: masking_utils = None if verbose: diff --git a/optimum/torch_export_patches/_patch_torch.py b/optimum/torch_export_patches/_patch_torch.py index 3327964f..fb90a725 100644 --- a/optimum/torch_export_patches/_patch_torch.py +++ b/optimum/torch_export_patches/_patch_torch.py @@ -1,6 +1,6 @@ import inspect import os -from typing import Any, Callable +from typing import Any, Callable, Sequence import torch diff --git a/optimum/torch_export_patches/_treenode_diffusers.py b/optimum/torch_export_patches/_treenode_diffusers.py new file mode 100644 index 00000000..6f091119 --- /dev/null +++ b/optimum/torch_export_patches/_treenode_diffusers.py @@ -0,0 +1,23 @@ +try: + from diffusers.models.unets.unet_2d_condition import UNet2DConditionOutput +except ImportError: + try: + import diffusers + except ImportError: + diffusers = None + UNet2DConditionOutput = None + if diffusers: + raise + +from optimum.torch_export_patches import _treenode_transformers + + +SUPPORTED_DATACLASSES: set[type] = set() + + +if UNet2DConditionOutput is not None: + ( + flatten_u_net2_d_condition_output, + flatten_with_keys_u_net2_d_condition_output, + unflatten_u_net2_d_condition_output, + ) = _treenode_transformers.make_serialization_function_for_dataclass(UNet2DConditionOutput, SUPPORTED_DATACLASSES) diff --git a/optimum/torch_export_patches/_treenode_registry.py b/optimum/torch_export_patches/_treenode_registry.py index 24224bcf..cafb4de3 100644 --- a/optimum/torch_export_patches/_treenode_registry.py +++ b/optimum/torch_export_patches/_treenode_registry.py @@ -8,7 +8,7 @@ from packaging import version as pv from transformers import cache_utils, modeling_outputs -from optimum.torch_export_patches import _treenode +from optimum.torch_export_patches import _treenode_transformers try: @@ -96,12 +96,12 @@ def register_cache_serialization( """ wrong: dict[type, str | None] = {} if patch_transformers: - WRONG_REGISTRATIONS = { + _wrong_registration = { cache_utils.DynamicCache: "4.50", modeling_outputs.BaseModelOutput: None, } - wrong |= WRONG_REGISTRATIONS + wrong |= _wrong_registration if patch_diffusers: def _make_wrong_registrations() -> dict[type, str | None]: @@ -163,26 +163,28 @@ def serialization_functions( all_functions: dict[type, str | None] = {} if patch_transformers: - from optimum.torch_export_patches._treenode import ( + from optimum.torch_export_patches._treenode_transformers import ( SUPPORTED_DATACLASSES, flatten_dynamic_cache, flatten_encoder_decoder_cache, + flatten_hybrid_cache, flatten_mamba_cache, flatten_sliding_window_cache, flatten_static_cache, flatten_with_keys_dynamic_cache, flatten_with_keys_encoder_decoder_cache, + flatten_with_keys_hybrid_cache, flatten_with_keys_mamba_cache, flatten_with_keys_sliding_window_cache, flatten_with_keys_static_cache, unflatten_dynamic_cache, unflatten_encoder_decoder_cache, + unflatten_hybrid_cache, unflatten_mamba_cache, unflatten_sliding_window_cache, unflatten_static_cache, ) - - from .serialization.transformers_impl import ( + from optimum.torch_export_patches._treenode_transformers import ( __dict__ as dtr, ) @@ -198,6 +200,14 @@ def serialization_functions( # f_check=make_dynamic_cache([(torch.rand((4, 4, 4)), torch.rand((4, 4, 4)))]), verbose=verbose, ), + cache_utils.HybridCache: lambda verbose=verbose: register_class_serialization( + cache_utils.HybridCache, + flatten_hybrid_cache, + unflatten_hybrid_cache, + flatten_with_keys_hybrid_cache, + # f_check=make_dynamic_cache([(torch.rand((4, 4, 4)), torch.rand((4, 4, 4)))]), + verbose=verbose, + ), MambaCache: lambda verbose=verbose: register_class_serialization( MambaCache, flatten_mamba_cache, @@ -230,14 +240,14 @@ def serialization_functions( classes.update(transformers_classes) if patch_diffusers: - from .serialization.diffusers_impl import SUPPORTED_DATACLASSES - from .serialization.diffusers_impl import __dict__ as dfu + from optimum.torch_export_patches._treenode_diffusers import SUPPORTED_DATACLASSES + from optimum.torch_export_patches._treenode_diffusers import __dict__ as dfu all_functions.update(dfu) supported_classes |= SUPPORTED_DATACLASSES for cls in supported_classes: - lname = _treenode._lower_name_with_(cls.__name__) + lname = _treenode_transformers.lower_name_with_(cls.__name__) assert f"flatten_{lname}" in all_functions, ( f"Unable to find function 'flatten_{lname}' in {list(all_functions)}" ) diff --git a/optimum/torch_export_patches/_treenode.py b/optimum/torch_export_patches/_treenode_transformers.py similarity index 92% rename from optimum/torch_export_patches/_treenode.py rename to optimum/torch_export_patches/_treenode_transformers.py index 6d5ae4ac..00b4f7a7 100644 --- a/optimum/torch_export_patches/_treenode.py +++ b/optimum/torch_export_patches/_treenode_transformers.py @@ -17,20 +17,10 @@ from transformers.cache_utils import MambaCache from transformers.modeling_outputs import BaseModelOutput -from ...helpers.cache_helper import ( - CacheKeyValue, - make_dynamic_cache, - make_hybrid_cache, - make_sliding_window_cache, - make_static_cache, -) +from optimum.torch_export_patches import _cache_creater SUPPORTED_DATACLASSES: set[type] = set() -WRONG_REGISTRATIONS = { - DynamicCache: "4.50", - BaseModelOutput: None, -} ############ @@ -102,7 +92,7 @@ def flatten_dynamic_cache( dynamic_cache: DynamicCache, ) -> tuple[list[Any], torch.utils._pytree.Context]: """Serializes a :class:`transformers.cache_utils.DynamicCache` with python objects.""" - ca = CacheKeyValue(dynamic_cache) + ca = _cache_creater.CacheKeyValue(dynamic_cache) flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] return [f[1] for f in flat], [f[0] for f in flat] @@ -117,7 +107,7 @@ def flatten_with_keys_dynamic_cache( def unflatten_dynamic_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> DynamicCache: """Restores a :class:`transformers.cache_utils.DynamicCache` from python objects.""" - return make_dynamic_cache(list(zip(values[0], values[1]))) + return _cache_creater.make_dynamic_cache(list(zip(values[0], values[1]))) ############# @@ -129,7 +119,7 @@ def flatten_hybrid_cache( cache: HybridCache, ) -> tuple[list[Any], torch.utils._pytree.Context]: """Serializes a :class:`transformers.cache_utils.HybridCache` with python objects.""" - ca = CacheKeyValue(cache) + ca = _cache_creater.CacheKeyValue(cache) flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] return [f[1] for f in flat], [f[0] for f in flat] @@ -144,7 +134,7 @@ def flatten_with_keys_hybrid_cache( def unflatten_hybrid_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> HybridCache: """Restores a :class:`transformers.cache_utils.HybridCache` from python objects.""" - return make_hybrid_cache(list(zip(values[0], values[1]))) + return _cache_creater.make_hybrid_cache(list(zip(values[0], values[1]))) ############# @@ -156,7 +146,7 @@ def flatten_static_cache( cache: StaticCache, ) -> tuple[list[Any], torch.utils._pytree.Context]: """Serializes a :class:`transformers.cache_utils.StaticCache` with python objects.""" - ca = CacheKeyValue(cache) + ca = _cache_creater.CacheKeyValue(cache) assert not ca.key_cache or cache.max_cache_len == ca.key_cache[0].shape[2], ( f"Serialization doet not work when " f"cache.max_cache_len={cache.max_cache_len} != " @@ -176,7 +166,7 @@ def flatten_with_keys_static_cache( def unflatten_static_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> StaticCache: """Restores a :class:`transformers.cache_utils.StaticCache` from python objects.""" - return make_static_cache(list(zip(values[0], values[1])), max_cache_len=values[0][0].shape[2]) + return _cache_creater.make_static_cache(list(zip(values[0], values[1])), max_cache_len=values[0][0].shape[2]) #################### @@ -190,7 +180,7 @@ def flatten_sliding_window_cache( """Serializes a :class:`transformers.cache_utils.SlidingWindowCache` with python objects. """ - ca = CacheKeyValue(cache) + ca = _cache_creater.CacheKeyValue(cache) flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] return [f[1] for f in flat], [f[0] for f in flat] @@ -210,7 +200,7 @@ def unflatten_sliding_window_cache( ) -> SlidingWindowCache: """Restores a :class:`transformers.cache_utils.SlidingWindowCache` from python objects.""" key_cache, value_cache = values - return make_sliding_window_cache(list(zip(values[0], values[1]))) + return _cache_creater.make_sliding_window_cache(list(zip(values[0], values[1]))) ##################### @@ -260,14 +250,12 @@ def unflatten_encoder_decoder_cache( ############# -( - flatten_base_model_output, - flatten_with_keys_base_model_output, - unflatten_base_model_output, -) = _make_serialization_function_for_dataclass(BaseModelOutput, SUPPORTED_DATACLASSES) +def lower_name_with_(name): + s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() -def _make_serialization_function_for_dataclass( +def make_serialization_function_for_dataclass( cls: type, supported_classes: set[type] ) -> tuple[Callable, Callable, Callable]: """Automatically creates serialization function for a class decorated with @@ -289,7 +277,7 @@ def unflatten_cls(values: list[Any], context: torch.utils._pytree.Context, outpu """Restores an instance of ``%s`` from python objects.""" return cls(**dict(zip(context, values))) - name = _lower_name_with_(cls.__name__) + name = lower_name_with_(cls.__name__) flatten_cls.__name__ = f"flatten_{name}" flatten_with_keys_cls.__name__ = f"flatten_with_keys_{name}" unflatten_cls.__name__ = f"unflatten_{name}" @@ -300,6 +288,8 @@ def unflatten_cls(values: list[Any], context: torch.utils._pytree.Context, outpu return flatten_cls, flatten_with_keys_cls, unflatten_cls -def _lower_name_with_(name): - s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) - return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() +( + flatten_base_model_output, + flatten_with_keys_base_model_output, + unflatten_base_model_output, +) = make_serialization_function_for_dataclass(BaseModelOutput, SUPPORTED_DATACLASSES) diff --git a/optimum/torch_export_patches/_type_utils.py b/optimum/torch_export_patches/_type_utils.py new file mode 100644 index 00000000..1e179f42 --- /dev/null +++ b/optimum/torch_export_patches/_type_utils.py @@ -0,0 +1,117 @@ +import numpy as np +import onnx +import onnx.helper as oh +import torch +from onnx import ( + TensorProto, +) + + +def np_dtype_to_tensor_dtype(dt: np.dtype) -> int: + """Converts a numpy dtype into an ONNX element type. + + Args: + dt: The numpy dtype to convert. + + Returns: + The corresponding ONNX tensor type as an integer. + + Raises: + ValueError: If the dtype cannot be converted to an ONNX type. + """ + try: + return oh.np_dtype_to_tensor_dtype(dt) + except ValueError: + try: + import ml_dtypes + except ImportError: + ml_dtypes = None # type: ignore + if ml_dtypes is not None: + if dt == ml_dtypes.bfloat16: + return TensorProto.BFLOAT16 + if dt == ml_dtypes.float8_e4m3fn: + return TensorProto.FLOAT8E4M3FN + if dt == ml_dtypes.float8_e4m3fnuz: + return TensorProto.FLOAT8E4M3FNUZ + if dt == ml_dtypes.float8_e5m2: + return TensorProto.FLOAT8E5M2 + if dt == ml_dtypes.float8_e5m2fnuz: + return TensorProto.FLOAT8E5M2FNUZ + if dt == np.float32: + return TensorProto.FLOAT + if dt == np.float16: + return TensorProto.FLOAT16 + if dt == np.float64: + return TensorProto.DOUBLE + if dt == np.int64: + return TensorProto.INT64 + if dt == np.uint64: + return TensorProto.UINT64 + if dt == np.int16: + return TensorProto.INT16 + if dt == np.uint16: + return TensorProto.UINT16 + if dt == np.int32: + return TensorProto.INT32 + if dt == np.int8: + return TensorProto.INT8 + if dt == np.uint8: + return TensorProto.UINT8 + if dt == np.uint32: + return TensorProto.UINT32 + if dt == np.bool: + return TensorProto.BOOL + if dt == np.complex64: + return TensorProto.COMPLEX64 + if dt == np.complex128: + return TensorProto.COMPLEX128 + raise ValueError(f"Unable to convert type {dt}") + + +def torch_dtype_to_onnx_dtype(to: torch.dtype) -> int: + """Converts a torch dtype into an ONNX element type. + + Args: + to: The torch dtype to convert. + + Returns: + The corresponding ONNX tensor type as an integer. + + Raises: + NotImplementedError: If the torch dtype cannot be converted to an ONNX type. + """ + if to == torch.float32: + return onnx.TensorProto.FLOAT + if to == torch.float16: + return onnx.TensorProto.FLOAT16 + if to == torch.bfloat16: + return onnx.TensorProto.BFLOAT16 + if to == torch.float64: + return onnx.TensorProto.DOUBLE + if to == torch.int64: + return onnx.TensorProto.INT64 + if to == torch.int32: + return onnx.TensorProto.INT32 + if to == torch.uint64: + return onnx.TensorProto.UINT64 + if to == torch.uint32: + return onnx.TensorProto.UINT32 + if to == torch.bool: + return onnx.TensorProto.BOOL + if to == torch.SymInt: + return onnx.TensorProto.INT64 + if to == torch.int16: + return onnx.TensorProto.INT16 + if to == torch.uint16: + return onnx.TensorProto.UINT16 + if to == torch.int8: + return onnx.TensorProto.INT8 + if to == torch.uint8: + return onnx.TensorProto.UINT8 + if to == torch.SymFloat: + return onnx.TensorProto.FLOAT + if to == torch.complex64: + return onnx.TensorProto.COMPLEX64 + if to == torch.complex128: + return onnx.TensorProto.COMPLEX128 + raise NotImplementedError(f"Unable to convert torch dtype {to!r} to onnx dtype.") diff --git a/optimum/torch_export_patches/_verbose.py b/optimum/torch_export_patches/_verbose.py new file mode 100644 index 00000000..a21f5490 --- /dev/null +++ b/optimum/torch_export_patches/_verbose.py @@ -0,0 +1,686 @@ +import functools +from dataclasses import fields, is_dataclass +from typing import Any + +import numpy as np +import onnx +from transformers import cache_utils, configuration_utils + +from optimum.torch_export_patches import _cache_creater, _type_utils + + +@functools.cache +def onnx_dtype_name(itype: int, exc: bool = True) -> str: + """Returns the ONNX name for a specific element type. + + .. runpython:: + :showcode: + + import onnx + from optimum.torch_export_patches import _verbose + + itype = onnx.TensorProto.BFLOAT16 + print(_verbose.onnx_dtype_name(itype)) + print(_verbose.onnx_dtype_name(7)) + """ + for k in dir(onnx.TensorProto): + if "FLOAT" in k or "INT" in k or "TEXT" in k or "BOOL" in k: + v = getattr(onnx.TensorProto, k) + if v == itype: + return k + if exc: + raise ValueError(f"Unexpected value itype: {itype}") + if itype == 0: + return "UNDEFINED" + return "UNEXPECTED" + + +def string_type( + obj: Any, + with_shape: bool = False, + with_min_max: bool = False, + with_device: bool = False, + ignore: bool = False, + limit: int = 20, + verbose: int = 0, +) -> str: + """Displays the types of an object as a string. + + The function displays something like the following for a tensor: + + T7s2x7[0.5:6:A3.56] + ^^^+-^^----+------^ + || | | + || | +-- information about the content of a tensor or array + || | [min,max:A] + || | + || +-- a shape + || + |+-- integer following the code defined by onnx.TensorProto, + | 7 is onnx.TensorProto.INT64 (see onnx_dtype_name) + | + +-- A,T,F + A is an array from numpy + T is a Tensor from pytorch + F is a FakeTensor from pytorch + + The element types for a tensor are displayed as integer to shorten the message. + The semantic is defined by :class:`onnx.TensorProto` and can be obtained + by :func:`onnx_diagnostic.helpers.onnx_helper.onnx_dtype_name`. + + Examples: + Basic usage: + >>> from optimum.torch_export_patches import _verbose + >>> print(_verbose.string_type((1, ["r", 6.6]))) + + With pytorch: + >>> import torch + >>> from optimum.torch_export_patches import _verbose + >>> inputs = ( + ... torch.rand((3, 4), dtype=torch.float16), + ... [ + ... torch.rand((5, 6), dtype=torch.float16), + ... torch.rand((5, 6, 7), dtype=torch.float16), + ... ] + ... ) + >>> # with shapes + >>> print(_verbose.string_type(inputs, with_shape=True)) + >>> # with min max + >>> print(_verbose.string_type(inputs, with_shape=True, with_min_max=True)) + + Args: + obj: The object to analyze and display type information for. + with_shape: If True, displays shapes as well. + with_min_max: If True, displays information about the values. + with_device: If True, display the device. + ignore: If True, just prints the type for unknown types. + limit: Maximum number of elements to display for collections. + verbose: Verbosity level (to show the path it followed to get that print). + + Returns: + A string representation of the object's type and optionally its shape, + min/max values, and device information. + """ + if obj is None: + if verbose: + print(f"[string_type] A:{type(obj)}") + return "None" + + # tuple + if isinstance(obj, tuple): + if len(obj) == 1: + s = string_type( + obj[0], + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + ignore=ignore, + limit=limit, + verbose=verbose, + ) + if verbose: + print(f"[string_type] C:{type(obj)}") + return f"({s},)" + if len(obj) < limit: + js = ",".join( + string_type( + o, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + ignore=ignore, + limit=limit, + verbose=verbose, + ) + for o in obj + ) + if verbose: + print(f"[string_type] D:{type(obj)}") + return f"({js})" + tt = string_type( + obj[0], + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + ignore=ignore, + limit=limit, + verbose=verbose, + ) + if with_min_max and all(isinstance(_, (int, float, bool)) for _ in obj): + mini, maxi, avg = min(obj), max(obj), sum(float(_) for _ in obj) / len(obj) + if verbose: + print(f"[string_type] E:{type(obj)}") + return f"#{len(obj)}({tt},...)[{mini},{maxi}:A[{avg}]]" + if verbose: + print(f"[string_type] F:{type(obj)}") + return f"#{len(obj)}({tt},...)" + # list + if isinstance(obj, list): + if len(obj) < limit: + js = ",".join( + string_type( + o, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + ignore=ignore, + limit=limit, + verbose=verbose, + ) + for o in obj + ) + if verbose: + print(f"[string_type] G:{type(obj)}") + return f"#{len(obj)}[{js}]" + tt = string_type( + obj[0], + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + ignore=ignore, + limit=limit, + verbose=verbose, + ) + if with_min_max and all(isinstance(_, (int, float, bool)) for _ in obj): + mini, maxi, avg = min(obj), max(obj), sum(float(_) for _ in obj) / len(obj) + if verbose: + print(f"[string_type] H:{type(obj)}") + return f"#{len(obj)}[{tt},...][{mini},{maxi}:{avg}]" + if verbose: + print(f"[string_type] I:{type(obj)}") + return f"#{len(obj)}[{tt},...]" + # set + if isinstance(obj, set): + if len(obj) < 10: + js = ",".join( + string_type( + o, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + ignore=ignore, + limit=limit, + verbose=verbose, + ) + for o in obj + ) + if verbose: + print(f"[string_type] J:{type(obj)}") + return f"{{{js}}}" + if with_min_max and all(isinstance(_, (int, float, bool)) for _ in obj): + mini, maxi, avg = min(obj), max(obj), sum(float(_) for _ in obj) / len(obj) + if verbose: + print(f"[string_type] K:{type(obj)}") + return f"{{...}}#{len(obj)}[{mini},{maxi}:A{avg}]" + if verbose: + print(f"[string_type] L:{type(obj)}") + return f"{{...}}#{len(obj)}" if with_shape else "{...}" + # dict + if isinstance(obj, dict) and type(obj) is dict: + if len(obj) == 0: + if verbose: + print(f"[string_type] M:{type(obj)}") + return "{}" + + import torch + + if all(isinstance(k, int) for k in obj) and all( + isinstance( + v, + ( + str, + torch.export.dynamic_shapes._Dim, + torch.export.dynamic_shapes._DerivedDim, + torch.export.dynamic_shapes._DimHint, + ), + ) + for v in obj.values() + ): + # This is dynamic shapes + rows = [] + for k, v in obj.items(): + if isinstance(v, str): + rows.append(f"{k}:DYN({v})") + else: + rows.append(f"{k}:{string_type(v, verbose=verbose)}") + if verbose: + print(f"[string_type] DS0:{type(obj)}") + return f"{{{','.join(rows)}}}" + + kws = { + "with_shape": with_shape, + "with_min_max": with_min_max, + "with_device": with_device, + "ignore": ignore, + "limit": limit, + "verbose": verbose, + } + s = ",".join(f"{kv[0]}:{string_type(kv[1], **kws)}" for kv in obj.items()) + if all(isinstance(k, int) for k in obj): + if verbose: + print(f"[string_type] N:{type(obj)}") + return f"{{{s}}}" + if verbose: + print(f"[string_type] O:{type(obj)}") + return f"dict({s})" + # array + if isinstance(obj, np.ndarray): + if with_min_max: + s = string_type(obj, with_shape=with_shape) + if len(obj.shape) == 0: + return f"{s}={obj}" + if obj.size == 0: + return f"{s}[empty]" + n_nan = np.isnan(obj.reshape((-1,))).astype(int).sum() + if n_nan > 0: + nob = obj.ravel() + nob = nob[~np.isnan(nob)] + if nob.size == 0: + if verbose: + print(f"[string_type] A1:{type(obj)}") + return f"{s}[N{n_nan}nans]" + if verbose: + print(f"[string_type] A2:{type(obj)}") + return f"{s}[{nob.min()},{nob.max()}:A{nob.astype(float).mean()}N{n_nan}nans]" + if verbose: + print(f"[string_type] A3:{type(obj)}") + return f"{s}[{obj.min()},{obj.max()}:A{obj.astype(float).mean()}]" + i = _type_utils.np_dtype_to_tensor_dtype(obj.dtype) + if not with_shape: + if verbose: + print(f"[string_type] A4:{type(obj)}") + return f"A{i}r{len(obj.shape)}" + if verbose: + print(f"[string_type] A5:{type(obj)}") + return f"A{i}s{'x'.join(map(str, obj.shape))}" + + import torch + + # Dim, SymInt + if isinstance(obj, torch.export.dynamic_shapes._DerivedDim): + if verbose: + print(f"[string_type] Y1:{type(obj)}") + return "DerivedDim" + if isinstance(obj, torch.export.dynamic_shapes._Dim): + if verbose: + print(f"[string_type] Y2:{type(obj)}") + return f"Dim({obj.__name__})" + if isinstance(obj, torch.SymInt): + if verbose: + print(f"[string_type] Y3:{type(obj)}") + return "SymInt" + if isinstance(obj, torch.SymFloat): + if verbose: + print(f"[string_type] Y4:{type(obj)}") + return "SymFloat" + + if isinstance(obj, torch.export.dynamic_shapes._DimHint): + cl = ( + torch.export.dynamic_shapes._DimHintType + if hasattr(torch.export.dynamic_shapes, "_DimHintType") + else torch.export.Dim + ) + if obj in (torch.export.Dim.DYNAMIC, cl.DYNAMIC): + if verbose: + print(f"[string_type] Y8:{type(obj)}") + return "DYNAMIC" + if obj in (torch.export.Dim.AUTO, cl.AUTO): + if verbose: + print(f"[string_type] Y9:{type(obj)}") + return "AUTO" + if verbose: + print(f"[string_type] Y7:{type(obj)}") + return str(obj) + + if isinstance(obj, bool): + if with_min_max: + if verbose: + print(f"[string_type] W1:{type(obj)}") + return f"bool={obj}" + if verbose: + print(f"[string_type] W2:{type(obj)}") + return "bool" + if isinstance(obj, int): + if with_min_max: + if verbose: + print(f"[string_type] W3:{type(obj)}") + return f"int={obj}" + if verbose: + print(f"[string_type] W4:{type(obj)}") + return "int" + if isinstance(obj, float): + if with_min_max: + if verbose: + print(f"[string_type] W6:{type(obj)}") + return f"float={obj}" + if verbose: + print(f"[string_type] W8:{type(obj)}") + return "float" + if isinstance(obj, str): + if verbose: + print(f"[string_type] W9:{type(obj)}") + return "str" + if isinstance(obj, slice): + if verbose: + print(f"[string_type] W10:{type(obj)}") + return "slice" + + if is_dataclass(obj): + # That includes torch.export.Dim.AUTO, torch.export.Dim.DYNAMIC so they need to be + # handled before that. + values = {f.name: getattr(obj, f.name, None) for f in fields(obj)} + values = {k: v for k, v in values.items() if v is not None} + s = string_type( + values, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + ignore=ignore, + limit=limit, + verbose=verbose, + ) + if verbose: + print(f"[string_type] B:{type(obj)}") + return f"{obj.__class__.__name__}{s[4:]}" + + # Tensors + if isinstance(obj, torch._subclasses.fake_tensor.FakeTensor): + i = _type_utils.torch_dtype_to_onnx_dtype(obj.dtype) + prefix = ("G" if obj.get_device() >= 0 else "C") if with_device else "" + if not with_shape: + if verbose: + print(f"[string_type] F1:{type(obj)}") + return f"{prefix}F{i}r{len(obj.shape)}" + if verbose: + print(f"[string_type] F2:{type(obj)}") + return f"{prefix}F{i}s{'x'.join(map(str, obj.shape))}" + if isinstance(obj, torch.Tensor): + if with_min_max: + s = string_type(obj, with_shape=with_shape, with_device=with_device) + if len(obj.shape) == 0: + if verbose: + print(f"[string_type] T1:{type(obj)}") + return f"{s}={obj}" + if obj.numel() == 0: + if verbose: + print(f"[string_type] T2:{type(obj)}") + return f"{s}[empty]" + n_nan = obj.reshape((-1,)).isnan().to(int).sum() + if n_nan > 0: + nob = obj.reshape((-1,)) + nob = nob[~nob.isnan()] + if obj.dtype in {torch.complex64, torch.complex128}: + if verbose: + print(f"[string_type] T3:{type(obj)}") + return f"{s}[{nob.abs().min()},{nob.abs().max():A{nob.mean()}N{n_nan}nans}]" + if verbose: + print(f"[string_type] T5:{type(obj)}") + return f"{s}[{obj.min()},{obj.max()}:A{obj.to(float).mean()}N{n_nan}nans]" + if obj.dtype in {torch.complex64, torch.complex128}: + if verbose: + print(f"[string_type] T6:{type(obj)}") + return f"{s}[{obj.abs().min()},{obj.abs().max()}:A{obj.abs().mean()}]" + if verbose: + print(f"[string_type] T7:{type(obj)}") + return f"{s}[{obj.min()},{obj.max()}:A{obj.to(float).mean()}]" + i = _type_utils.torch_dtype_to_onnx_dtype(obj.dtype) + prefix = ("G" if obj.get_device() >= 0 else "C") if with_device else "" + if not with_shape: + if verbose: + print(f"[string_type] T8:{type(obj)}") + return f"{prefix}T{i}r{len(obj.shape)}" + if verbose: + print(f"[string_type] T9:{type(obj)}") + return f"{prefix}T{i}s{'x'.join(map(str, obj.shape))}" + + if obj.__class__.__name__ == "OrtValue": + if not obj.has_value(): + if verbose: + print(f"[string_type] V1:{type(obj)}") + return "OV()" + if not obj.is_tensor(): + if verbose: + print(f"[string_type] V2:{type(obj)}") + return "OV(NOTENSOR)" + if with_min_max: + try: + t = obj.numpy() + except Exception: + # pass unable to convert into numpy (bfloat16, ...) + if verbose: + print(f"[string_type] V3:{type(obj)}") + return "OV(NO-NUMPY:FIXIT)" + if verbose: + print(f"[string_type] V4:{type(obj)}") + return f"OV({string_type(t, with_shape=with_shape, with_min_max=with_min_max)})" + dt = obj.element_type() + shape = obj.shape() + if with_shape: + if verbose: + print(f"[string_type] V5:{type(obj)}") + return f"OV{dt}s{'x'.join(map(str, shape))}" + if verbose: + print(f"[string_type] V6:{type(obj)}") + return f"OV{dt}r{len(shape)}" + + # others classes + + if obj.__class__.__name__ == "MambaCache": + c = string_type( + obj.conv_states, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + d = string_type( + obj.ssm_states, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + if verbose: + print(f"[string_type] CACHE1:{type(obj)}") + return f"MambaCache(conv_states={c}, ssm_states={d})" + + if obj.__class__.__name__ in { + "DynamicCache", + "SlidingWindowCache", + "StaticCache", + "HybridCache", + }: + ca = _cache_creater.CacheKeyValue(obj) + kc = string_type( + ca.key_cache, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + vc = string_type( + ca.value_cache, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + if verbose: + print(f"[string_type] CACHE2:{type(obj)}") + return f"{obj.__class__.__name__}(key_cache={kc}, value_cache={vc})" + + if obj.__class__.__name__ == "StaticLayer": + kc = string_type( + list(obj.keys), + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + vc = string_type( + list(obj.values), + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + if verbose: + print(f"[string_type] SL:{type(obj)}") + return f"{obj.__class__.__name__}(keys={kc}, values={vc})" + + if obj.__class__.__name__ == "EncoderDecoderCache": + att = string_type( + obj.self_attention_cache, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + cross = string_type( + obj.cross_attention_cache, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + if verbose: + print(f"[string_type] CACHE3:{type(obj)}") + return f"{obj.__class__.__name__}(self_attention_cache={att}, cross_attention_cache={cross})" + + if obj.__class__ in torch.utils._pytree.SUPPORTED_NODES: + args = _cache_creater.flatten_unflatten_for_dynamic_shapes(obj) + att = string_type( + args, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + if verbose: + print(f"[string_type] DS:{type(obj)}") + return f"{obj.__class__.__name__}[serialized]({att})" + + if type(obj).__name__ == "Node" and hasattr(obj, "meta"): + # torch.fx.node.Node + if verbose: + print(f"[string_type] TT1:{type(obj)}") + return f"%{obj.target}" + if type(obj).__name__ == "ValueInfoProto": + if verbose: + print(f"[string_type] OO1:{type(obj)}") + return f"OT{obj.type.tensor_type.elem_type}" + + if obj.__class__.__name__ == "BatchFeature": + s = string_type( + obj.data, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + if verbose: + print(f"[string_type] TT2:{type(obj)}") + return f"BatchFeature(data={s})" + + if obj.__class__.__name__ == "BatchEncoding": + s = string_type( + obj.data, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + if verbose: + print(f"[string_type] TT3:{type(obj)}") + return f"BatchEncoding(data={s})" + + if obj.__class__.__name__ == "VirtualTensor": + if verbose: + print(f"[string_type] TT4:{type(obj)}") + return f"{obj.__class__.__name__}(name={obj.name!r}, dtype={obj.dtype}, shape={obj.shape})" + + if obj.__class__.__name__ == "KeyValuesWrapper": + assert isinstance(obj, cache_utils.KeyValuesWrapper), f"Unexpected type {type(obj)}" + if verbose: + print(f"[string_type] KW0:{type(obj)}") + s = string_type( + list(obj), + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + return f"{obj.__class__.__name__}[{obj.cache_type}]{s}" + + if obj.__class__.__name__ == "DynamicLayer": + assert isinstance(obj, cache_utils.DynamicLayer), f"Unexpected type {type(obj)}" + if verbose: + print(f"[string_type] LY0:{type(obj)}") + s1 = string_type( + obj.keys, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + s2 = string_type( + obj.values, + with_shape=with_shape, + with_min_max=with_min_max, + with_device=with_device, + limit=limit, + verbose=verbose, + ) + return f"{obj.__class__.__name__}(keys={s1}, values={s2})" + + if isinstance(obj, torch.nn.Module): + if verbose: + print(f"[string_type] MM:{type(obj)}") + return f"{obj.__class__.__name__}(...)" + + if isinstance(obj, (torch.device, torch.dtype, torch.memory_format, torch.layout)): + if verbose: + print(f"[string_type] TT7:{type(obj)}") + return f"{obj.__class__.__name__}({obj})" + + if isinstance( # TreeSpec, MappingKey, SequenceKey + obj, + ( + torch.utils._pytree.TreeSpec, + torch.utils._pytree.MappingKey, + torch.utils._pytree.SequenceKey, + ), + ): + if verbose: + print(f"[string_type] TT8:{type(obj)}") + return repr(obj).replace(" ", "").replace("\n", " ") + + if ignore: + if verbose: + print(f"[string_type] CACHE4:{type(obj)}") + return f"{obj.__class__.__name__}(...)" + + if obj.__class__.__name__.endswith("Config"): + if isinstance(obj, configuration_utils.PretrainedConfig): + if verbose: + print(f"[string_type] CONFIG:{type(obj)}") + s = str(obj.to_diff_dict()).replace("\n", "").replace(" ", "") + return f"{obj.__class__.__name__}(**{s})" + if obj.__class__.__name__ in {"TorchModelContainer", "InferenceSession"}: + return f"{obj.__class__.__name__}(...)" + if verbose: + print(f"[string_type] END:{type(obj)}") + raise AssertionError(f"Unsupported type {type(obj).__name__!r} - {type(obj)}") diff --git a/tests/patch/test_torch_export_patches.py b/tests/patch/test_torch_export_patches.py index 1a198252..e610fa03 100644 --- a/tests/patch/test_torch_export_patches.py +++ b/tests/patch/test_torch_export_patches.py @@ -1,9 +1,118 @@ import unittest +from typing import Any + +import numpy as np from optimum.torch_export_patches._core import torch_export_patches class TestOnnxExportErrors(unittest.TestCase): + def assertEqualArrayAny(self, expected: Any, value: Any, atol: float = 0, rtol: float = 0, msg: str = ""): # noqa: N802 + if isinstance(expected, (tuple, list, dict)): + self.assertIsInstance(value, type(expected), msg=msg) + self.assertEqual(len(expected), len(value), msg=msg) + if isinstance(expected, dict): + for k in expected: + self.assertIn(k, value, msg=msg) + self.assertEqualArrayAny(expected[k], value[k], msg=msg, atol=atol, rtol=rtol) + else: + excs = [] + for i, (e, g) in enumerate(zip(expected, value)): + try: + self.assertEqualArrayAny(e, g, msg=msg, atol=atol, rtol=rtol) + except AssertionError as e: + excs.append(f"Error at position {i} due to {e}") + if excs: + msg_ = "\n".join(excs) + msg = f"{msg}\n{msg_}" if msg else msg_ + raise AssertionError(f"Found {len(excs)} discrepancies\n{msg}") + elif expected.__class__.__name__ in ("DynamicCache", "StaticCache"): + atts = {"key_cache", "value_cache"} + self.assertEqualArrayAny( + {k: expected.__dict__.get(k, None) for k in atts}, + {k: value.__dict__.get(k, None) for k in atts}, + atol=atol, + rtol=rtol, + ) + elif isinstance(expected, (int, float, str)): + self.assertEqual(expected, value, msg=msg) + elif hasattr(expected, "shape"): + self.assertEqual(type(expected), type(value), msg=msg) + self.assertEqualArray(expected, value, msg=msg, atol=atol, rtol=rtol) + elif expected is None: + assert value is None, f"Expected is None but value is of type {type(value)}" + else: + raise AssertionError(f"Comparison not implemented for types {type(expected)} and {type(value)}") + + def assertEqualArray( # noqa: N802 + self, + expected: Any, + value: Any, + atol: float = 0, + rtol: float = 0, + msg: str | None = None, + ): + if hasattr(expected, "detach") and hasattr(value, "detach"): + if msg: + try: + self.assertEqual(expected.dtype, value.dtype) + except AssertionError as e: + raise AssertionError(msg) from e + try: + self.assertEqual(expected.shape, value.shape) + except AssertionError as e: + raise AssertionError(msg) from e + else: + self.assertEqual(expected.dtype, value.dtype) + self.assertEqual(expected.shape, value.shape) + + import torch + + try: + torch.testing.assert_close(value, expected, atol=atol, rtol=rtol) + except AssertionError as e: + expected_max = torch.abs(expected).max() + expected_value = torch.abs(value).max() + rows = [ + f"{msg}\n{e}" if msg else str(e), + f"expected max value={expected_max}", + f"expected computed value={expected_value}", + ] + raise AssertionError("\n".join(rows)) + return + + if hasattr(expected, "detach"): + expected = expected.detach().cpu().numpy() + if hasattr(value, "detach"): + value = value.detach().cpu().numpy() + if msg: + try: + self.assertEqual(expected.dtype, value.dtype) + except AssertionError as e: + raise AssertionError(msg) from e + try: + self.assertEqual(expected.shape, value.shape) + except AssertionError as e: + raise AssertionError(msg) from e + else: + self.assertEqual(expected.dtype, value.dtype) + self.assertEqual(expected.shape, value.shape) + + try: + np.testing.assert_allclose(desired=expected, actual=value, atol=atol, rtol=rtol) + except AssertionError as e: + expected_max = np.abs(expected).max() + expected_value = np.abs(value).max() + te = expected.astype(int) if expected.dtype == np.bool_ else expected + tv = value.astype(int) if value.dtype == np.bool_ else value + rows = [ + f"{msg}\n{e}" if msg else str(e), + f"expected max value={expected_max}", + f"expected computed value={expected_value}\n", + f"ratio={te / tv}\ndiff={te - tv}", + ] + raise AssertionError("\n".join(rows)) + # @skipif_ci_windows("not working on Windows") def test_pytree_flatten_mamba_cache(self): import torch From 59106c3682a75615221c34c0b72ded8e02b3a0d7 Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Mon, 28 Jul 2025 23:58:58 +0000 Subject: [PATCH 05/10] bump ci env to 3.10 --- .github/workflows/test_patch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_patch.yml b/.github/workflows/test_patch.yml index 2d8391ee..facb3587 100644 --- a/.github/workflows/test_patch.yml +++ b/.github/workflows/test_patch.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.9] + python-version: ['3.10'] runs-on: [ubuntu-22.04] runs-on: ${{ matrix.runs-on }} From f00e6a2fdf946df3c03cb00ab8e790c99d8c87d2 Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Tue, 29 Jul 2025 23:50:47 +0000 Subject: [PATCH 06/10] pip install optree --- .github/workflows/test_patch.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_patch.yml b/.github/workflows/test_patch.yml index facb3587..3a96dcc4 100644 --- a/.github/workflows/test_patch.yml +++ b/.github/workflows/test_patch.yml @@ -37,7 +37,8 @@ jobs: pip install --upgrade pip pip install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu pip install .[tests,onnxruntime] diffusers - + pip install optree + - name: Test with pytest run: | pytest tests/patch/test_torch_export_patches.py -n auto -vvvv --durations=0 From 8108f7eda708a71b9457169944fb1a9f4daece4e Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Wed, 30 Jul 2025 00:21:36 +0000 Subject: [PATCH 07/10] set torch nightly on CI --- .github/workflows/test_patch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_patch.yml b/.github/workflows/test_patch.yml index 3a96dcc4..63e1b55d 100644 --- a/.github/workflows/test_patch.yml +++ b/.github/workflows/test_patch.yml @@ -35,10 +35,10 @@ jobs: - name: Install dependencies run: | pip install --upgrade pip - pip install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + pip install --no-cache-dir --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu pip install .[tests,onnxruntime] diffusers pip install optree - + - name: Test with pytest run: | pytest tests/patch/test_torch_export_patches.py -n auto -vvvv --durations=0 From 019f634999debeb30e1c0b43134680c1e31c1a1c Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Wed, 30 Jul 2025 21:12:22 +0000 Subject: [PATCH 08/10] refactor tests --- .../{_cache_creater.py => _cache_creator.py} | 16 +- optimum/torch_export_patches/_patch_torch.py | 259 ------------ .../_treenode_registry.py | 2 +- .../_treenode_transformers.py | 18 +- optimum/torch_export_patches/_verbose.py | 8 +- tests/patch/test_torch_export_patches.py | 382 ++++++++++++------ 6 files changed, 289 insertions(+), 396 deletions(-) rename optimum/torch_export_patches/{_cache_creater.py => _cache_creator.py} (97%) diff --git a/optimum/torch_export_patches/_cache_creater.py b/optimum/torch_export_patches/_cache_creator.py similarity index 97% rename from optimum/torch_export_patches/_cache_creater.py rename to optimum/torch_export_patches/_cache_creator.py index 4e0f46b7..810426f2 100644 --- a/optimum/torch_export_patches/_cache_creater.py +++ b/optimum/torch_export_patches/_cache_creator.py @@ -78,8 +78,8 @@ def make_dynamic_cache( :showcode: import torch - from onnx_diagnostic.helpers import string_type - from onnx_diagnostic.helpers.cache_helper import make_dynamic_cache + from optimum.torch_export_patches._verbose import string_type + from optimum.torch_export_patches._cache_creator import make_dynamic_cache n_layers = 2 bsize, nheads, slen, dim = 2, 4, 3, 7 @@ -128,8 +128,8 @@ def make_dynamic_cache( :showcode: import torch - from onnx_diagnostic.helpers import string_type - from onnx_diagnostic.helpers.cache_helper import make_dynamic_cache + from optimum.torch_export_patches._verbose import string_type + from optimum.torch_export_patches._cache_creator import make_dynamic_cache n_layers = 2 bsize, nheads, slen, dim = 2, 4, 3, 7 @@ -169,8 +169,8 @@ def make_static_cache( :showcode: import torch - from onnx_diagnostic.helpers import string_type - from onnx_diagnostic.helpers.cache_helper import make_static_cache + from optimum.torch_export_patches._verbose import string_type + from optimum.torch_export_patches._cache_creator import make_static_cache n_layers = 2 bsize, nheads, slen, dim = 2, 4, 3, 7 @@ -367,8 +367,8 @@ def make_hybrid_cache( :showcode: import torch - from onnx_diagnostic.helpers import string_type - from onnx_diagnostic.helpers.cache_helper import make_hybrid_cache + from optimum.torch_export_patches._verbose import string_type + from optimum.torch_export_patches._cache_creator import make_hybrid_cache n_layers = 2 bsize, nheads, slen, dim = 2, 4, 3, 7 diff --git a/optimum/torch_export_patches/_patch_torch.py b/optimum/torch_export_patches/_patch_torch.py index fb90a725..04d1bfb8 100644 --- a/optimum/torch_export_patches/_patch_torch.py +++ b/optimum/torch_export_patches/_patch_torch.py @@ -4,66 +4,6 @@ import torch -def catch_produce_guards_and_solve_constraints( - previous_function: Callable, - fake_mode: "FakeTensorMode", - gm: "torch.fx.GraphModule", - dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None, - equalities_inputs: "EqualityConstraint", - original_signature: inspect.Signature, - _is_torch_jit_trace: bool = False, - verbose: int = 0, -): - try: - return previous_function( - fake_mode=fake_mode, - gm=gm, - dynamic_shapes=dynamic_shapes, - equalities_inputs=equalities_inputs, - original_signature=original_signature, - _is_torch_jit_trace=_is_torch_jit_trace, - ) - except Exception as e: - if not int(os.environ.get("SKIP_SOLVE_CONSTRAINTS", "1")): - raise - if verbose: - print( - f"[_catch_produce_guards_and_solve_constraints] ERROR: " - f"produce_guards_and_solve_constraints failed, " - f"use SKIP_SOLVE_CONSTRAINTS=0 to avoid skipping\n" - f"fake_mode={fake_mode}\n" - f"dynamic_shapes={dynamic_shapes}\n" - f"equalities_inputs={equalities_inputs}\n" - f"original_signature={original_signature}\n" - f"_is_torch_jit_trace={_is_torch_jit_trace}\n" - f"exc={e}\ngm={gm}" - ) - torch._dynamo.reset() - - -def patch__check_input_constraints_for_graph( - previous_function: Callable, - input_placeholders: list[torch.fx.Node], - flat_args_with_path, - range_constraints, - verbose: int = 0, -) -> None: - try: - return previous_function(input_placeholders, flat_args_with_path, range_constraints) - except Exception as e: - if not int(os.environ.get("SKIP_SOLVE_CONSTRAINTS", "1")): - raise - if verbose: - print( - f"[_check_input_constraints_for_graph] ERROR: " - f"_check_input_constraints_for_graph failed, " - f"use SKIP_SOLVE_CONSTRAINTS=0 to avoid skipping\n" - f"input_placeholders={input_placeholders}\n" - f"range_constraints={range_constraints}\n" - f"exc={e}" - ) - torch._dynamo.reset() - def patched_infer_size(a, b): """Patches ``torch._subclasses.fake_impls.infer_size``.""" @@ -148,205 +88,6 @@ def patched__broadcast_shapes(*_shapes): return common_shape - -class patched_ShapeEnv: - - def _check_frozen( - self, expr: "sympy.Basic", concrete_val: "sympy.Basic" # noqa: F821 - ) -> None: - if self.frozen: - self.counter["ignored_backward_guard"] += 1 - raise AssertionError( - f"[patched_ShapeEnv] Ignored guard {expr} == {concrete_val}, " - f"this could result in accuracy problems." - ) - - def _set_replacement( - self, a: "sympy.Symbol", tgt: "sympy.Expr", msg: str # noqa: F821 - ) -> None: - """ - Adds or updates a replacement for a symbol. - Use this instead of `self.replacements[a] = tgt`. - """ - if tgt == self.replacements.get(a, None): - return - - if a in tgt.free_symbols: - return - - import sympy - from torch._logging import structured - from torch.utils._traceback import CapturedTraceback - from torch._logging import trace_structured - from torch._guards import TracingContext - from torch.utils._sympy.functions import FloorToInt, CeilToInt - from torch.utils._sympy.solve import try_solve - from torch.fx.experimental.symbolic_shapes import ( - _is_supported_equivalence, - ValueRanges, - ) - - # Precondition: a == tgt - assert isinstance(a, sympy.Symbol) - - if self.allow_complex_guards_as_runtime_asserts and not _is_supported_equivalence(tgt): - # continuing leads to placeholder shapes - # having complex expressions that we can't resolve - return - - # Handles nested tensor symbolic variables which don't have - # var_to_range bounds - tgt_bound = None - if a in self.var_to_range: - src_bound = self.var_to_range[a] - - # First, refine the value range of a based on the computed value range - # of tgt. This is always OK to do, even if we decide not to do the - # substitution in the end. This might be a no-op, if a already has - # a tighter bound - tgt_bound = self.bound_sympy(tgt) - self._update_var_to_range(a, tgt_bound) - - # Next, check if we can update the range of free symbols in tgt - # based on the range in a. But only do it if: - # - the source bound non-trivially improves over what we get out of - # the existing bounds. - # - the replacement is univariate and we can invert the tgt expression - if not tgt_bound.issubset(src_bound) and len(tgt.free_symbols) == 1: - b = next(iter(tgt.free_symbols)) - # Try to invert the equality - r = try_solve(sympy.Eq(a, tgt), b, floordiv_inequality=False) - if r is not None: - self.log.debug( - "set_replacement: solve for %s in %s == %s gives %s", - b, - a, - tgt, - r, - ) - # The solution here can be non-integral, for example, if - # we have s0 = 2*s1, then s1 = s0/2. What we would like - # to do is calculated the bounds in arbitrary precision, - # and then requantize the bound to integers when we are - # done. - rat_b_bound = self.bound_sympy(r[1]) - b_bound = ValueRanges( - CeilToInt(rat_b_bound.lower), FloorToInt(rat_b_bound.upper) - ) - self._update_var_to_range(b, b_bound, self.var_to_range_sloc[a]) - tgt_bound = self.bound_sympy(tgt) - assert tgt_bound.issubset( - src_bound - ), f"{tgt_bound=} not a subset of {src_bound=}" - - # TODO: Should we propagate size-like-ness? - # - # Pros: if u0 is size-like, intuitively u0 == u1 should cause u1 - # to become size-like. - # - # Cons: if u0 is size-like, what about u0 - 1 == u1? You CAN'T - # propagate in this case, because what if u0 == 0, then u1 is negative - # and clearly isn't a size. So, at minimum, any f(x) whose value - # range isn't [0, inf] given x in [0, inf] cannot propagate - # size-like-ness. But there are many situations where you could - # imagine u1 is going to be size-like and actually you just didn't - # have a refined enough value range on u0. Since even innocuous - # looking arithmetic operations can destroy size-like-ness, it's - # best to not propagate it at all and force the user to annotate it - # as necessary. - # - # Compromise: we preserve size-like-ness only for exact equality - # and nothing else. - if a in self.size_like and isinstance(tgt, sympy.Symbol): - self.size_like.add(tgt) - elif isinstance(tgt, sympy.Symbol) and tgt in self.size_like: - self.size_like.add(a) - - # Now, decide if we will do the substitution. - # - # - If the source has a non-trivial range, only substitute if - # we preserve this range. Note that we may have propagated - # the src_range to free variables in tgt when tgt is univariate - # and we could find an inverse, which helps us achieve this. - # This ensures we never "forget" about user defined ranges, - # even if they end up being defined on composite formulas - # like s0 + s1. - # - # - If the variable is unbacked, only substitute if the substitution - # would preserve the bounds also under size-like-ness conditions. - - if not tgt_bound.issubset(src_bound): - self.log.debug( - "skipped set_replacement %s = %s (%s) [%s not subset of %s]", - a, - tgt, - msg, - tgt_bound, - src_bound, - ) - return - elif a in self.size_like: - tgt_bound_so = self.bound_sympy(tgt, size_oblivious=True) - src_bound_so = self.bound_sympy(a, size_oblivious=True) - if not tgt_bound_so.issubset(src_bound_so): - self.log.debug( - "skipped set_replacement %s = %s (%s) " - "[%s not subset of %s (size-oblivious conditions)]", - a, - tgt, - msg, - tgt_bound_so, - src_bound_so, - ) - return - - if isinstance(tgt, (sympy.Integer, sympy.Float)): - # specializing to a constant, which is likely unexpected (unless - # you specified dynamic=True) - - user_tb = TracingContext.extract_stack() - trace_structured( - "symbolic_shape_specialization", - metadata_fn=lambda: { - "symbol": repr(a), - "sources": [s.name() for s in self.var_to_sources.get(a, [])], - "value": repr(tgt), - "reason": msg, - "stack": structured.from_traceback( - CapturedTraceback.extract(skip=1).summary() - ), - "user_stack": (structured.from_traceback(user_tb) if user_tb else None), - }, - ) - - # if config.print_specializations: - # self.log.warning( - # "Specializing %s to %s", self.var_to_sources[a][0].name(), tgt - # ) - # self.log.debug("SPECIALIZATION", stack_info=True) - assert msg != "range_refined_to_singleton", ( - f"patched_ShapeEnv: A dynamic dimension becomes static! " - f"a={a!r}, tgt={tgt!r}, msg={msg!r}, tgt_bound={tgt_bound}" - ) - # log.info("set_replacement %s = %s (%s) %s", a, tgt, msg, tgt_bound) - self.replacements[a] = tgt - # NB: the replacement may get refined, but the user will find the - # FIRST one most useful (TODO: Maybe we could consider tracking all of - # them) - if a not in self.replacements_slocs: - self.replacements_slocs[a] = self._get_sloc() - self._update_version_counter() - - # When specializing 'a == tgt', the equality should be also conveyed to - # Z3, in case an expression uses 'a'. - self._add_target_expr(sympy.Eq(a, tgt, evaluate=False)) - - def _log_guard( - self, prefix: str, g: "SympyBoolean", forcing_spec: bool # noqa: F821 - ) -> None: - self._log_guard_remember(prefix=prefix, g=g, forcing_spec=forcing_spec) - - def patched_vmap(func, in_dims=0, out_dims=0): """Python implementation of :func:`torch.vmap`. diff --git a/optimum/torch_export_patches/_treenode_registry.py b/optimum/torch_export_patches/_treenode_registry.py index cafb4de3..35747067 100644 --- a/optimum/torch_export_patches/_treenode_registry.py +++ b/optimum/torch_export_patches/_treenode_registry.py @@ -31,7 +31,7 @@ def register_class_serialization( """Registers a class. It can be undone with - :func:`onnx_diagnostic.torch_export_patches.onnx_export_serialization.unregister_class_serialization`. + :func:`optimum.torch_export_patches._treenode_registry.unregister_class_serialization`. Args: cls: class to register diff --git a/optimum/torch_export_patches/_treenode_transformers.py b/optimum/torch_export_patches/_treenode_transformers.py index 00b4f7a7..59117f2f 100644 --- a/optimum/torch_export_patches/_treenode_transformers.py +++ b/optimum/torch_export_patches/_treenode_transformers.py @@ -17,7 +17,7 @@ from transformers.cache_utils import MambaCache from transformers.modeling_outputs import BaseModelOutput -from optimum.torch_export_patches import _cache_creater +from optimum.torch_export_patches import _cache_creator SUPPORTED_DATACLASSES: set[type] = set() @@ -92,7 +92,7 @@ def flatten_dynamic_cache( dynamic_cache: DynamicCache, ) -> tuple[list[Any], torch.utils._pytree.Context]: """Serializes a :class:`transformers.cache_utils.DynamicCache` with python objects.""" - ca = _cache_creater.CacheKeyValue(dynamic_cache) + ca = _cache_creator.CacheKeyValue(dynamic_cache) flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] return [f[1] for f in flat], [f[0] for f in flat] @@ -107,7 +107,7 @@ def flatten_with_keys_dynamic_cache( def unflatten_dynamic_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> DynamicCache: """Restores a :class:`transformers.cache_utils.DynamicCache` from python objects.""" - return _cache_creater.make_dynamic_cache(list(zip(values[0], values[1]))) + return _cache_creator.make_dynamic_cache(list(zip(values[0], values[1]))) ############# @@ -119,7 +119,7 @@ def flatten_hybrid_cache( cache: HybridCache, ) -> tuple[list[Any], torch.utils._pytree.Context]: """Serializes a :class:`transformers.cache_utils.HybridCache` with python objects.""" - ca = _cache_creater.CacheKeyValue(cache) + ca = _cache_creator.CacheKeyValue(cache) flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] return [f[1] for f in flat], [f[0] for f in flat] @@ -134,7 +134,7 @@ def flatten_with_keys_hybrid_cache( def unflatten_hybrid_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> HybridCache: """Restores a :class:`transformers.cache_utils.HybridCache` from python objects.""" - return _cache_creater.make_hybrid_cache(list(zip(values[0], values[1]))) + return _cache_creator.make_hybrid_cache(list(zip(values[0], values[1]))) ############# @@ -146,7 +146,7 @@ def flatten_static_cache( cache: StaticCache, ) -> tuple[list[Any], torch.utils._pytree.Context]: """Serializes a :class:`transformers.cache_utils.StaticCache` with python objects.""" - ca = _cache_creater.CacheKeyValue(cache) + ca = _cache_creator.CacheKeyValue(cache) assert not ca.key_cache or cache.max_cache_len == ca.key_cache[0].shape[2], ( f"Serialization doet not work when " f"cache.max_cache_len={cache.max_cache_len} != " @@ -166,7 +166,7 @@ def flatten_with_keys_static_cache( def unflatten_static_cache(values: list[Any], context: torch.utils._pytree.Context, output_type=None) -> StaticCache: """Restores a :class:`transformers.cache_utils.StaticCache` from python objects.""" - return _cache_creater.make_static_cache(list(zip(values[0], values[1])), max_cache_len=values[0][0].shape[2]) + return _cache_creator.make_static_cache(list(zip(values[0], values[1])), max_cache_len=values[0][0].shape[2]) #################### @@ -180,7 +180,7 @@ def flatten_sliding_window_cache( """Serializes a :class:`transformers.cache_utils.SlidingWindowCache` with python objects. """ - ca = _cache_creater.CacheKeyValue(cache) + ca = _cache_creator.CacheKeyValue(cache) flat = [("key_cache", ca.key_cache), ("value_cache", ca.value_cache)] return [f[1] for f in flat], [f[0] for f in flat] @@ -200,7 +200,7 @@ def unflatten_sliding_window_cache( ) -> SlidingWindowCache: """Restores a :class:`transformers.cache_utils.SlidingWindowCache` from python objects.""" key_cache, value_cache = values - return _cache_creater.make_sliding_window_cache(list(zip(values[0], values[1]))) + return _cache_creator.make_sliding_window_cache(list(zip(values[0], values[1]))) ##################### diff --git a/optimum/torch_export_patches/_verbose.py b/optimum/torch_export_patches/_verbose.py index a21f5490..5d36ff69 100644 --- a/optimum/torch_export_patches/_verbose.py +++ b/optimum/torch_export_patches/_verbose.py @@ -6,7 +6,7 @@ import onnx from transformers import cache_utils, configuration_utils -from optimum.torch_export_patches import _cache_creater, _type_utils +from optimum.torch_export_patches import _cache_creator, _type_utils @functools.cache @@ -66,7 +66,7 @@ def string_type( The element types for a tensor are displayed as integer to shorten the message. The semantic is defined by :class:`onnx.TensorProto` and can be obtained - by :func:`onnx_diagnostic.helpers.onnx_helper.onnx_dtype_name`. + by :func:`optimum.torch_export_patches._verbose.onnx_dtype_name`. Examples: Basic usage: @@ -492,7 +492,7 @@ def string_type( "StaticCache", "HybridCache", }: - ca = _cache_creater.CacheKeyValue(obj) + ca = _cache_creator.CacheKeyValue(obj) kc = string_type( ca.key_cache, with_shape=with_shape, @@ -556,7 +556,7 @@ def string_type( return f"{obj.__class__.__name__}(self_attention_cache={att}, cross_attention_cache={cross})" if obj.__class__ in torch.utils._pytree.SUPPORTED_NODES: - args = _cache_creater.flatten_unflatten_for_dynamic_shapes(obj) + args = _cache_creator.flatten_unflatten_for_dynamic_shapes(obj) att = string_type( args, with_shape=with_shape, diff --git a/tests/patch/test_torch_export_patches.py b/tests/patch/test_torch_export_patches.py index e610fa03..06bac574 100644 --- a/tests/patch/test_torch_export_patches.py +++ b/tests/patch/test_torch_export_patches.py @@ -1,12 +1,16 @@ import unittest -from typing import Any +from typing import Any, ClassVar import numpy as np +import torch +import torch.utils._pytree as py_pytree from optimum.torch_export_patches._core import torch_export_patches -class TestOnnxExportErrors(unittest.TestCase): +class TestTorchExportPatchesComprehensive(unittest.TestCase): + """Comprehensive tests for torch_export_patches functionality.""" + def assertEqualArrayAny(self, expected: Any, value: Any, atol: float = 0, rtol: float = 0, msg: str = ""): # noqa: N802 if isinstance(expected, (tuple, list, dict)): self.assertIsInstance(value, type(expected), msg=msg) @@ -26,7 +30,7 @@ def assertEqualArrayAny(self, expected: Any, value: Any, atol: float = 0, rtol: msg_ = "\n".join(excs) msg = f"{msg}\n{msg_}" if msg else msg_ raise AssertionError(f"Found {len(excs)} discrepancies\n{msg}") - elif expected.__class__.__name__ in ("DynamicCache", "StaticCache"): + elif expected.__class__.__name__ in ("DynamicCache", "StaticCache", "HybridCache", "SlidingWindowCache"): atts = {"key_cache", "value_cache"} self.assertEqualArrayAny( {k: expected.__dict__.get(k, None) for k in atts}, @@ -44,14 +48,7 @@ def assertEqualArrayAny(self, expected: Any, value: Any, atol: float = 0, rtol: else: raise AssertionError(f"Comparison not implemented for types {type(expected)} and {type(value)}") - def assertEqualArray( # noqa: N802 - self, - expected: Any, - value: Any, - atol: float = 0, - rtol: float = 0, - msg: str | None = None, - ): + def assertEqualArray(self, expected: Any, value: Any, atol: float = 0, rtol: float = 0, msg: str | None = None): # noqa: N802 if hasattr(expected, "detach") and hasattr(value, "detach"): if msg: try: @@ -66,8 +63,6 @@ def assertEqualArray( # noqa: N802 self.assertEqual(expected.dtype, value.dtype) self.assertEqual(expected.shape, value.shape) - import torch - try: torch.testing.assert_close(value, expected, atol=atol, rtol=rtol) except AssertionError as e: @@ -113,134 +108,291 @@ def assertEqualArray( # noqa: N802 ] raise AssertionError("\n".join(rows)) - # @skipif_ci_windows("not working on Windows") - def test_pytree_flatten_mamba_cache(self): - import torch - import torch.utils._pytree as py_pytree - + def test_context_manager_basic_functionality(self): + """Test that torch_export_patches context manager works correctly.""" + with torch_export_patches(verbose=0) as modificator: + self.assertIsNotNone(modificator) + self.assertTrue(callable(modificator)) + + # Test without patching + with torch_export_patches(patch=False, verbose=0) as modificator: + self.assertIsNotNone(modificator) + self.assertTrue(callable(modificator)) + + def test_patch_flags_combinations(self): + """Test different combinations of patch flags.""" + test_cases = [ + {"patch_sympy": True, "patch_torch": False, "patch_transformers": False, "patch_diffusers": False}, + {"patch_sympy": False, "patch_torch": True, "patch_transformers": False, "patch_diffusers": False}, + {"patch_sympy": False, "patch_torch": False, "patch_transformers": True, "patch_diffusers": False}, + {"patch_sympy": False, "patch_torch": False, "patch_transformers": False, "patch_diffusers": True}, + {"patch_sympy": True, "patch_torch": True, "patch_transformers": True, "patch_diffusers": True}, + ] + + for config in test_cases: + with self.subTest(config=config), torch_export_patches(**config, verbose=0) as modificator: + self.assertIsNotNone(modificator) + + def test_static_cache_serialization(self): + """Test StaticCache serialization and deserialization.""" try: - from transformers.models.mamba.modeling_mamba import MambaCache + from transformers.cache_utils import StaticCache except ImportError: - from transformers.cache_utils import MambaCache + self.skipTest("StaticCache not available") class Config: def __init__(self): - self.intermediate_size = 8 - self.state_size = 16 - self.conv_kernel = 32 - self.num_hidden_layers = 64 - self.dtype = torch.float16 + self.num_attention_heads = 4 + self.num_key_value_heads = 4 + self.hidden_size = 64 + self.head_dim = 16 + self.num_hidden_layers = 3 - cache = MambaCache(Config(), max_batch_size=1, device="cpu") + config = Config() + cache = StaticCache(config, max_batch_size=2, max_cache_len=5, device="cpu", dtype=torch.float32) - with torch_export_patches(verbose=1): + # Add some data + key_states = torch.randn(2, 4, 5, 16) + value_states = torch.randn(2, 4, 5, 16) + cache.update(key_states, value_states, layer_idx=0) + + with torch_export_patches(patch_transformers=True, verbose=0): values, spec = py_pytree.tree_flatten(cache) - cache2 = py_pytree.tree_unflatten(values, spec) - self.assertEqual(cache.max_batch_size, cache2.max_batch_size) - self.assertEqual(cache.intermediate_size, cache2.intermediate_size) - self.assertEqual(cache.ssm_state_size, cache2.ssm_state_size) - self.assertEqual(cache.conv_kernel_size, cache2.conv_kernel_size) - self.assertEqualArrayAny(cache.conv_states, cache2.conv_states) - self.assertEqualArrayAny(cache.ssm_states, cache2.ssm_states) - - # @skipif_ci_windows("not working on Windows") - def test_exportable_mamba_cache(self): - import torch - from transformers.models.mamba.modeling_mamba import MambaCache + restored_cache = py_pytree.tree_unflatten(values, spec) + + self.assertIsInstance(restored_cache, StaticCache) + self.assertEqual(cache.max_cache_len, restored_cache.max_cache_len) + self.assertEqualArrayAny(cache.key_cache, restored_cache.key_cache) + self.assertEqualArrayAny(cache.value_cache, restored_cache.value_cache) + + def test_hybrid_cache_serialization(self): + """Test HybridCache serialization and deserialization.""" + try: + from transformers.cache_utils import HybridCache + except ImportError: + self.skipTest("HybridCache not available") class Config: def __init__(self): - self.intermediate_size = 8 - self.state_size = 16 - self.conv_kernel = 32 - self.num_hidden_layers = 64 - self.dtype = torch.float16 + self.num_attention_heads = 4 + self.num_key_value_heads = 4 + self.hidden_size = 64 + self.head_dim = 16 + self.sliding_window = 4 + self.num_hidden_layers = 3 - class Model(torch.nn.Module): - def forward(self, x: torch.Tensor, cache: MambaCache): - x1 = cache.ssm_states[0] + x - x2 = cache.conv_states[0][:, :, ::2] + x1 - return x2 + config = Config() + cache = HybridCache(config, max_batch_size=2, max_cache_len=10, device="cpu", dtype=torch.float32) - cache = MambaCache(Config(), max_batch_size=1, device="cpu") - x = torch.ones(2, 8, 16).to(torch.float16) - model = Model() - model(x, cache) - - with torch_export_patches(verbose=1, patch_transformers=True): - cache = MambaCache(Config(), max_batch_size=1, device="cpu") - torch.export.export(Model(), (x, cache)) + with torch_export_patches(patch_transformers=True, verbose=0): + values, spec = py_pytree.tree_flatten(cache) + restored_cache = py_pytree.tree_unflatten(values, spec) + self.assertIsInstance(restored_cache, HybridCache) - # @skipif_ci_windows("not working on Windows") - def test_exportable_mamba_cache_dynamic(self): - import torch - from transformers.models.mamba.modeling_mamba import MambaCache + def test_sliding_window_cache_serialization(self): + """Test SlidingWindowCache serialization and deserialization.""" + try: + from transformers.cache_utils import SlidingWindowCache + except ImportError: + self.skipTest("SlidingWindowCache not available") class Config: def __init__(self): - self.intermediate_size = 8 - self.state_size = 16 - self.conv_kernel = 32 - self.num_hidden_layers = 2 - self.dtype = torch.float16 + self.num_attention_heads = 4 + self.num_key_value_heads = 4 + self.hidden_size = 64 + self.head_dim = 16 + self.sliding_window = 5 + self.num_hidden_layers = 3 - class Model(torch.nn.Module): - def forward(self, x: torch.Tensor, cache: MambaCache): - x1 = cache.ssm_states[0] + x - x2 = cache.conv_states[0][:, :, ::2] + x1 - return x2 - - cache = MambaCache(Config(), max_batch_size=1, device="cpu") - x = torch.ones(2, 8, 16).to(torch.float16) - model = Model() - model(x, cache) - dynamic = torch.export.Dim.DYNAMIC - - with torch_export_patches(): - cache = MambaCache(Config(), max_batch_size=2, device="cpu") - torch.export.export( - Model(), - (x, cache), - dynamic_shapes=({0: dynamic}, [[{0: dynamic}, {0: dynamic}], [{0: dynamic}, {0: dynamic}]]), - ) + config = Config() + cache = SlidingWindowCache(config, max_batch_size=2, max_cache_len=10, device="cpu", dtype=torch.float32) - def test_exportable_dynamic_shapes_constraints(self): - import torch + with torch_export_patches(patch_transformers=True, verbose=0): + values, spec = py_pytree.tree_flatten(cache) + restored_cache = py_pytree.tree_unflatten(values, spec) + self.assertIsInstance(restored_cache, SlidingWindowCache) - class CustomCache: - def __init__(self, shape=None): - self.cache = [torch.zeros(shape), torch.zeros(shape)] if shape else [] + def test_encoder_decoder_cache_serialization(self): + """Test EncoderDecoderCache serialization and deserialization.""" + try: + from transformers.cache_utils import DynamicCache, EncoderDecoderCache + except ImportError: + self.skipTest("EncoderDecoderCache not available") + + self_attention_cache = DynamicCache() + cross_attention_cache = DynamicCache() - def flatten_cache(cache): - return [cache.cache], ["cache"] + # Add some data + key_states = torch.randn(2, 4, 5, 16) + value_states = torch.randn(2, 4, 5, 16) + self_attention_cache.update(key_states, value_states, layer_idx=0) + cross_attention_cache.update(key_states, value_states, layer_idx=0) - def unflatten_cache(values, context, output_type=None): - cache = CustomCache() - cache.cache = values[0] - return cache + cache = EncoderDecoderCache(self_attention_cache, cross_attention_cache) + + with torch_export_patches(patch_transformers=True, verbose=0): + values, spec = py_pytree.tree_flatten(cache) + restored_cache = py_pytree.tree_unflatten(values, spec) - def flatten_with_keys_cache(d): - values, context = flatten_cache(d) - return [(torch.utils._pytree.MappingKey(k), v) for k, v in zip(context, values)], context + self.assertIsInstance(restored_cache, EncoderDecoderCache) + self.assertIsInstance(restored_cache.self_attention_cache, DynamicCache) + self.assertIsInstance(restored_cache.cross_attention_cache, DynamicCache) - torch.utils._pytree.register_pytree_node( - CustomCache, - flatten_cache, - unflatten_cache, - serialized_type_name=f"{CustomCache.__module__}.{CustomCache.__name__}", - flatten_with_keys_fn=flatten_with_keys_cache, - ) + def test_torch_vmap_patching(self): + """Test torch.vmap patching functionality.""" class Model(torch.nn.Module): - def forward(self, x, cache): - return cache.cache[0][0, :] + x - - model = Model() - model.eval() - x, cache = torch.rand((2, 4)), CustomCache((2, 4)) - model(x, cache) - dynamic = torch.export.Dim.DYNAMIC - torch.export.export(model, (x, cache), dynamic_shapes=({0: dynamic}, [[{0: dynamic}, {0: dynamic}]])) + def forward(self, x): + return torch.vmap(lambda t: t * 2)(x) + + x = torch.randn(3, 4) + + with torch_export_patches(patch_torch=True, verbose=0): + torch.export.export(Model(), (x,)) + + def test_model_with_cache_export(self): + """Test exporting a model that uses cache objects.""" + try: + from transformers.cache_utils import DynamicCache + except ImportError: + self.skipTest("DynamicCache not available") + + class ModelWithCache(torch.nn.Module): + def forward(self, x: torch.Tensor, cache: DynamicCache): + if cache.key_cache and len(cache.key_cache) > 0: + cached_key = cache.key_cache[0] + return x + cached_key.sum() + return x + + model = ModelWithCache() + x = torch.randn(2, 4, 8) + cache = DynamicCache() + key_states = torch.randn(2, 4, 8, 16) + value_states = torch.randn(2, 4, 8, 16) + cache.update(key_states, value_states, layer_idx=0) + + with torch_export_patches(patch_transformers=True, verbose=0): + # This should not raise an error + exported = torch.export.export(model, (x, cache)) + self.assertIsNotNone(exported) + + def test_custom_patches(self): + """Test applying custom patches.""" + + class CustomPatchedClass: + _PATCHES_: ClassVar[list[str]] = ["test_method"] + _PATCHED_CLASS_: ClassVar[type] = torch.nn.Linear + + @staticmethod + def test_method(self): + return "patched" + + original_method = getattr(torch.nn.Linear, "test_method", None) + + try: + with torch_export_patches(custom_patches=[CustomPatchedClass], verbose=0): + linear = torch.nn.Linear(2, 2) + self.assertTrue(hasattr(linear, "test_method")) + self.assertEqual(linear.test_method(), "patched") + + finally: + # Clean up + if original_method is None: + if hasattr(torch.nn.Linear, "test_method"): + delattr(torch.nn.Linear, "test_method") + else: + torch.nn.Linear.test_method = original_method + + def test_verbose_output(self): + """Test that verbose output is generated correctly.""" + import io + from contextlib import redirect_stdout + + captured_output = io.StringIO() + + with redirect_stdout(captured_output), torch_export_patches(verbose=2): + pass + + output = captured_output.getvalue() + # Should contain some verbose information + self.assertTrue(len(output) > 0) + + def test_patch_isolation(self): + """Test that patches are properly isolated and don't leak.""" + original_vmap = torch.vmap + + with torch_export_patches(patch_torch=True, verbose=0): + # Inside context, vmap should be patched + self.assertNotEqual(torch.vmap, original_vmap) + + # Outside context, vmap should be restored + self.assertEqual(torch.vmap, original_vmap) + + def test_error_handling_invalid_config(self): + """Test error handling for invalid configurations.""" + # Test that rewrite parameter raises an error + with self.assertRaises(ValueError), torch_export_patches(rewrite=["some_method"]): + pass + + with self.assertRaises(ValueError), torch_export_patches(dump_rewriting="some_file"): + pass + + def test_dataclass_serialization(self): + """Test dataclass serialization functionality.""" + try: + from transformers.modeling_outputs import BaseModelOutput + except ImportError: + self.skipTest("BaseModelOutput not available") + + output = BaseModelOutput(last_hidden_state=torch.randn(2, 8, 64), hidden_states=None, attentions=None) + + with torch_export_patches(patch_transformers=True, verbose=0): + values, spec = py_pytree.tree_flatten(output) + restored_output = py_pytree.tree_unflatten(values, spec) + + self.assertIsInstance(restored_output, BaseModelOutput) + self.assertEqualArray(output.last_hidden_state, restored_output.last_hidden_state) + + def test_diffusers_patches(self): + """Test diffusers-specific patches.""" + try: + from diffusers.models.unets.unet_2d_condition import UNet2DConditionOutput + except ImportError: + self.skipTest("UNet2DConditionOutput not available") + + output = UNet2DConditionOutput(sample=torch.randn(2, 4, 8, 8)) + + with torch_export_patches(patch_diffusers=True, verbose=0): + values, spec = py_pytree.tree_flatten(output) + restored_output = py_pytree.tree_unflatten(values, spec) + + self.assertIsInstance(restored_output, UNet2DConditionOutput) + self.assertEqualArray(output.sample, restored_output.sample) + + def test_multiple_cache_operations(self): + """Test multiple cache operations in sequence.""" + try: + from transformers.cache_utils import DynamicCache + except ImportError: + self.skipTest("DynamicCache not available") + + with torch_export_patches(patch_transformers=True, verbose=0): + cache = DynamicCache() + + # Test multiple updates + for layer_idx in range(3): + key_states = torch.randn(2, 4, 5, 16) + value_states = torch.randn(2, 4, 5, 16) + cache.update(key_states, value_states, layer_idx=layer_idx) + + # Test serialization after multiple operations + values, spec = py_pytree.tree_flatten(cache) + restored_cache = py_pytree.tree_unflatten(values, spec) + + self.assertEqual(len(cache.key_cache), len(restored_cache.key_cache)) + for i in range(len(cache.key_cache)): + self.assertEqualArray(cache.key_cache[i], restored_cache.key_cache[i]) if __name__ == "__main__": From d978c730d0bc9e57388296eadcc20ac629dfa06c Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Wed, 30 Jul 2025 21:13:52 +0000 Subject: [PATCH 09/10] remove exc --- optimum/torch_export_patches/_verbose.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/optimum/torch_export_patches/_verbose.py b/optimum/torch_export_patches/_verbose.py index 5d36ff69..8b667161 100644 --- a/optimum/torch_export_patches/_verbose.py +++ b/optimum/torch_export_patches/_verbose.py @@ -10,7 +10,7 @@ @functools.cache -def onnx_dtype_name(itype: int, exc: bool = True) -> str: +def onnx_dtype_name(itype: int) -> str: """Returns the ONNX name for a specific element type. .. runpython:: @@ -28,8 +28,7 @@ def onnx_dtype_name(itype: int, exc: bool = True) -> str: v = getattr(onnx.TensorProto, k) if v == itype: return k - if exc: - raise ValueError(f"Unexpected value itype: {itype}") + if itype == 0: return "UNDEFINED" return "UNEXPECTED" From 2b95667b9729b8f74fb02845166e286a7f755b7b Mon Sep 17 00:00:00 2001 From: Ti-Tai Wang Date: Fri, 8 Aug 2025 00:01:26 +0000 Subject: [PATCH 10/10] fix tests --- tests/patch/test_torch_export_patches.py | 38 ++++-------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/tests/patch/test_torch_export_patches.py b/tests/patch/test_torch_export_patches.py index 06bac574..085b3820 100644 --- a/tests/patch/test_torch_export_patches.py +++ b/tests/patch/test_torch_export_patches.py @@ -159,7 +159,8 @@ def __init__(self): with torch_export_patches(patch_transformers=True, verbose=0): values, spec = py_pytree.tree_flatten(cache) restored_cache = py_pytree.tree_unflatten(values, spec) - + # If the registration stops working, this will be hit. + self.assertNotIsInstance(values[0], StaticCache) self.assertIsInstance(restored_cache, StaticCache) self.assertEqual(cache.max_cache_len, restored_cache.max_cache_len) self.assertEqualArrayAny(cache.key_cache, restored_cache.key_cache) @@ -187,6 +188,7 @@ def __init__(self): with torch_export_patches(patch_transformers=True, verbose=0): values, spec = py_pytree.tree_flatten(cache) restored_cache = py_pytree.tree_unflatten(values, spec) + self.assertNotIsInstance(values[0], HybridCache) self.assertIsInstance(restored_cache, HybridCache) def test_sliding_window_cache_serialization(self): @@ -211,6 +213,7 @@ def __init__(self): with torch_export_patches(patch_transformers=True, verbose=0): values, spec = py_pytree.tree_flatten(cache) restored_cache = py_pytree.tree_unflatten(values, spec) + self.assertNotIsInstance(values[0], SlidingWindowCache) self.assertIsInstance(restored_cache, SlidingWindowCache) def test_encoder_decoder_cache_serialization(self): @@ -235,6 +238,7 @@ def test_encoder_decoder_cache_serialization(self): values, spec = py_pytree.tree_flatten(cache) restored_cache = py_pytree.tree_unflatten(values, spec) + self.assertNotIsInstance(values[0], EncoderDecoderCache) self.assertIsInstance(restored_cache, EncoderDecoderCache) self.assertIsInstance(restored_cache.self_attention_cache, DynamicCache) self.assertIsInstance(restored_cache.cross_attention_cache, DynamicCache) @@ -338,38 +342,6 @@ def test_error_handling_invalid_config(self): with self.assertRaises(ValueError), torch_export_patches(dump_rewriting="some_file"): pass - def test_dataclass_serialization(self): - """Test dataclass serialization functionality.""" - try: - from transformers.modeling_outputs import BaseModelOutput - except ImportError: - self.skipTest("BaseModelOutput not available") - - output = BaseModelOutput(last_hidden_state=torch.randn(2, 8, 64), hidden_states=None, attentions=None) - - with torch_export_patches(patch_transformers=True, verbose=0): - values, spec = py_pytree.tree_flatten(output) - restored_output = py_pytree.tree_unflatten(values, spec) - - self.assertIsInstance(restored_output, BaseModelOutput) - self.assertEqualArray(output.last_hidden_state, restored_output.last_hidden_state) - - def test_diffusers_patches(self): - """Test diffusers-specific patches.""" - try: - from diffusers.models.unets.unet_2d_condition import UNet2DConditionOutput - except ImportError: - self.skipTest("UNet2DConditionOutput not available") - - output = UNet2DConditionOutput(sample=torch.randn(2, 4, 8, 8)) - - with torch_export_patches(patch_diffusers=True, verbose=0): - values, spec = py_pytree.tree_flatten(output) - restored_output = py_pytree.tree_unflatten(values, spec) - - self.assertIsInstance(restored_output, UNet2DConditionOutput) - self.assertEqualArray(output.sample, restored_output.sample) - def test_multiple_cache_operations(self): """Test multiple cache operations in sequence.""" try: