Repository navigation
Exportable kimi #47096
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Exportable kimi #47096
Changes from 5 commits
7f0d32e
bc3524f
cf1c752
a86222c
6a675a5
bf9fd99
7714e02
db38288
f8b6a16
5dd283d
966434a
f31932c
30b0f92
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -427,6 +427,25 @@ def patch(self, *sizes): | |
| return patch | ||
|
|
||
|
|
||
| @register_patch("executorch", "torch.reshape", "torch.Tensor.reshape") | ||
| def _patch_reshape(original): | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. hmm ok so here we have the issue indirectly re contiguous?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the issue was always in executorch, imo this is more direct 😆 |
||
| """Materialise a non-contiguous input before ``reshape``. | ||
|
|
||
| ExecuTorch's edge-lowering reshape reference refuses a non-contiguous input (e.g. the | ||
| ``transpose(1, 2).reshape(...)`` in the packed vision-attention forward). A plain | ||
| ``.contiguous()`` gets folded away by functionalization, but a ``.clone()`` survives. Eager | ||
| ``reshape`` already copies a non-contiguous tensor, so this adds no extra work — it just moves | ||
| the copy where ExecuTorch's lowering needs it. | ||
| """ | ||
|
|
||
| def patch(input, *shape): | ||
| if not input.is_contiguous(): | ||
| input = input.clone() | ||
| return original(input, *shape) | ||
|
|
||
| return patch | ||
|
|
||
|
|
||
| # ── Stage 3: ExecuTorch patches ─────────────────────────────────────────────── | ||
| # Reversible swaps of ExecuTorch internals (passes, verifiers, op dicts) that crash | ||
| # on legitimate dynamic-shape patterns: `SpecPropPass.update_placeholder_tensor_specs`, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -58,8 +58,8 @@ | |
|
|
||
| from ..modeling_utils import PreTrainedModel | ||
| from ..vision_utils import ( | ||
| get_vision_attention_seqlens, | ||
| get_vision_bilinear_indices_and_weights, | ||
| get_vision_cu_seqlens, | ||
| get_vision_merged_shape, | ||
| get_vision_nearest_position_ids, | ||
| get_vision_position_ids, | ||
|
|
@@ -433,9 +433,10 @@ def _prepare_grid_thw_vision_inputs(model: torch.nn.Module, inputs: dict[str, An | |
| `window_index`/`cu_window_seqlens`/`max_window_seqlen` (XNet-style window attn) and | ||
| `bilinear_indices`/`bilinear_weights` (interpolation-based merging). | ||
|
|
||
| Optional helpers are gated by the presence of their config attribute on the encoder | ||
| (`window_size`+`patch_size` for window attention, `num_grid_per_side` for bilinear), | ||
| so a model that doesn't use that feature won't get its kwarg injected. | ||
| Optional helpers are gated by a submodule attribute (`window_size`+`patch_size` for window | ||
| attention, `num_grid_per_side` for bilinear) or, for model-specific ones, by the encoder's | ||
| modeling module defining the helper (`get_vision_frame_index` / `get_vision_temporal_merge_index` | ||
| for kimi_k25) — so a model that doesn't use a feature won't get its kwarg injected. | ||
| """ | ||
| grid_thw = inputs["grid_thw"] | ||
| spatial_merge_size = _find_submodule_attr(model, "spatial_merge_size") | ||
|
|
@@ -444,9 +445,13 @@ def _prepare_grid_thw_vision_inputs(model: torch.nn.Module, inputs: dict[str, An | |
| # none (its encoder hard-codes `1` because spatial merging happens in the projector). | ||
| spatial_merge_size = inputs.get("merge_sizes", 1) | ||
|
|
||
| cu_seqlens, max_seqlen = get_vision_attention_seqlens(grid_thw, model.config, kwargs=inputs) | ||
| inputs["cu_seqlens"] = cu_seqlens | ||
| inputs["max_seqlen"] = max_seqlen | ||
| # kimi_k25-style encoders define their own per-frame / temporal-merge precompute helpers in their | ||
| # modeling module (resolved below) and attend over the whole clip, so `cu_seqlens` is per-clip | ||
| # (matching the encoder's util call). Other grid_thw encoders lack these and stay per-frame. | ||
| module = sys.modules[type(model).__module__] | ||
| temporal_encoder = hasattr(module, "get_vision_frame_index") | ||
| inputs["cu_seqlens"] = get_vision_cu_seqlens(grid_thw, merge_temporal=temporal_encoder) | ||
| inputs["max_seqlen"] = get_max_seqlen(inputs["cu_seqlens"], model.config, kwargs=inputs) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why not use get_vision_attention_seqlens as before?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes sure |
||
| # 3-axis (t, h, w) rotary encoders expose an ``axis_dim`` attr on their rotary_emb | ||
| # (minimax_m3_vl); default 2-axis (h, w) covers qwen2_5_vl / qwen3_vl / glm4v / paddleocr_vl. | ||
| include_temporal = _find_submodule_attr(model, "axis_dim") is not None | ||
|
|
@@ -464,9 +469,27 @@ def _prepare_grid_thw_vision_inputs(model: torch.nn.Module, inputs: dict[str, An | |
|
|
||
| num_grid_per_side = _find_submodule_attr(model, "num_grid_per_side") | ||
| if num_grid_per_side is not None: | ||
| inputs["bilinear_indices"], inputs["bilinear_weights"] = get_vision_bilinear_indices_and_weights( | ||
| grid_thw, num_grid_per_side, spatial_merge_size | ||
| if hasattr(module, "get_vision_bicubic_indices_and_weights"): | ||
| # kimi_k25 resamples its learned grid bicubically (helper defined in its own module). | ||
| inputs["bicubic_indices"], inputs["bicubic_weights"] = module.get_vision_bicubic_indices_and_weights( | ||
| grid_thw, num_grid_per_side | ||
| ) | ||
| else: | ||
| inputs["bilinear_indices"], inputs["bilinear_weights"] = get_vision_bilinear_indices_and_weights( | ||
| grid_thw, num_grid_per_side, spatial_merge_size | ||
| ) | ||
|
|
||
| # Per-frame additive position table (kimi_k25): gathered by frame index instead of a per-clip loop. | ||
| if temporal_encoder: | ||
| inputs["frame_index"] = module.get_vision_frame_index(grid_thw) | ||
|
|
||
| # Temporal-pooling spatial merger (kimi_k25): one gather index replaces its per-clip merge loop. | ||
| if hasattr(module, "get_vision_temporal_merge_index"): | ||
| merge_kernel_size = _find_submodule_attr(model, "merge_kernel_size") | ||
| kernel_height, kernel_width = ( | ||
| merge_kernel_size if not isinstance(merge_kernel_size, int) else (merge_kernel_size, merge_kernel_size) | ||
| ) | ||
| inputs["temporal_merge_index"] = module.get_vision_temporal_merge_index(grid_thw, kernel_height, kernel_width) | ||
|
|
||
|
|
||
| @register_export_input_preparer("target_sizes") | ||
|
|
@@ -695,9 +718,13 @@ def _find_multimodal_submodules(model: PreTrainedModel) -> dict[str, torch.nn.Mo | |
| return found | ||
|
|
||
|
|
||
| def is_multimodal(model: PreTrainedModel) -> bool: | ||
| """Returns `True` if the model is multi-modal with modal encoders and a language model.""" | ||
| return bool(_find_multimodal_submodules(model)) | ||
| def is_multimodal(model: PreTrainedModel | torch.nn.Module) -> bool: | ||
| """Returns `True` if the model is multi-modal with modal encoders and a language model. | ||
|
|
||
| A non-`PreTrainedModel` (e.g. a bare `nn.Module`) has no canonical `get_encoder`/`get_decoder` | ||
| accessors and is trivially not multi-modal, so it short-circuits to `False`. | ||
| """ | ||
| return isinstance(model, PreTrainedModel) and bool(_find_multimodal_submodules(model)) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. just fmi where do we suddenly call on a normal nn module?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. not suddenly tbh, this happens during testing and i didn't catch it before |
||
|
|
||
|
|
||
| def decompose_multimodal(model: PreTrainedModel, inputs: dict[str, Any]) -> dict[str, tuple[torch.nn.Module, dict]]: | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
hmm why remove?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
not needed in the default (dynamo export) path, if a backend needs it it should handle it on its own (see patch_reshape)