add only_llm_forward

Author: hhaAndroid

xtuner/v1/model/compose/qwen3_vl/modeling_qwen3_vl.py

@@ -28,6 +28,7 @@ class Qwen3VLForConditionalGeneration(BaseComposeModel):
 
     def __init__(self, config: Qwen3VLBaseConfig):
         super().__init__(config)  # type: ignore[arg-type]
+        self.only_llm_forward = config.only_llm_forward
 
         # if type(self.language_model) is Qwen3MoE:
         #     # TODO(YHC): This is a hack to make the language model compatible with HF
@@ -143,8 +144,9 @@ def forward(
         sequence_parallel_mesh = seq_ctx.sequence_parallel_mesh
 
         inputs_embeds = self.language_model.embed_tokens(input_ids)  # type: ignore
-
+        
         if pixel_values is not None:
+            assert self.only_llm_forward is False, "only_llm_forward is True, but pixel_values is not None. Please check your config setting."
             assert image_grid_thw is not None
             assert input_ids is not None
             visual_embeds, deepstack_visual_embeds = self.get_visual_features(pixel_values,
@@ -170,12 +172,13 @@ def forward(
                 deepstack_visual_embeds = None
                 visual_pos_masks = None
         else:
-            pixel_values_dump = torch.randn(4, 1536, device=inputs_embeds.device, dtype=inputs_embeds.dtype)
-            image_grid_thw = torch.tensor([[1, 2, 2]], device=inputs_embeds.device)
-            viusal_embeds, deepstack_visual_embeds = self.get_visual_features(pixel_values_dump, image_grid_thw)
-            inputs_embeds = inputs_embeds + viusal_embeds.sum() * 0.0
-            for deepstack_visual_embed in deepstack_visual_embeds:
-                inputs_embeds = inputs_embeds + deepstack_visual_embed.sum() * 0.0
+            if not self.only_llm_forward:
+                pixel_values_dump = torch.randn(4, 1536, device=inputs_embeds.device, dtype=inputs_embeds.dtype)
+                image_grid_thw = torch.tensor([[1, 2, 2]], device=inputs_embeds.device)
+                viusal_embeds, deepstack_visual_embeds = self.get_visual_features(pixel_values_dump, image_grid_thw)
+                inputs_embeds = inputs_embeds + viusal_embeds.sum() * 0.0
+                for deepstack_visual_embed in deepstack_visual_embeds:
+                    inputs_embeds = inputs_embeds + deepstack_visual_embed.sum() * 0.0
 
             deepstack_visual_embeds = None
             visual_pos_masks = None

xtuner/v1/model/compose/qwen3_vl/qwen3_vl_config.py

@@ -87,6 +87,8 @@ class Qwen3VLBaseConfig(BaseComposeConfig):
     freeze_vision: bool = False
     freeze_projector: bool = False
     freeze_language: bool = False
+    # If true, skip the forward of vit+projector. Only enable when the whole training process is pure text task.
+    only_llm_forward: bool = False
 
     def build(self):
         from .modeling_qwen3_vl import Qwen3VLForConditionalGeneration