Skip to content
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
f352b0d
upstream
ywang96 Sep 12, 2025
667e973
fix & add co-author
ywang96 Sep 12, 2025
8fe70a7
Update tests/models/registry.py
ywang96 Sep 12, 2025
5f6afa1
add missing processor test
Isotr0py Sep 12, 2025
9c5808f
revert str
ywang96 Sep 12, 2025
c7fe668
fix processor test hashing
Isotr0py Sep 12, 2025
aa2330f
fix frames indices
Isotr0py Sep 12, 2025
e1f8397
fix hit_rate 1.0
Isotr0py Sep 12, 2025
9c7939c
typo
Isotr0py Sep 12, 2025
5d4f6dd
fix placeholder replacement
Isotr0py Sep 12, 2025
0d88363
fix video example
Isotr0py Sep 13, 2025
574884c
Merge branch 'main' into upstream-qwen-3-vl
ywang96 Sep 13, 2025
6ec3968
fix vit backend
ywang96 Sep 13, 2025
0f80a19
fix online serving metadata
Isotr0py Sep 13, 2025
ba54870
avoid hardcode fps=1
Isotr0py Sep 14, 2025
f7c37a9
oops fps=1
Isotr0py Sep 14, 2025
a6c5d7e
Merge branch 'main' into upstream-qwen-3-vl
ywang96 Sep 15, 2025
cbf6dee
Merge branch 'main' into upstream-qwen-3-vl
Isotr0py Sep 15, 2025
bec9e7e
catch up and fix processor test
Isotr0py Sep 15, 2025
5f3cf0e
Merge branch 'main' into upstream-qwen-3-vl
ywang96 Sep 16, 2025
5027f31
fix model path
ywang96 Sep 16, 2025
d0133e2
fix qwen_vl_utils compatibility
Isotr0py Sep 16, 2025
07e4f52
fix fps
ywang96 Sep 16, 2025
78afc5b
Merge branch 'main' into upstream-qwen-3-vl
ywang96 Sep 16, 2025
44f89b2
fix
ywang96 Sep 16, 2025
c7ea6f7
cleanup
ywang96 Sep 16, 2025
10bd983
do not modify metadata
ywang96 Sep 16, 2025
a4c0d34
fix example and online fps
Isotr0py Sep 16, 2025
e11895e
Merge branch 'main' into upstream-qwen-3-vl
ywang96 Sep 16, 2025
438f146
clarify
ywang96 Sep 16, 2025
52e2b2b
Merge branch 'main' into upstream-qwen-3-vl
ywang96 Sep 17, 2025
887a833
Merge branch 'main' into upstream-qwen-3-vl
ywang96 Sep 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/models/supported_models.md
Original file line number Diff line number Diff line change
Expand Up @@ -668,6 +668,8 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
| `Qwen2_5_VLForConditionalGeneration` | Qwen2.5-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen2.5-VL-3B-Instruct`, `Qwen/Qwen2.5-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
| `Qwen2_5OmniThinkerForConditionalGeneration` | Qwen2.5-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `Qwen/Qwen2.5-Omni-3B`, `Qwen/Qwen2.5-Omni-7B` | ✅︎ | ✅︎ | ✅︎ |
| `Qwen3VLForConditionalGeneration` | Qwen3-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3-VL-4B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
| `Qwen3VLMoeForConditionalGeneration` | Qwen3-VL-MOE | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
| `RForConditionalGeneration` | R-VL-4B | T + I<sup>E+</sup> | `YannQi/R-4B` | | ✅︎ | ✅︎ |
| `SkyworkR1VChatModel` | Skywork-R1V-38B | T + I | `Skywork/Skywork-R1V-38B` | | ✅︎ | ✅︎ |
| `SmolVLMForConditionalGeneration` | SmolVLM2 | T + I | `SmolVLM2-2.2B-Instruct` | ✅︎ | | ✅︎ |
Expand Down
88 changes: 87 additions & 1 deletion examples/offline_inference/vision_language.py
Original file line number Diff line number Diff line change
Expand Up @@ -1497,6 +1497,80 @@ def run_qwen2_5_omni(questions: list[str], modality: str):
)


# Qwen3-VL-Dense
def run_qwen3_vl(questions: list[str], modality: str) -> ModelRequestData:
model_name = "Qwen/Qwen3-VL-4B-Instruct"

engine_args = EngineArgs(
model=model_name,
max_model_len=4096,
max_num_seqs=5,
mm_processor_kwargs={
"min_pixels": 28 * 28,
"max_pixels": 1280 * 28 * 28,
"fps": 1,
},
limit_mm_per_prompt={modality: 1},
)
Comment thread
Isotr0py marked this conversation as resolved.
Resolved

if modality == "image":
placeholder = "<|image_pad|>"
elif modality == "video":
placeholder = "<|video_pad|>"

prompts = [
(
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
f"{question}<|im_end|>\n"
"<|im_start|>assistant\n"
)
for question in questions
]

return ModelRequestData(
engine_args=engine_args,
prompts=prompts,
)


# Qwen3-VL-MOE
def run_qwen3_vl_moe(questions: list[str], modality: str) -> ModelRequestData:
model_name = "Qwen/Qwen3-VL-30B-A3B-Instruct"

engine_args = EngineArgs(
model=model_name,
max_model_len=4096,
max_num_seqs=5,
mm_processor_kwargs={
"min_pixels": 28 * 28,
"max_pixels": 1280 * 28 * 28,
"fps": 1,
},
limit_mm_per_prompt={modality: 1},
)

if modality == "image":
placeholder = "<|image_pad|>"
elif modality == "video":
placeholder = "<|video_pad|>"

prompts = [
(
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
f"{question}<|im_end|>\n"
"<|im_start|>assistant\n"
)
for question in questions
]

return ModelRequestData(
engine_args=engine_args,
prompts=prompts,
)
Comment thread
ywang96 marked this conversation as resolved.
Resolved


# R-4B
def run_r_vl(questions: list[str], modality: str) -> ModelRequestData:
assert modality == "image"
Expand Down Expand Up @@ -1707,6 +1781,8 @@ def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
"qwen2_vl": run_qwen2_vl,
"qwen2_5_vl": run_qwen2_5_vl,
"qwen2_5_omni": run_qwen2_5_omni,
"qwen3_vl": run_qwen3_vl,
"qwen3_vl_moe": run_qwen3_vl_moe,
"rvl": run_r_vl,
"skywork_chat": run_skyworkr1v,
"smolvlm": run_smolvlm,
Expand All @@ -1716,6 +1792,15 @@ def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
}


MODELS_NEED_VIDEO_METADATA = [
"glm4_1v",
"glm4_5v",
"glm4_5v_fp8",
"qwen3_vl",
"qwen3_vl_moe",
]


def get_multi_modal_input(args):
"""
return {
Expand All @@ -1740,12 +1825,13 @@ def get_multi_modal_input(args):

if args.modality == "video":
# Input video and question
needs_metadata = args.model_type in MODELS_NEED_VIDEO_METADATA
video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays
metadata = VideoAsset(name="baby_reading", num_frames=args.num_frames).metadata
vid_questions = ["Why is this video funny?"]

return {
"data": [(video, metadata)] if args.model_type == "glm4_1v" else video,
"data": ([(video, metadata)] if needs_metadata else video),
"questions": vid_questions,
}

Expand Down
34 changes: 33 additions & 1 deletion tests/models/multimodal/processing/test_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""
# Ensure video metadata is included
if "video" in mm_data:
# GLM4.1V doesn't support multiple videos
video = mm_data["video"]
mm_data["video"] = (video, {
"total_num_frames": len(video),
Expand All @@ -41,6 +42,33 @@ def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
return mm_data


def qwen3_vl_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""
Patch the multimodal data for Qwen3-VL model.
"""

def create_metadata(frames: np.ndarray):
num_frames = len(frames)
return {
"total_num_frames": num_frames,
"fps": 2.0,
"duration": num_frames / 2.0,
"video_backend": "opencv",
"frames_indices": list(range(num_frames)),
}

# Ensure video metadata is included
if "video" in mm_data:
video = mm_data["video"]
if isinstance(video, list):
# multiple videos
mm_data["video"] = [(vid, create_metadata(vid)) for vid in video]
else:
# single video
mm_data["video"] = (video, create_metadata(video))
return mm_data


def _test_processing_correctness(
model_id_or_arch: str,
hit_rate: float,
Expand Down Expand Up @@ -181,8 +209,10 @@ def _test_processing_correctness(
}

MM_DATA_PATCHES = {
# GLM4.1V requires video metadata to be included in the input
# GLM4.1V and Qwen3-VL requires video metadata to be included in the input
"glm4v": glm4_1v_patch_mm_data,
"qwen3_vl": qwen3_vl_patch_mm_data,
"qwen3_vl_moe": qwen3_vl_patch_mm_data,
}


Expand Down Expand Up @@ -328,6 +358,8 @@ def _test_processing_correctness_one(
"Qwen/Qwen2.5-VL-3B-Instruct",
"Qwen/Qwen2-Audio-7B-Instruct",
"Qwen/Qwen2.5-Omni-3B",
"Qwen/Qwen3-VL-4B-Instruct",
"Qwen/Qwen3-VL-30B-A3B-Instruct",
Comment thread
Isotr0py marked this conversation as resolved.
Resolved
"YannQi/R-4B",
"Skywork/Skywork-R1V-38B",
"HuggingFaceTB/SmolVLM2-2.2B-Instruct",
Expand Down
6 changes: 6 additions & 0 deletions tests/models/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -560,6 +560,12 @@ def check_available_online(
max_model_len=4096),
"Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
"Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"), # noqa: E501
"Qwen3VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen3-VL-4B-Instruct", # noqa: E501
max_model_len=4096,
min_transformers_version="4.57"), # noqa: E501
"Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen3-VL-30B-A3B-Instruct", # noqa: E501
max_model_len=4096,
min_transformers_version="4.57"),
"RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B",
trust_remote_code=True),
"SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B",
Expand Down
14 changes: 9 additions & 5 deletions vllm/assets/video.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def video_to_pil_images_list(path: str,
return [Image.fromarray(frame) for frame in frames]


def video_get_metadata(path: str) -> dict[str, Any]:
def video_get_metadata(path: str, num_frames: int = -1) -> dict[str, Any]:
cap = cv2.VideoCapture(path)
if not cap.isOpened():
raise ValueError(f"Could not open video file {path}")
Expand All @@ -85,11 +85,15 @@ def video_get_metadata(path: str) -> dict[str, Any]:
fps = cap.get(cv2.CAP_PROP_FPS)
duration = total_frames / fps if fps > 0 else 0

if num_frames == -1 or num_frames > total_frames:
num_frames = total_frames

metadata = {
"total_num_frames": total_frames,
"fps": fps,
"total_num_frames": num_frames,
"fps": 1,
"duration": duration,
"video_backend": "opencv"
"video_backend": "opencv",
"frames_indices": list(range(num_frames)),
}
return metadata

Expand Down Expand Up @@ -126,7 +130,7 @@ def np_ndarrays(self) -> npt.NDArray:

@property
def metadata(self) -> dict[str, Any]:
ret = video_get_metadata(self.video_path)
ret = video_get_metadata(self.video_path, self.num_frames)
return ret

def get_audio(self, sampling_rate: Optional[float] = None) -> npt.NDArray:
Expand Down
2 changes: 2 additions & 0 deletions vllm/model_executor/layers/rotary_embedding/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ def get_rope(
is_neox_style,
dtype,
mrope_section=rope_scaling["mrope_section"],
mrope_interleaved=rope_scaling.get("mrope_interleaved",
False),
)
else:
rotary_emb = RotaryEmbedding(
Expand Down
Loading