Skip to content

[serge] Fix 5 integration tests for model qwen3_vl_moe failing with other, raising RuntimeError (other (5)) #50145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sergereview wants to merge 1 commit into
base: main
Choose a base branch
from
+52 −18
Open

[serge] Fix 5 integration tests for model qwen3_vl_moe failing with other, raising RuntimeError (other (5)) #50145

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 52 additions & 18 deletions tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.
"""Testing suite for the PyTorch Qwen3VLMoe model."""

import tempfile
import unittest

import pytest
Expand All @@ -27,6 +28,8 @@
from transformers.models.qwen3_vl_moe.configuration_qwen3_vl_moe import Qwen3VLMoeTextConfig, Qwen3VLMoeVisionConfig
from transformers.testing_utils import (
Expectations,
backend_device_count,
get_cpu_ram_total_gib,
require_flash_attn,
require_torch,
require_torch_accelerator,
Expand Down Expand Up @@ -301,6 +304,49 @@ def test_reverse_loading_mapping(self, check_keys_were_modified=False):
class Qwen3VLMoeIntegrationTest(MemoryCleanupMixin, unittest.TestCase):
maxDiff = None

@classmethod
def setUpClass(cls):
super().setUpClass()
# Assigned here rather than in the class body: `MemoryCleanupMixin` snapshots the class body and
# protects it from teardown, so a checkpoint parked on a class-body `model` is never released.
cls.model = None
cls.offload_dir = None

@classmethod
def get_model(cls):
if cls.model is None:
cls.offload_dir = tempfile.TemporaryDirectory()
# device_map="auto" fills GPUs to ~100%, leaving no room for the transpose temporary
# buffer that fuses per-expert weight shards into a single gate_up_proj tensor during
# from_pretrained — causing CUDA OOM on multi-GPU runners. A 70% per-GPU max_memory cap
# reserves the headroom.
n = backend_device_count(torch_device)
if n > 0 and torch_device != "cpu":
torch_accel = getattr(torch, torch_device)
per_device = int(
min(torch_accel.get_device_properties(i).total_memory for i in range(n)) * 0.70 / 1024**3
)
max_memory = dict.fromkeys(range(n), f"{per_device}GiB")
max_memory["cpu"] = (
f"{int(get_cpu_ram_total_gib() * 0.9)}GiB" # To avoid runner failing with exit code 137.
)
else:
max_memory = None
cls.model = Qwen3VLMoeForConditionalGeneration.from_pretrained(
"Qwen/Qwen3-VL-30B-A3B-Instruct",
dtype="auto",
device_map="auto",
max_memory=max_memory,
offload_folder=cls.offload_dir.name,
)
return cls.model

@classmethod
def tearDownClass(cls):
if cls.offload_dir is not None:
cls.offload_dir.cleanup()
super().tearDownClass()

def setUp(self):
super().setUp()

Expand Down Expand Up @@ -351,9 +397,7 @@ def setUp(self):

@slow
def test_small_model_integration_test(self):
model = Qwen3VLMoeForConditionalGeneration.from_pretrained(
"Qwen/Qwen3-VL-30B-A3B-Instruct", dtype="auto", device_map="auto"
)
model = self.get_model()

inputs = self.processor.apply_chat_template(
self.message, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt"
Expand Down Expand Up @@ -394,9 +438,7 @@ def test_small_model_integration_test(self):

@slow
def test_small_model_integration_test_batch(self):
model = Qwen3VLMoeForConditionalGeneration.from_pretrained(
"Qwen/Qwen3-VL-30B-A3B-Instruct", dtype="auto", device_map="auto"
)
model = self.get_model()
batch_messages = [self.message] * 2
inputs = self.processor.apply_chat_template(
batch_messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt"
Expand Down Expand Up @@ -462,9 +504,7 @@ def test_small_model_integration_test_with_video(self):

@slow
def test_small_model_integration_test_expand(self):
model = Qwen3VLMoeForConditionalGeneration.from_pretrained(
"Qwen/Qwen3-VL-30B-A3B-Instruct", dtype="auto", device_map="auto"
)
model = self.get_model()
inputs = self.processor.apply_chat_template(
self.message, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt"
).to(torch_device)
Expand All @@ -490,9 +530,7 @@ def test_small_model_integration_test_expand(self):

@slow
def test_small_model_integration_test_expand_with_video(self):
model = Qwen3VLMoeForConditionalGeneration.from_pretrained(
"Qwen/Qwen3-VL-30B-A3B-Instruct", dtype="auto", device_map="auto"
)
model = self.get_model()
inputs = self.processor.apply_chat_template(
self.message3, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt"
).to(torch_device)
Expand All @@ -511,9 +549,7 @@ def test_small_model_integration_test_expand_with_video(self):

@slow
def test_small_model_integration_test_batch_wo_image(self):
model = Qwen3VLMoeForConditionalGeneration.from_pretrained(
"Qwen/Qwen3-VL-30B-A3B-Instruct", dtype="auto", device_map="auto"
)
model = self.get_model()
message_wo_image = [
{"role": "user", "content": [{"type": "text", "text": "Who are you?"}]},
]
Expand Down Expand Up @@ -541,9 +577,7 @@ def test_small_model_integration_test_batch_wo_image(self):

@slow
def test_small_model_integration_test_batch_different_resolutions(self):
model = Qwen3VLMoeForConditionalGeneration.from_pretrained(
"Qwen/Qwen3-VL-30B-A3B-Instruct", dtype="auto", device_map="auto"
)
model = self.get_model()
batched_messages = [self.message, self.message2]
inputs = self.processor.apply_chat_template(
batched_messages,
Expand Down
Loading