Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/transformers/models/auto/feature_extraction_auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ def register(config_class, feature_extractor_class, exist_ok=False):
Args:
config_class ([`PretrainedConfig`]):
The configuration corresponding to the model to register.
feature_extractor_class ([`FeatureExtractorMixin`]): The feature extractor to register.
feature_extractor_class ([`FeatureExtractionMixin`]): The feature extractor to register.
"""
FEATURE_EXTRACTOR_MAPPING.register(config_class, feature_extractor_class, exist_ok=exist_ok)

Expand Down
2 changes: 1 addition & 1 deletion src/transformers/models/csm/generation_csm.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ def generate(
streamer: Optional["BaseStreamer"] = None,
output_audio: Optional[bool] = False,
**kwargs,
) -> Union[GenerateNonBeamOutput, torch.LongTensor]:
) -> Union[CsmGenerateOutput, torch.LongTensor, list[torch.FloatTensor]]:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice catch!

r"""
This method overrides [`~generation.utils.GenerationMixin.generate`] to match the specifics of the Csm model.
Indeed, Csm model requires a custom generation sampling step:
Expand Down
3 changes: 3 additions & 0 deletions src/transformers/models/csm/processing_csm.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,9 @@ def save_audio(
for audio_value, p in zip(audio, saving_path):
if isinstance(audio_value, torch.Tensor):
audio_value = audio_value.cpu().float().numpy()
if audio_value.ndim == 2 and audio_value.shape[0] in (1, 2):
# (nb_channels, audio_length) -> (audio_length, nb_channels), as expected by `soundfile`
audio_value = np.transpose(audio_value)
Comment on lines +195 to +197

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't get why you're adding this for CSM and Dia. The fact that the models are mono is more of a design choice than a parameter, and such things should not change.

Nevertheless, if we add a save_audio in audio_utils and rely on this function for saving audio in the processor, it will make sense to have it there, WDYT?

@ebezzam ebezzam Jul 31, 2025 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agree with centralizing/standardizing a save_audio function in audio_utils to avoid code duplication in different processors.

Another idea @eustlb, in several audio libraries I've seen the use of an Audio objects to group together useful properties and methods. WDYT? Here's an example below (quickly drafted with ChatGPT):

import soundfile as sf
import numpy as np
import os

class Audio:
    def __init__(self, waveform: np.ndarray, sample_rate: int, path: str = None):
        """
        Args:
            waveform: NumPy array of shape (channels, samples).
            sample_rate: Sampling rate in Hz.
            path: Optional original file path.
        """
        if waveform.ndim == 1:
            waveform = waveform[np.newaxis, :]  # Convert to (1, samples)
        elif waveform.shape[0] > waveform.shape[1]:
            # Assume it's (samples, channels) — transpose to (channels, samples)
            waveform = waveform.T

        self.waveform = waveform
        self.sample_rate = sample_rate
        self.path = path

    @classmethod
    def load(cls, path: str):
        """
        Load audio from file (converted to (channels, samples)).
        """
        if not os.path.isfile(path):
            raise FileNotFoundError(f"Audio file not found: {path}")

        data, sample_rate = sf.read(path, always_2d=True)  # (samples, channels)
        waveform = data.T  # Transpose to (channels, samples)
        return cls(waveform, sample_rate, path)

    def save(self, path: str):
        """
        Save audio to file (converted to (samples, channels)).
        """
        data = self.waveform.T  # Convert to (samples, channels)
        sf.write(path, data, self.sample_rate)
        self.path = path

    def duration_seconds(self) -> float:
        return self.waveform.shape[1] / self.sample_rate

    # can have others things for spectrogram, MFCC, augmentations, etc which can call functional versions

It can make working audio with much more convenient (not having to pass sample rate each time and channel dimension consistency). And can be used alongside functional approach

sf.write(p, audio_value, sampling_rate)

def __call__(
Expand Down
5 changes: 5 additions & 0 deletions src/transformers/models/dia/processing_dia.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
from pathlib import Path
from typing import Optional, Union

import numpy as np

from ...audio_utils import AudioInput, make_list_of_audio
from ...feature_extraction_utils import BatchFeature
from ...processing_utils import AudioKwargs, ProcessingKwargs, ProcessorMixin, Unpack
Expand Down Expand Up @@ -406,6 +408,9 @@ def save_audio(
for audio_value, p in zip(audio, saving_path):
if isinstance(audio_value, torch.Tensor):
audio_value = audio_value.cpu().float().numpy()
if audio_value.ndim == 2 and audio_value.shape[0] in (1, 2):
# (nb_channels, audio_length) -> (audio_length, nb_channels), as expected by `soundfile`
audio_value = np.transpose(audio_value)
sf.write(p, audio_value, sampling_rate)

@staticmethod
Expand Down
48 changes: 44 additions & 4 deletions src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@

from ...activations import ACT2FN
from ...cache_utils import Cache, DynamicCache
from ...generation import GenerationMixin
from ...generation import GenerateDecoderOnlyOutput, GenerationMixin
from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask
from ...modeling_flash_attention_utils import FlashAttentionKwargs
from ...modeling_layers import GradientCheckpointingLayer
Expand Down Expand Up @@ -3669,6 +3669,38 @@ def forward(
############################


@dataclass
class Qwen2_5OmniGenerateOutput(GenerateDecoderOnlyOutput):
"""
Outputs of Qwen2_5OmniForConditionalGeneration.generate.

Args:
sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
if all batches finished early due to the `eos_token_id`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for
each generated token), with each tensor of shape `(batch_size, config.vocab_size)`.
logits (`tuple(torch.FloatTensor)` *optional*, returned when `output_logits=True`):
Unprocessed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for
each generated token), with each tensor of shape `(batch_size, config.vocab_size)`.
attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
hidden_states (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_hidden_states=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size, generated_length, hidden_size)`.
past_key_values (`tuple(tuple(torch.FloatTensor)))`, *optional*, returned when `use_cache=True`):
Returns the model cache, used to speed up decoding. Different models have a different cache format, check
audio (`torch.FloatTensor` of shape `(sequence_length)`):
The generated audio. Qwen2.5Omni does not support batched inference with audio output.
"""

audio: Optional[torch.Tensor] = None


@auto_docstring(
custom_intro="""
The full Qwen2.5Omni model, a multimodal model composed of 3 sub-models:
Expand Down Expand Up @@ -3781,8 +3813,9 @@ def generate(
talker_temperature: float = 0.9,
talker_eos_token_id: list[int] = [8292, 8294],
talker_repetition_penalty: float = 1.05,
return_dict_in_generate: bool = False,
**kwargs,
):
) -> Union[Qwen2_5OmniGenerateOutput, torch.LongTensor, tuple[torch.LongTensor, torch.FloatTensor]]:
r"""
Generate text response and audio from input.

Expand All @@ -3800,6 +3833,7 @@ def generate(
- With a *thinker_*, *talker_*, *token2wav_* prefix, they will be input for the `generate` method of the
thinker, talker and token2wav respectively. It has the priority over the keywords without a prefix.
Returns:
A `Qwen2_5OmniGenerateOutput` when `return_dict_in_generate=True`, otherwise:
When `return_audio=False`:
- **Text** (`torch.Tensor`): Generated text token sequence.
When `return_audio=True`:
Expand Down Expand Up @@ -3868,7 +3902,10 @@ def generate(
thinker_result = self.thinker.generate(input_ids=input_ids, **thinker_kwargs)

if not generate_audio:
return thinker_result
if return_dict_in_generate:
return Qwen2_5OmniGenerateOutput(sequences=thinker_result)
else:
return thinker_result

# 2. Generate speech tokens from talker module
embeds_to_talker = thinker_result.hidden_states[0][0].clone().to(input_ids.device)
Expand Down Expand Up @@ -3989,7 +4026,10 @@ def generate(
**token2wav_kwargs,
)

return thinker_result.sequences, wav.float()
if return_dict_in_generate:
return Qwen2_5OmniGenerateOutput(sequences=thinker_result.sequences, audio=wav.float())
else:
return thinker_result.sequences, wav.float()


__all__ = [
Expand Down
48 changes: 44 additions & 4 deletions src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@

from ...cache_utils import Cache

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Trusting you on that one!

from ...configuration_utils import PretrainedConfig, layer_type_validation
from ...generation import GenerationMixin
from ...generation import GenerateDecoderOnlyOutput, GenerationMixin
from ...modeling_flash_attention_utils import is_flash_attn_available
from ...modeling_outputs import BaseModelOutput, ModelOutput
from ...modeling_rope_utils import rope_config_validation
Expand Down Expand Up @@ -3966,6 +3966,38 @@ def forward(
############################


@dataclass
class Qwen2_5OmniGenerateOutput(GenerateDecoderOnlyOutput):
"""
Outputs of Qwen2_5OmniForConditionalGeneration.generate.

Args:
sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
if all batches finished early due to the `eos_token_id`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for
each generated token), with each tensor of shape `(batch_size, config.vocab_size)`.
logits (`tuple(torch.FloatTensor)` *optional*, returned when `output_logits=True`):
Unprocessed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for
each generated token), with each tensor of shape `(batch_size, config.vocab_size)`.
attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
hidden_states (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_hidden_states=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size, generated_length, hidden_size)`.
past_key_values (`tuple(tuple(torch.FloatTensor)))`, *optional*, returned when `use_cache=True`):
Returns the model cache, used to speed up decoding. Different models have a different cache format, check
audio (`torch.FloatTensor` of shape `(sequence_length)`):
The generated audio. Qwen2.5Omni does not support batched inference with audio output.
"""

audio: Optional[torch.Tensor] = None


@auto_docstring(
custom_intro="""
The full Qwen2.5Omni model, a multimodal model composed of 3 sub-models:
Expand Down Expand Up @@ -4078,8 +4110,9 @@ def generate(
talker_temperature: float = 0.9,
talker_eos_token_id: list[int] = [8292, 8294],
talker_repetition_penalty: float = 1.05,
return_dict_in_generate: bool = False,
**kwargs,
):
) -> Union[Qwen2_5OmniGenerateOutput, torch.LongTensor, tuple[torch.LongTensor, torch.FloatTensor]]:
r"""
Generate text response and audio from input.

Expand All @@ -4097,6 +4130,7 @@ def generate(
- With a *thinker_*, *talker_*, *token2wav_* prefix, they will be input for the `generate` method of the
thinker, talker and token2wav respectively. It has the priority over the keywords without a prefix.
Returns:
A `Qwen2_5OmniGenerateOutput` when `return_dict_in_generate=True`, otherwise:
When `return_audio=False`:
- **Text** (`torch.Tensor`): Generated text token sequence.
When `return_audio=True`:
Expand Down Expand Up @@ -4165,7 +4199,10 @@ def generate(
thinker_result = self.thinker.generate(input_ids=input_ids, **thinker_kwargs)

if not generate_audio:
return thinker_result
if return_dict_in_generate:
return Qwen2_5OmniGenerateOutput(sequences=thinker_result)
else:
return thinker_result

# 2. Generate speech tokens from talker module
embeds_to_talker = thinker_result.hidden_states[0][0].clone().to(input_ids.device)
Expand Down Expand Up @@ -4286,7 +4323,10 @@ def generate(
**token2wav_kwargs,
)

return thinker_result.sequences, wav.float()
if return_dict_in_generate:
return Qwen2_5OmniGenerateOutput(sequences=thinker_result.sequences, audio=wav.float())
else:
return thinker_result.sequences, wav.float()


__all__ = [
Expand Down
2 changes: 0 additions & 2 deletions src/transformers/pipelines/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1427,8 +1427,6 @@ def __call__(self, inputs, *args, num_workers=None, batch_size=None, **kwargs):
is_list = isinstance(inputs, list)

is_iterable = is_dataset or is_generator or is_list

# TODO make the get_iterator work also for `tf` (and `flax`).
can_use_iterator = self.framework == "pt" and (is_dataset or is_generator or is_list)

if is_list:
Expand Down
Loading