Repository navigation
[pipelines] text-to-audio pipeline standardization #39796
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
7788a27
2421868
33ead43
d7fb429
5714b8b
6ef93c5
85d9c5b
779e1e9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -192,6 +192,9 @@ def save_audio( | |
| for audio_value, p in zip(audio, saving_path): | ||
| if isinstance(audio_value, torch.Tensor): | ||
| audio_value = audio_value.cpu().float().numpy() | ||
| if audio_value.ndim == 2 and audio_value.shape[0] in (1, 2): | ||
| # (nb_channels, audio_length) -> (audio_length, nb_channels), as expected by `soundfile` | ||
| audio_value = np.transpose(audio_value) | ||
|
Comment on lines
+195
to
+197
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't get why you're adding this for CSM and Dia. The fact that the models are mono is more of a design choice than a parameter, and such things should not change. Nevertheless, if we add a
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agree with centralizing/standardizing a Another idea @eustlb, in several audio libraries I've seen the use of an import soundfile as sf
import numpy as np
import os
class Audio:
def __init__(self, waveform: np.ndarray, sample_rate: int, path: str = None):
"""
Args:
waveform: NumPy array of shape (channels, samples).
sample_rate: Sampling rate in Hz.
path: Optional original file path.
"""
if waveform.ndim == 1:
waveform = waveform[np.newaxis, :] # Convert to (1, samples)
elif waveform.shape[0] > waveform.shape[1]:
# Assume it's (samples, channels) — transpose to (channels, samples)
waveform = waveform.T
self.waveform = waveform
self.sample_rate = sample_rate
self.path = path
@classmethod
def load(cls, path: str):
"""
Load audio from file (converted to (channels, samples)).
"""
if not os.path.isfile(path):
raise FileNotFoundError(f"Audio file not found: {path}")
data, sample_rate = sf.read(path, always_2d=True) # (samples, channels)
waveform = data.T # Transpose to (channels, samples)
return cls(waveform, sample_rate, path)
def save(self, path: str):
"""
Save audio to file (converted to (samples, channels)).
"""
data = self.waveform.T # Convert to (samples, channels)
sf.write(path, data, self.sample_rate)
self.path = path
def duration_seconds(self) -> float:
return self.waveform.shape[1] / self.sample_rate
# can have others things for spectrogram, MFCC, augmentations, etc which can call functional versionsIt can make working audio with much more convenient (not having to pass sample rate each time and channel dimension consistency). And can be used alongside functional approach |
||
| sf.write(p, audio_value, sampling_rate) | ||
|
|
||
| def __call__( | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,7 +43,7 @@ | |
|
|
||
| from ...cache_utils import Cache | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Trusting you on that one! |
||
| from ...configuration_utils import PretrainedConfig, layer_type_validation | ||
| from ...generation import GenerationMixin | ||
| from ...generation import GenerateDecoderOnlyOutput, GenerationMixin | ||
| from ...modeling_flash_attention_utils import is_flash_attn_available | ||
| from ...modeling_outputs import BaseModelOutput, ModelOutput | ||
| from ...modeling_rope_utils import rope_config_validation | ||
|
|
@@ -3966,6 +3966,38 @@ def forward( | |
| ############################ | ||
|
|
||
|
|
||
| @dataclass | ||
| class Qwen2_5OmniGenerateOutput(GenerateDecoderOnlyOutput): | ||
| """ | ||
| Outputs of Qwen2_5OmniForConditionalGeneration.generate. | ||
|
|
||
| Args: | ||
| sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`): | ||
| The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter | ||
| if all batches finished early due to the `eos_token_id`. | ||
| scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True`): | ||
| Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax) | ||
| at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for | ||
| each generated token), with each tensor of shape `(batch_size, config.vocab_size)`. | ||
| logits (`tuple(torch.FloatTensor)` *optional*, returned when `output_logits=True`): | ||
| Unprocessed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax) | ||
| at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for | ||
| each generated token), with each tensor of shape `(batch_size, config.vocab_size)`. | ||
| attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True`): | ||
| Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of | ||
| `torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`. | ||
| hidden_states (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_hidden_states=True`): | ||
| Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of | ||
| `torch.FloatTensor` of shape `(batch_size, generated_length, hidden_size)`. | ||
| past_key_values (`tuple(tuple(torch.FloatTensor)))`, *optional*, returned when `use_cache=True`): | ||
| Returns the model cache, used to speed up decoding. Different models have a different cache format, check | ||
| audio (`torch.FloatTensor` of shape `(sequence_length)`): | ||
| The generated audio. Qwen2.5Omni does not support batched inference with audio output. | ||
| """ | ||
|
|
||
| audio: Optional[torch.Tensor] = None | ||
|
|
||
|
|
||
| @auto_docstring( | ||
| custom_intro=""" | ||
| The full Qwen2.5Omni model, a multimodal model composed of 3 sub-models: | ||
|
|
@@ -4078,8 +4110,9 @@ def generate( | |
| talker_temperature: float = 0.9, | ||
| talker_eos_token_id: list[int] = [8292, 8294], | ||
| talker_repetition_penalty: float = 1.05, | ||
| return_dict_in_generate: bool = False, | ||
| **kwargs, | ||
| ): | ||
| ) -> Union[Qwen2_5OmniGenerateOutput, torch.LongTensor, tuple[torch.LongTensor, torch.FloatTensor]]: | ||
| r""" | ||
| Generate text response and audio from input. | ||
|
|
||
|
|
@@ -4097,6 +4130,7 @@ def generate( | |
| - With a *thinker_*, *talker_*, *token2wav_* prefix, they will be input for the `generate` method of the | ||
| thinker, talker and token2wav respectively. It has the priority over the keywords without a prefix. | ||
| Returns: | ||
| A `Qwen2_5OmniGenerateOutput` when `return_dict_in_generate=True`, otherwise: | ||
| When `return_audio=False`: | ||
| - **Text** (`torch.Tensor`): Generated text token sequence. | ||
| When `return_audio=True`: | ||
|
|
@@ -4165,7 +4199,10 @@ def generate( | |
| thinker_result = self.thinker.generate(input_ids=input_ids, **thinker_kwargs) | ||
|
|
||
| if not generate_audio: | ||
| return thinker_result | ||
| if return_dict_in_generate: | ||
| return Qwen2_5OmniGenerateOutput(sequences=thinker_result) | ||
| else: | ||
| return thinker_result | ||
|
|
||
| # 2. Generate speech tokens from talker module | ||
| embeds_to_talker = thinker_result.hidden_states[0][0].clone().to(input_ids.device) | ||
|
|
@@ -4286,7 +4323,10 @@ def generate( | |
| **token2wav_kwargs, | ||
| ) | ||
|
|
||
| return thinker_result.sequences, wav.float() | ||
| if return_dict_in_generate: | ||
| return Qwen2_5OmniGenerateOutput(sequences=thinker_result.sequences, audio=wav.float()) | ||
| else: | ||
| return thinker_result.sequences, wav.float() | ||
|
|
||
|
|
||
| __all__ = [ | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nice catch!