Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
751f88a
handle single timestamp ending
eustlb Oct 31, 2024
000dccd
include last timestamp token
eustlb Oct 31, 2024
70c8aac
handle single timestamp ending
eustlb Oct 31, 2024
3f9f7ea
Merge branch 'huggingface:main' into fix-whispertokenizer
eustlb Oct 31, 2024
e753206
avoid floating points arithm limitations
eustlb Oct 31, 2024
c53fb2c
ensure float64 operations
eustlb Oct 31, 2024
185fb55
new test
eustlb Oct 31, 2024
429904c
make fixup
eustlb Oct 31, 2024
1c72244
make copies
eustlb Oct 31, 2024
ad4f355
Merge branch 'main' into fix-whispertokenizer
eustlb Oct 31, 2024
09af9de
Merge branch 'main' into fix-whispertokenizer
eustlb Nov 2, 2024
7d6f9b4
handle edge case double tokens ending with different tokens
eustlb Nov 4, 2024
937cd2a
handle single timestamp ending
eustlb Nov 4, 2024
9b1d51e
make fixup
eustlb Nov 4, 2024
a3cbe9f
handle conditioning on prev segments
eustlb Nov 5, 2024
7c0da36
fix
eustlb Nov 5, 2024
4a21249
Update src/transformers/models/whisper/generation_whisper.py
eustlb Nov 21, 2024
5b195a8
Merge branch 'main' into fix-whispertokenizer
eustlb Nov 22, 2024
e8f2f69
[run-slow] whisper
Nov 27, 2024
d71d40a
Merge branch 'main' into fix-whispertokenizer
eustlb Nov 27, 2024
961d5f6
Merge branch 'main' into fix-whispertokenizer
eustlb Nov 27, 2024
e04aa92
Merge branch 'main' into fix-whispertokenizer
eustlb Dec 2, 2024
69abfe0
Merge branch 'main' into fix-whispertokenizer
eustlb Dec 4, 2024
16b1ecb
Merge branch 'main' into fix-whispertokenizer
eustlb Dec 4, 2024
5fba3e0
don't call item() to avoid unnecessary sync
eustlb Dec 5, 2024
9624ce9
Merge branch 'main' into fix-whispertokenizer
eustlb Dec 5, 2024
88587bb
fix
eustlb Dec 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/transformers/models/whisper/generation_whisper.py
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,7 @@ def generate(
cur_bsz=cur_bsz,
batch_idx_map=batch_idx_map,
)
time_offset = seek * time_precision / input_stride
time_offset = seek.to(torch.float64) * time_precision / input_stride

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Original Whisper code base does such computations in float64. We need to ensure we do the same, especially wince we are comparing in the tests with the original Whisper outputs.

seek_num_frames = (max_frames - seek).clamp(max=num_segment_frames)

# 6.2 cut out next 30s segment from input features
Expand Down Expand Up @@ -1799,6 +1799,9 @@ def _retrieve_segment(
segments = []
if single_timestamp_ending:
slices.append(len(seek_sequence))
else:
# we want to include the last timestamp token in the last segment to know it was no single ending
slices[-1] += 1
Comment on lines +1817 to +1819

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's the only way to know latter that we have a double token ending segment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So you're offsetting the last timestamp by one when the last two tokens are timestamp ? let's say we have [..., T1, T2], you're doing [..., T1, T2+1] ?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

slices is a list of indexes along the generated sequence of tokens seek_sequence. Let's say we have 100 tokens and it was not single ending, meaning last two tokens are timestamps [..., T1, T2]. For this reason, slices[-1] = 99 yet when we will slice after the segments with seek_sequence[last_slice:current_slice], we want to make sure we include T2 in the slice (so that the we further know it is a double timestamp ending segment as explained in the PR's comment) → by adding 1 to last slice, we ensure last iteration will slice 'seek_sequence[last_slice:100]` and that T2 will get included


last_slice = 0
# Add each segment to list of all segments
Expand Down
26 changes: 20 additions & 6 deletions src/transformers/models/whisper/tokenization_whisper.py
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,7 @@ def basic_normalize(text, remove_diacritics=False):
normalizer = BasicTextNormalizer(remove_diacritics=remove_diacritics)
return normalizer(text)

def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_precision=0.02) -> str:
def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_precision=0.02, segment_size=1500) -> str:
"""
Timestamp tokens are above the special tokens' id range and are ignored by `decode()`. This method decodes
given tokens with timestamps tokens annotated, e.g. "<|1.08|>".
Expand All @@ -538,15 +538,22 @@ def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_pre

cur_max_timestamp = 0.0
prev_segments_len = 0.0
# track if last timestamp was single ending
penultimate_timestamp = 0.0

for token in token_ids:
if token >= timestamp_begin:
timestamp = float((token - timestamp_begin) * time_precision)

if timestamp < cur_max_timestamp:
# next segment has started
prev_segments_len += cur_max_timestamp
last_was_single_ending = not cur_max_timestamp == penultimate_timestamp
if last_was_single_ending:
prev_segments_len += time_precision * segment_size
else:
prev_segments_len += cur_max_timestamp

penultimate_timestamp = cur_max_timestamp
cur_max_timestamp = timestamp

outputs.append(f"<|{(timestamp + prev_segments_len):.2f}|>")
Expand All @@ -558,7 +565,7 @@ def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_pre
]
return "".join(outputs)

def _compute_offsets(self, token_ids, time_precision=0.02):
def _compute_offsets(self, token_ids, time_precision=0.02, segment_size=1500):
"""
Compute offsets for a given tokenized input

Expand All @@ -567,6 +574,8 @@ def _compute_offsets(self, token_ids, time_precision=0.02):
List of tokenized input ids. Can be obtained using the `__call__` method.
time_precision (`float`, *optional*, defaults to 0.02):
The time ratio to convert from token to time.
segment_size (`int`, *optional*, defaults to 1500):
The number of features in the input mel spectrogram.
"""
offsets = []
# ensure torch tensor of token ids is placed on cpu
Expand Down Expand Up @@ -597,7 +606,12 @@ def _compute_offsets(self, token_ids, time_precision=0.02):

if start_timestamp_position < cur_max_timestamp:
# next segment has started
prev_segments_len += cur_max_timestamp
# here in the worst case we have [<|start_timestamp_position before|>, <|cur_max_timestamp|>, <|start_timestamp_position|>], so last_slice (idx of start_timestamp_position) - 2 is safe
is_single_ending = not token_ids[last_slice - 2] == token_ids[last_slice - 1]
if is_single_ending:
prev_segments_len += segment_size
else:
prev_segments_len += cur_max_timestamp

cur_max_timestamp = end_timestamp_position

Expand All @@ -609,8 +623,8 @@ def _compute_offsets(self, token_ids, time_precision=0.02):
{
"text": text,
"timestamp": (
(start_timestamp_position + prev_segments_len) * time_precision,
(end_timestamp_position + prev_segments_len) * time_precision,
start_timestamp_position * time_precision + prev_segments_len * time_precision,
end_timestamp_position * time_precision + prev_segments_len * time_precision,
Comment on lines +632 to +633

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's done this way in the original codebase: summing float64 after the multiplication with the position. We avoid this way annoying floating-point arithmetic issues.

),
}
)
Expand Down
87 changes: 87 additions & 0 deletions tests/models/whisper/test_modeling_whisper.py
Original file line number Diff line number Diff line change
Expand Up @@ -2128,6 +2128,93 @@ def test_tiny_longform_timestamps_generation(self):
transcript = processor.batch_decode(generated_ids["sequences"], skip_special_tokens=True, output_offsets=True)
self.assertEqual(transcript[0]["offsets"], EXPECTED_TRANSCRIPT)

@slow
def test_small_longform_timestamps_generation(self):
processor = WhisperProcessor.from_pretrained("openai/whisper-small.en")
model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-small.en")
model.to(torch_device)

dataset = load_dataset("distil-whisper/librispeech_long", "clean", split="validation")
sample = dataset[0]["audio"]["array"]
sampling_rate = dataset[0]["audio"]["sampling_rate"]

sample = [*sample[:15 * sampling_rate], *np.zeros(16 * sampling_rate).tolist(), *sample[15 * sampling_rate:]]
sample = np.array(sample)

input_features = processor(
sample,
sampling_rate=16_000,
padding="longest",
truncation=False,
return_attention_mask=True,
return_tensors="pt",
).input_features

input_features = input_features.to(torch_device)
generated_ids = model.generate(input_features, return_timestamps=True, return_segments=True)

EXPECTED_TRANSCRIPT = [
{
"text": " Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.",
"timestamp": (0.0, 6.38),
},
{
"text": " Nor is Mr. Quilter's manner less interesting than his matter.",
"timestamp": (6.38, 11.32),
},
{
"text": " He tells us that at this festive season of the year,",
"timestamp": (11.32, 15.0),
},
{
"text": " With Christmas and roast beef looming before us, similes drawn from eating and its results",
"timestamp": (30.0, 36.76),
},
{
"text": " occur most readily to the mind.",
"timestamp": (36.76, 39.80),
},
{
"text": " He has grave doubts whether Sir Frederick Layton's work is really Greek after all and",
"timestamp": (39.80, 45.36),
},
{
"text": " can discover in it but little of rocky Ithaca.",
"timestamp": (45.36, 49.0),
},
{
"text": " Lenell's pictures are a sort of up-guards-and-atom paintings, and Mason's exquisite ittles",
"timestamp": (49.0, 56.28),
},
{
"text": " are as national as a jingo poem. Mr. Burkett fosters landscape's smile at one much in",
"timestamp": (56.28, 64.12),
},
{
"text": " the same way that Mr. Karker used to flash his teeth. And Mr. John Collier gives his",
"timestamp": (64.12, 70.76),
},
{
"text": " sitter a cheerful slap on the back before he says, like a shampoo or in a Turkish bath,",
"timestamp": (70.76, 77.16),
},
{
"text": " Next Man",
"timestamp": (77.16, 78.16),
},
]

transcript = processor.batch_decode(generated_ids["sequences"], skip_special_tokens=True, output_offsets=True)
self.assertEqual(transcript[0]["offsets"], EXPECTED_TRANSCRIPT)

transcript_segments = [
{
"text": processor.decode(seg["tokens"], skip_special_tokens=True),
"timestamp": (seg["start"].item(), seg["end"].item())
} for seg in generated_ids["segments"][0]
]
self.assertEqual(transcript_segments, EXPECTED_TRANSCRIPT)

Comment thread
eustlb marked this conversation as resolved.
Resolved
@slow
def test_large_timestamp_generation(self):
set_seed(0)
Expand Down