Repository navigation
[Whisper] Fix whisper tokenizer #34537
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 7 commits
751f88a
000dccd
70c8aac
3f9f7ea
e753206
c53fb2c
185fb55
429904c
1c72244
ad4f355
09af9de
7d6f9b4
937cd2a
9b1d51e
a3cbe9f
7c0da36
4a21249
5b195a8
e8f2f69
d71d40a
961d5f6
e04aa92
69abfe0
16b1ecb
5fba3e0
9624ce9
88587bb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -629,7 +629,7 @@ def generate( | |
| cur_bsz=cur_bsz, | ||
| batch_idx_map=batch_idx_map, | ||
| ) | ||
| time_offset = seek * time_precision / input_stride | ||
| time_offset = seek.to(torch.float64) * time_precision / input_stride | ||
| seek_num_frames = (max_frames - seek).clamp(max=num_segment_frames) | ||
|
|
||
| # 6.2 cut out next 30s segment from input features | ||
|
|
@@ -1799,6 +1799,9 @@ def _retrieve_segment( | |
| segments = [] | ||
| if single_timestamp_ending: | ||
| slices.append(len(seek_sequence)) | ||
| else: | ||
| # we want to include the last timestamp token in the last segment to know it was no single ending | ||
| slices[-1] += 1 | ||
|
Comment on lines
+1817
to
+1819
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That's the only way to know latter that we have a double token ending segment
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So you're offsetting the last timestamp by one when the last two tokens are timestamp ? let's say we have
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
|
|
||
| last_slice = 0 | ||
| # Add each segment to list of all segments | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -528,7 +528,7 @@ def basic_normalize(text, remove_diacritics=False): | |
| normalizer = BasicTextNormalizer(remove_diacritics=remove_diacritics) | ||
| return normalizer(text) | ||
|
|
||
| def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_precision=0.02) -> str: | ||
| def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_precision=0.02, segment_size=1500) -> str: | ||
| """ | ||
| Timestamp tokens are above the special tokens' id range and are ignored by `decode()`. This method decodes | ||
| given tokens with timestamps tokens annotated, e.g. "<|1.08|>". | ||
|
|
@@ -538,15 +538,22 @@ def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_pre | |
|
|
||
| cur_max_timestamp = 0.0 | ||
| prev_segments_len = 0.0 | ||
| # track if last timestamp was single ending | ||
| penultimate_timestamp = 0.0 | ||
|
|
||
| for token in token_ids: | ||
| if token >= timestamp_begin: | ||
| timestamp = float((token - timestamp_begin) * time_precision) | ||
|
|
||
| if timestamp < cur_max_timestamp: | ||
| # next segment has started | ||
| prev_segments_len += cur_max_timestamp | ||
| last_was_single_ending = not cur_max_timestamp == penultimate_timestamp | ||
| if last_was_single_ending: | ||
| prev_segments_len += time_precision * segment_size | ||
| else: | ||
| prev_segments_len += cur_max_timestamp | ||
|
|
||
| penultimate_timestamp = cur_max_timestamp | ||
| cur_max_timestamp = timestamp | ||
|
|
||
| outputs.append(f"<|{(timestamp + prev_segments_len):.2f}|>") | ||
|
|
@@ -558,7 +565,7 @@ def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_pre | |
| ] | ||
| return "".join(outputs) | ||
|
|
||
| def _compute_offsets(self, token_ids, time_precision=0.02): | ||
| def _compute_offsets(self, token_ids, time_precision=0.02, segment_size=1500): | ||
| """ | ||
| Compute offsets for a given tokenized input | ||
|
|
||
|
|
@@ -567,6 +574,8 @@ def _compute_offsets(self, token_ids, time_precision=0.02): | |
| List of tokenized input ids. Can be obtained using the `__call__` method. | ||
| time_precision (`float`, *optional*, defaults to 0.02): | ||
| The time ratio to convert from token to time. | ||
| segment_size (`int`, *optional*, defaults to 1500): | ||
| The number of features in the input mel spectrogram. | ||
| """ | ||
| offsets = [] | ||
| # ensure torch tensor of token ids is placed on cpu | ||
|
|
@@ -597,7 +606,12 @@ def _compute_offsets(self, token_ids, time_precision=0.02): | |
|
|
||
| if start_timestamp_position < cur_max_timestamp: | ||
| # next segment has started | ||
| prev_segments_len += cur_max_timestamp | ||
| # here in the worst case we have [<|start_timestamp_position before|>, <|cur_max_timestamp|>, <|start_timestamp_position|>], so last_slice (idx of start_timestamp_position) - 2 is safe | ||
| is_single_ending = not token_ids[last_slice - 2] == token_ids[last_slice - 1] | ||
| if is_single_ending: | ||
| prev_segments_len += segment_size | ||
| else: | ||
| prev_segments_len += cur_max_timestamp | ||
|
|
||
| cur_max_timestamp = end_timestamp_position | ||
|
|
||
|
|
@@ -609,8 +623,8 @@ def _compute_offsets(self, token_ids, time_precision=0.02): | |
| { | ||
| "text": text, | ||
| "timestamp": ( | ||
| (start_timestamp_position + prev_segments_len) * time_precision, | ||
| (end_timestamp_position + prev_segments_len) * time_precision, | ||
| start_timestamp_position * time_precision + prev_segments_len * time_precision, | ||
| end_timestamp_position * time_precision + prev_segments_len * time_precision, | ||
|
Comment on lines
+632
to
+633
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's done this way in the original codebase: summing float64 after the multiplication with the position. We avoid this way annoying floating-point arithmetic issues. |
||
| ), | ||
| } | ||
| ) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Original Whisper code base does such computations in float64. We need to ensure we do the same, especially wince we are comparing in the tests with the original Whisper outputs.