Skip to content
Closed
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions tokenizers/src/pre_tokenizers/byte_level.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,24 @@ impl PostProcessor for ByteLevel {

<dyn PostProcessor>::default_process(encoding, pair_encoding, add_special_tokens)
}

fn process_chain(
&self,
mut encodings: Vec<Encoding>,
_add_special_tokens: bool,
) -> Result<Vec<Encoding>> {
for encoding in encodings.iter_mut() {
if self.trim_offsets {
process_offsets(encoding, self.add_prefix_space);
encoding
.get_overflowing_mut()
.iter_mut()
.for_each(|encoding| process_offsets(encoding, self.add_prefix_space));
}
}

Ok(encodings)
}
Comment thread
mishig25 marked this conversation as resolved.
Resolved
}

pub fn process_offsets(encoding: &mut Encoding, add_prefix_space: bool) {
Expand Down Expand Up @@ -529,6 +547,44 @@ mod tests {
);
}

#[test]
fn processer_chain() {
let start = Encoding::new(
vec![0; 5],
vec![],
vec![
"Ġ".into(),
"ĠĠĠĠHelloĠĠ".into(),
"ĠĠHello".into(),
"HelloĠĠ".into(),
"ĠĠĠĠ".into(),
],
vec![],

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We probably want to check some type_ids and other vec being filled here.

(Maybe you added the tests directly within the merge function which would be better, will have to check)

vec![(0, 1), (0, 11), (11, 18), (18, 25), (25, 29)],
vec![],
vec![],
vec![],
HashMap::new(),
);

let bytelevel = ByteLevel::default().trim_offsets(true);
let expected_encoding = bytelevel.process(start.clone(), None, false).unwrap();
let expected = vec![expected_encoding.clone()];
let expected_pair = vec![expected_encoding.clone(), expected_encoding.clone()];

assert_eq!(
expected,
bytelevel.process_chain(vec![start.clone()], false).unwrap()
);

assert_eq!(
expected_pair,
bytelevel
.process_chain(vec![start.clone(), start.clone()], false)
.unwrap()
);
}

#[test]
fn decode_unknown_characters() {
let byte_level = ByteLevel::default();
Expand Down
80 changes: 80 additions & 0 deletions tokenizers/src/processors/bert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,86 @@ impl PostProcessor for BertProcessing {

Ok(new_encoding)
}

fn process_chain(
&self,
encodings: Vec<Encoding>,
add_special_tokens: bool,
) -> Result<Vec<Encoding>> {
let encodings = encodings
.into_iter()
.map(|mut encoding| {
if !add_special_tokens {
return encoding;
}

let ids = [&[self.cls.1], encoding.get_ids(), &[self.sep.1]].concat();
let type_ids = [&[0], encoding.get_type_ids(), &[0]].concat();
let tokens = [
&[self.cls.0.clone()],
encoding.get_tokens(),
&[self.sep.0.clone()],
]
.concat();
let words = [&[None], encoding.get_word_ids(), &[None]].concat();
let offsets = [&[(0, 0)], encoding.get_offsets(), &[(0, 0)]].concat();
let special_tokens =
[&[1u32], &vec![0; encoding.get_ids().len()][..], &[1]].concat();
let attention_mask = vec![1; ids.len()];

// For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't contain
// the special tokens.
let sequence_ranges = HashMap::from_iter(vec![(0, 1..ids.len() - 1)]);

Encoding::new(
ids,
type_ids,
tokens,
words,
offsets,
special_tokens,
attention_mask,
encoding
.take_overflowing()
.into_iter()
.map(|encoding| {
let ids = [&[self.cls.1], encoding.get_ids(), &[self.sep.1]].concat();
let type_ids = [&[0], encoding.get_type_ids(), &[0]].concat();
let tokens = [
&[self.cls.0.clone()],
encoding.get_tokens(),
&[self.sep.0.clone()],
]
.concat();
let words = [&[None], encoding.get_word_ids(), &[None]].concat();
let offsets = [&[(0, 0)], encoding.get_offsets(), &[(0, 0)]].concat();
let special_tokens =
[&[1u32], &vec![0; encoding.get_ids().len()][..], &[1]].concat();
let attention_mask = vec![1; ids.len()];

// For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't
// contain the special tokens.
let sequence_ranges = HashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
Encoding::new(
ids,
type_ids,
tokens,
words,
offsets,
special_tokens,
attention_mask,
vec![],
sequence_ranges,
)
})
.collect(),
sequence_ranges,
)
})
.collect::<Vec<_>>();

Ok(encodings)
}
}

#[cfg(test)]
Expand Down
21 changes: 21 additions & 0 deletions tokenizers/src/processors/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
pub mod bert;
pub mod roberta;
pub mod sequence;
pub mod template;

// Re-export these as processors
Expand All @@ -10,6 +11,7 @@ use serde::{Deserialize, Serialize};
use crate::pre_tokenizers::byte_level::ByteLevel;
use crate::processors::bert::BertProcessing;
use crate::processors::roberta::RobertaProcessing;
use crate::processors::sequence::Sequence;
use crate::processors::template::TemplateProcessing;
use crate::{Encoding, PostProcessor, Result};

Expand All @@ -21,6 +23,7 @@ pub enum PostProcessorWrapper {
Bert(BertProcessing),
ByteLevel(ByteLevel),
Template(TemplateProcessing),
Sequence(Sequence),
}

impl PostProcessor for PostProcessorWrapper {
Expand All @@ -30,6 +33,7 @@ impl PostProcessor for PostProcessorWrapper {
Self::ByteLevel(bl) => bl.added_tokens(is_pair),
Self::Roberta(roberta) => roberta.added_tokens(is_pair),
Self::Template(template) => template.added_tokens(is_pair),
Self::Sequence(sequence) => sequence.added_tokens(is_pair),
}
}

Expand All @@ -46,6 +50,23 @@ impl PostProcessor for PostProcessorWrapper {
Self::Template(template) => {
template.process(encoding, pair_encoding, add_special_tokens)
}
Self::Sequence(sequence) => {
sequence.process(encoding, pair_encoding, add_special_tokens)
}
}
}

fn process_chain(
&self,
encodings: Vec<Encoding>,
add_special_tokens: bool,
) -> Result<Vec<Encoding>> {
match self {
Self::Bert(bert) => bert.process_chain(encodings, add_special_tokens),
Self::ByteLevel(bl) => bl.process_chain(encodings, add_special_tokens),
Self::Roberta(roberta) => roberta.process_chain(encodings, add_special_tokens),
Self::Template(template) => template.process_chain(encodings, add_special_tokens),
Self::Sequence(sequence) => sequence.process_chain(encodings, add_special_tokens),
}
}
}
Expand Down
88 changes: 88 additions & 0 deletions tokenizers/src/processors/roberta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,94 @@ impl PostProcessor for RobertaProcessing {

Ok(new_encoding)
}

fn process_chain(
&self,
encodings: Vec<Encoding>,
add_special_tokens: bool,
) -> Result<Vec<Encoding>> {
let encodings = encodings
.into_iter()
.map(|mut encoding| {
if self.trim_offsets {
process_offsets(&mut encoding, self.add_prefix_space);
encoding
.get_overflowing_mut()
.iter_mut()
.for_each(|encoding| process_offsets(encoding, self.add_prefix_space));
}

if !add_special_tokens {
return encoding;
}

let ids = [&[self.cls.1], encoding.get_ids(), &[self.sep.1]].concat();
let type_ids = [&[0], encoding.get_type_ids(), &[0]].concat();
let tokens = [
&[self.cls.0.clone()],
encoding.get_tokens(),
&[self.sep.0.clone()],
]
.concat();
let words = [&[None], encoding.get_word_ids(), &[None]].concat();
let offsets = [&[(0, 0)], encoding.get_offsets(), &[(0, 0)]].concat();
let special_tokens =
[&[1u32], &vec![0; encoding.get_ids().len()][..], &[1]].concat();
let attention_mask = vec![1; ids.len()];

// For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't contain
// the special tokens.
let sequence_ranges = HashMap::from_iter(vec![(0, 1..ids.len() - 1)]);

Encoding::new(
ids,
type_ids,
tokens,
words,
offsets,
special_tokens,
attention_mask,
encoding
.take_overflowing()
.into_iter()
.map(|encoding| {
let ids = [&[self.cls.1], encoding.get_ids(), &[self.sep.1]].concat();
let type_ids = [&[0], encoding.get_type_ids(), &[0]].concat();
let tokens = [
&[self.cls.0.clone()],
encoding.get_tokens(),
&[self.sep.0.clone()],
]
.concat();
let words = [&[None], encoding.get_word_ids(), &[None]].concat();
let offsets = [&[(0, 0)], encoding.get_offsets(), &[(0, 0)]].concat();
let special_tokens =
[&[1u32], &vec![0; encoding.get_ids().len()][..], &[1]].concat();
let attention_mask = vec![1; ids.len()];

// For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't
// contain the special tokens.
let sequence_ranges = HashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
Encoding::new(
ids,
type_ids,
tokens,
words,
offsets,
special_tokens,
attention_mask,
vec![],
sequence_ranges,
)
})
.collect(),
sequence_ranges,
)
})
.collect::<Vec<_>>();

Ok(encodings)
}
}

#[cfg(test)]
Expand Down
Loading