Skip to content

Tokenizer fixes #8379

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 9 commits into
base: master
Choose a base branch
from
Draft

Tokenizer fixes #8379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e8b3955
Fix pyparse problems: gcc inline functions
Jul 8, 2024
9307c3f
Test l/r-strip for more than 4 spaces
Jul 8, 2024
a943b42
Improve mismatch range localization
Jul 8, 2024
dec64ef
Compare vocabs
Jul 8, 2024
c184db7
Options to mange token text decoding errors:
Jul 8, 2024
3eb1900
Skip literal UNUSED token checks
Jul 9, 2024
c4956e4
update test: fix special and added token lists
Jul 11, 2024
9b8e05b
Merge commit 'f4444d99' into tokenizer-fixes
Jul 11, 2024
3db5058
Merge branch 'master' into tokenizer-fixes
Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions src/llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21195,7 +21195,8 @@ int32_t llama_tokenize(
return res.size();
}

static std::string llama_decode_text(const std::string & text) {
// errors: 'c': copy, 'i': ignore, 'r': replace 0xFFFD, 'v': verbose
static std::string llama_decode_text(const std::string & text, const char errors = 'v') {
std::string decoded_text;

const auto cpts = unicode_cpts_from_utf8(text);
Expand All @@ -21204,11 +21205,21 @@ static std::string llama_decode_text(const std::string & text) {
try {
decoded_text += unicode_utf8_to_byte(utf8);
} catch (const std::out_of_range & /*e*/) {
decoded_text += "[UNK_BYTE_0x";
for (const auto c : utf8) {
decoded_text += format("%02x", (uint8_t) c);
switch (errors) {
case 'c':
decoded_text += utf8; // copy original
break;
case 'r':
decoded_text += "\xEF\xBF\xBD"; // 0xFFFD REPLACEMENT CHARACTER
break;
case 'v':
decoded_text += format("[UNK_BYTE_0x%02X]", cpt);
break;
case 'i':
default:
// ignore
break;
}
decoded_text += text + "]";
}
}

Expand Down Expand Up @@ -21274,7 +21285,7 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token
if (attr & (attr_special | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
return _try_copy(token_text.data(), token_text.size());
} else if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
std::string result = llama_decode_text(token_text);
std::string result = llama_decode_text(token_text, 'c'); // copy on error //TODO: use a tokenizer variable
return _try_copy(result.data(), result.size());
}
break;
Expand Down
153 changes: 119 additions & 34 deletions tests/test-tokenizer-random.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,25 @@
num = self.lib.llama_detokenize(self.model, self.token_ids, len(ids), self.text_buff, len(self.text_buff), remove_special, unparse_special)
return str(cast(Buffer, self.ffi.buffer(self.text_buff, num)), encoding="utf-8", errors="replace") # replace errors with '\uFFFD'

def get_vocab(self, detokenize=False) -> list[str]:
vocab: list[str] = []
num_tokens = self.lib.llama_n_vocab(self.model)
for id in range(num_tokens):
if detokenize:
text = self.detokenize([id], remove_special=False, unparse_special=True)
else:
text = self.lib.llama_token_get_text(self.model, id)
text = self.ffi.string(text)
text = str(text, encoding="utf-8", errors="replace") # replace errors with '\uFFFD'

Check failure on line 128 in tests/test-tokenizer-random.py

View workflow job for this annotation

GitHub Actions / pyright type-check 

Argument of type "bytes | str" cannot be assigned to parameter "object" of type "ReadableBuffer" in function "__new__"
  Type "bytes | str" is incompatible with type "ReadableBuffer"
    "str" is incompatible with protocol "Buffer"
      "__buffer__" is not present (reportArgumentType)
vocab.append(text)
return vocab


class Tokenizer:

def get_vocab(self, detokenize=False) -> list[str]:
raise NotImplementedError

def encode(self, text: str) -> list[int]:
raise NotImplementedError

Expand All @@ -129,7 +145,7 @@
class TokenizerGroundtruth (Tokenizer):

def __init__(self, dir_tokenizer: str):
self.model: PreTrainedTokenizer = AutoTokenizer.from_pretrained(dir_tokenizer)
self.model: PreTrainedTokenizer = AutoTokenizer.from_pretrained(dir_tokenizer, trust_remote_code=False)
# guess BOS and EOS
ids = self.encode("a")
assert 1 <= len(ids) <= 3
Expand All @@ -138,15 +154,24 @@
self.add_bos_token = getattr(self.model, "add_bos_token", add_bos_token)
self.add_eos_token = getattr(self.model, "add_eos_token", add_eos_token)
# build vocab
tokens = list(self.model.get_vocab().values())
self.vocab = self.model.batch_decode(tokens, skip_special_tokens=True)
self.vocab = list(sorted(self.vocab))
self.vocab = self.get_vocab(detokenize=True)
# tokens and lists
self.special_tokens = list(self.model.all_special_tokens)
self.added_tokens = self.model.batch_decode(self.model.added_tokens_encoder.values(), skip_special_tokens=False)
self.special_tokens = [self.vocab[i] for i in sorted(self.model.all_special_ids)]
self.added_tokens = [self.vocab[i] for i in sorted(self.model.added_tokens_encoder.values())]
self.bos_token = self.model.bos_token
self.eos_token = self.model.eos_token

def get_vocab(self, detokenize=False) -> list[str]:
max_token_id = max(self.model.get_vocab().values())
if detokenize:
ids = list(range(max_token_id + 1))
vocab = self.model.batch_decode(ids, skip_special_tokens=False)
Copy link
Collaborator

@compilade compilade Jul 9, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think this should be used in the convert script(s) instead of directly getting the strings from tokenizer.vocab?

EDIT: this might be a bad idea, since the tokenizer merges won't directly match with the strings from the vocab if that's done

else:
vocab = [None] * (max_token_id + 1)
for text, id in self.model.get_vocab().items():
vocab[id] = text

Check failure on line 172 in tests/test-tokenizer-random.py

View workflow job for this annotation

GitHub Actions / pyright type-check 

No overloads for "__setitem__" match the provided arguments (reportCallIssue)

Check failure on line 172 in tests/test-tokenizer-random.py

View workflow job for this annotation

GitHub Actions / pyright type-check 

Argument of type "str" cannot be assigned to parameter "value" of type "None" in function "__setitem__"
  "str" is incompatible with "None" (reportArgumentType)
return vocab

Check failure on line 173 in tests/test-tokenizer-random.py

View workflow job for this annotation

GitHub Actions / pyright type-check 

Expression of type "List[str] | list[None]" is incompatible with return type "list[str]"
  Type "List[str] | list[None]" is incompatible with type "list[str]"
    "list[None]" is incompatible with "list[str]"
      Type parameter "_T@list" is invariant, but "None" is not the same as "str"
      Consider switching from "list" to "Sequence" which is covariant (reportReturnType)

def encode(self, text: str) -> list[int]:
return self.model.encode(text, add_special_tokens=True)

Expand All @@ -163,6 +188,9 @@
self.libllama = LibLlama()
self.model = LibLlamaModel(self.libllama, vocab_file, mparams=dict(vocab_only=True), cparams=dict(n_ctx=4096))

def get_vocab(self, detokenize=False) -> list[str]:
return self.model.get_vocab(detokenize)

def encode(self, text: str) -> list[int]:
return self.model.tokenize(text, add_special=True, parse_special=True)

Expand Down Expand Up @@ -278,7 +306,7 @@


def generator_added_lr_strip(tokenizer: TokenizerGroundtruth) -> Iterator[str]:
WHITESPACES = ["", " ", " ", "\n", "\r\n", "\n\n", "\t", "\t\t"]
WHITESPACES = ["", " ", " ", "\n", "\r\n", "\n\n", "\t", "\t\t", " "]
all_tokens = list(sorted(set(tokenizer.special_tokens + tokenizer.added_tokens)))
for token in all_tokens:
for lstrip in WHITESPACES:
Expand Down Expand Up @@ -409,14 +437,6 @@

def compare_tokenizers(tokenizer1: TokenizerGroundtruth, tokenizer2: TokenizerLlamaCpp, generator: Iterator[str]):

def find_first_mismatch(ids1: list[int] | str, ids2: list[int] | str):
for i, (a, b) in enumerate(zip(ids1, ids2)):
if a != b:
return i
if len(ids1) == len(ids2):
return -1
return min(len(ids1), len(ids2))

def check_detokenizer(text: str, text1: str, text2: str) -> bool:
if text1 == text2: # equal to TokenizerGroundtruth?
return True
Expand All @@ -436,6 +456,7 @@
t_start = time.perf_counter()
encode_errors = 0
decode_errors = 0
total_tests = 0
MAX_ERRORS = 10

logger.info("%s: %s" % (generator.__qualname__, "ini"))
Expand All @@ -455,21 +476,44 @@
t_encode2 += t2 - t1
t_decode1 += t3 - t2
t_decode2 += t4 - t3
if encode_errors < MAX_ERRORS and ids1 != ids2:
i = find_first_mismatch(ids1, ids2)
ids1 = list(ids1)[max(0, i - 2) : i + 5 + 1]
ids2 = list(ids2)[max(0, i - 2) : i + 5 + 1]
# compare
encode_ok = ids1 == ids2
decode_ok = check_detokenizer(text, text1, text2)
encode_errors += not encode_ok
decode_errors += not decode_ok
total_tests += 1
if (encode_errors < MAX_ERRORS and not encode_ok) or (decode_errors < MAX_ERRORS and not decode_ok):
def _compare(text: str):
ids1 = tokenizer1.encode(text)
ids2 = tokenizer2.encode(text)
text1 = tokenizer1.decode(ids1)
text2 = tokenizer2.decode(ids1)
encode_ok = ids1 == ids2
decode_ok = check_detokenizer(text, text1, text2)
ok = encode_ok and decode_ok
return ok, ids1, ids2, text1, text2
a, b = 0, len(text)
for step in [64, 32, 16, 8, 4, 2, 1]:
while a < b:
t = max(a, b - step)
if _compare(text[a : t])[0]:
break
b = t
for step in [64, 32, 16, 8, 4, 2, 1]:
while a < b:
t = min(a + step, b)
if _compare(text[t : b])[0]:
break
a = t
ok, ids1, ids2, text1, text2 = _compare(text[a : b])
assert a <= b and not ok
logger.error(" Text:" + repr(text[a : b]))
logger.error(" " + " ".join(repr(x) + ":" + hex(ord(x)) for x in text[a : b]))
logger.error(" Expected: " + str(ids1))
logger.error(" Result: " + str(ids2))
encode_errors += 1
logger.error(" Expected: " + " ".join(repr(x) + ":" + hex(ord(x)) for x in text1))
logger.error(" Result: " + " ".join(repr(x) + ":" + hex(ord(x)) for x in text2))
logger.error(f" {encode_errors=}")
if decode_errors < MAX_ERRORS and not check_detokenizer(text, text1, text2):
i = find_first_mismatch(text1, text2)
text1 = list(text1[max(0, i - 2) : i + 5 + 1])
text2 = list(text2[max(0, i - 2) : i + 5 + 1])
logger.error(" Expected: " + " ".join(hex(ord(x)) for x in text1))
logger.error(" Result: " + " ".join(hex(ord(x)) for x in text2))
decode_errors += 1
logger.error(f" {decode_errors=}")
if encode_errors >= MAX_ERRORS and decode_errors >= MAX_ERRORS:
logger.error(f" EXIT: {encode_errors=} {decode_errors=}")
Expand All @@ -480,6 +524,44 @@
logger.info(f"{generator.__qualname__}: end, {t_encode1=:.3f} {t_encode2=:.3f} {t_decode1=:.3f} {t_decode2=:.3f} {t_total=:.3f}")


def compare_vocabs(tokenizer1: TokenizerGroundtruth, tokenizer2: TokenizerLlamaCpp):

MAX_PRINT_ERRORS = 10

logger.info("compare_vocabs: ini")

t_start = time.perf_counter()

for detokenize in (False, True):
vocab1 = tokenizer1.get_vocab(detokenize)
vocab2 = tokenizer2.get_vocab(detokenize)
if vocab1 != vocab2:
num_errors = 0
for i in range(max(len(vocab1), len(vocab2))):
text1 = vocab1[i] if i < len(vocab1) else None
text2 = vocab2[i] if i < len(vocab2) else None
if True: #WIP: SentencePiece adds more unused tokens than AutoTokenizer ?
if text1 is None:
if not text2 or text2.startswith('[PAD'): # is unused ? #TODO: use toktypes
text2 = None
else:
#TODO: is "UNUSED_TOKEN_" valid for all models ?
text1 = text1.replace("[UNUSED_TOKEN_", "[PAD")
#if text1 is None or text1.startswith("[UNUSED_TOKEN_"): # is unused ?
# text1 = ""
#if text2 is None or text2.startswith('[PAD'): # is unused ?
# text2 = ""
if text1 != text2:
num_errors += 1
if num_errors < MAX_PRINT_ERRORS:
logger.error(f" {detokenize=} id={i} expected={repr(text1)} result={repr(text2)}")
if num_errors:
logger.error(f" {num_errors=}")

t_total = time.perf_counter() - t_start
logger.info(f"compare_vocabs: end, {t_total=:.3f}")


def main(argv: list[str] | None = None):
parser = argparse.ArgumentParser()
parser.add_argument("vocab_file", type=str, help="path to vocab 'gguf' file")
Expand All @@ -493,13 +575,16 @@
tokenizer1 = TokenizerGroundtruth(args.dir_tokenizer)
tokenizer2 = TokenizerLlamaCpp(args.vocab_file)

# compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text())
# compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text_edge_cases())
compare_tokenizers(tokenizer1, tokenizer2, generator_ascii_lr_strip())
compare_tokenizers(tokenizer1, tokenizer2, generator_apostrophe())
compare_tokenizers(tokenizer1, tokenizer2, generator_unicodes())
compare_tokenizers(tokenizer1, tokenizer2, generator_vocab_words(tokenizer1))
compare_tokenizers(tokenizer1, tokenizer2, generator_added_lr_strip(tokenizer1))
compare_vocabs(tokenizer1, tokenizer2)

compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text())
compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text_edge_cases())
# compare_tokenizers(tokenizer1, tokenizer2, generator_representative(tokenizer1))
# compare_tokenizers(tokenizer1, tokenizer2, generator_ascii_lr_strip())
# compare_tokenizers(tokenizer1, tokenizer2, generator_apostrophe())
# compare_tokenizers(tokenizer1, tokenizer2, generator_unicodes())
# compare_tokenizers(tokenizer1, tokenizer2, generator_vocab_words(tokenizer1))
# compare_tokenizers(tokenizer1, tokenizer2, generator_added_lr_strip(tokenizer1))
# compare_tokenizers(tokenizer1, tokenizer2, generator_random_added_tokens(tokenizer1, 10_000))
# compare_tokenizers(tokenizer1, tokenizer2, generator_random_chars(10_000))
# compare_tokenizers(tokenizer1, tokenizer2, generator_random_unicodes(10_000))
Expand Down
Loading