diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 056611182fd9..0af6e942cae1 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -170,6 +170,7 @@ ("led", "LEDTokenizer" if is_tokenizers_available() else None), ("lighton_ocr", "Qwen2TokenizerFast" if is_tokenizers_available() else None), ("lilt", "RobertaTokenizer" if is_tokenizers_available() else None), + ("llama", "LlamaTokenizer" if is_tokenizers_available() else None), ("longformer", "RobertaTokenizer" if is_tokenizers_available() else None), ("longt5", "T5Tokenizer" if is_tokenizers_available() else None), ("luke", "LukeTokenizer"),