File tree Expand file tree Collapse file tree 1 file changed +1
-1
lines changed
examples/qualcomm/oss_scripts/llama Expand file tree Collapse file tree 1 file changed +1
-1
lines changed Original file line number Diff line number Diff line change 57
57
from executorch .examples .models .llama .source_transformation .quantize import (
58
58
get_quant_embedding_transform ,
59
59
)
60
- from executorch .examples .models .llama .tokenizer .tiktoken import Tokenizer as Tiktoken
61
60
from executorch .examples .qualcomm .oss_scripts .llama .model .static_llama import (
62
61
LlamaModel ,
63
62
ModelArgs ,
77
76
from executorch .extension .llm .export .builder import DType
78
77
from pytorch_tokenizers import get_tokenizer
79
78
from pytorch_tokenizers .llama2c import Llama2cTokenizer as SentencePieceTokenizer
79
+ from pytorch_tokenizers import TiktokenTokenizer as Tiktoken
80
80
81
81
from torch .ao .quantization .observer import MinMaxObserver
82
82
from torch .ao .quantization .quantize_pt2e import convert_pt2e , prepare_pt2e
You can’t perform that action at this time.
0 commit comments