fix qnn export (#9808)

Summary:
Pull Request resolved: #9808

One missing item for the new tokenizer lib

Reviewed By: kirklandsign

Differential Revision: D72263224

Author: cccclai

examples/qualcomm/oss_scripts/llama/llama.py

@@ -57,7 +57,6 @@
 from executorch.examples.models.llama.source_transformation.quantize import (
     get_quant_embedding_transform,
 )
-from executorch.examples.models.llama.tokenizer.tiktoken import Tokenizer as Tiktoken
 from executorch.examples.qualcomm.oss_scripts.llama.model.static_llama import (
     LlamaModel,
     ModelArgs,
@@ -77,6 +76,7 @@
 from executorch.extension.llm.export.builder import DType
 from pytorch_tokenizers import get_tokenizer
 from pytorch_tokenizers.llama2c import Llama2cTokenizer as SentencePieceTokenizer
+from pytorch_tokenizers import TiktokenTokenizer
 
 from torch.ao.quantization.observer import MinMaxObserver
 from torch.ao.quantization.quantize_pt2e import convert_pt2e, prepare_pt2e
@@ -141,7 +141,7 @@ def _kv_calibrate(
     # Llama2 tokenizer has no special tokens
     if isinstance(tokenizer, SentencePieceTokenizer):
         token_list = tokenizer.encode(user_prompts, bos=True, eos=False)
-    elif isinstance(tokenizer, Tiktoken):
+    elif isinstance(tokenizer, TiktokenTokenizer):
         token_list = tokenizer.encode(
             user_prompts, bos=True, eos=False, allowed_special="all"
         )
@@ -213,7 +213,7 @@ def _prefill_calibrate(
     # Llama2 tokenizer has no special tokens
     if isinstance(tokenizer, SentencePieceTokenizer):
         token_list = tokenizer.encode(user_prompts, bos=True, eos=False)
-    elif isinstance(tokenizer, Tiktoken):
+    elif isinstance(tokenizer, TiktokenTokenizer):
         token_list = tokenizer.encode(
             user_prompts, bos=True, eos=False, allowed_special="all"
         )
@@ -1111,7 +1111,7 @@ def export_llama(args) -> None:
         runtime_tokenizer_path = args.tokenizer_bin
     elif args.llama_model == "llama3_2":
         assert isinstance(
-            tokenizer, Tiktoken
+            tokenizer, TiktokenTokenizer
         ), f"Wrong tokenizer provided for llama3_2."
         runtime_tokenizer_path = args.tokenizer_model
     else: