Bug fix in tokenizer config (pytorch#305)

Summary:

It handles the case hhen params.json is explicitly given and the json doesn't contain mention about tokenizer config, but the command line does.

Test Plan:

python torchchat.py generate --device cpu --checkpoint-path /Users/mnachin/models/Meta-Llama-3-8B/original/consolidated.00.pth --params-path=/Users/mnachin/models/Meta-Llama-3-8B/original/params.json --temperature 0 --tiktoken

Author: mergennachin

build/builder.py

@@ -367,7 +367,10 @@ def tokenizer_setting_to_name(tiktoken: bool = False) -> str:
 def validate_args(model: Transformer, tokenizer_args: TokenizerArgs):
     use_tiktoken = model.config.use_tiktoken
     is_tiktoken = tokenizer_args.is_tiktoken
-    if use_tiktoken != is_tiktoken:
+
+    if use_tiktoken is None:
+        model.config.use_tiktoken = is_tiktoken
+    elif use_tiktoken != is_tiktoken:
         raise RuntimeError(f"model-specified tokenizer ({tokenizer_setting_to_name(use_tiktoken)} does not match provided tokenizer ({tokenizer_setting_to_name(is_tiktoken)}")
 
 def resolve_model_name(model: str) -> str:

build/model.py

@@ -37,7 +37,7 @@ class ModelArgs:
     norm_eps: float = 1e-5
     multiple_of: int = 256
     ffn_dim_multiplier: Optional[int] = None
-    use_tiktoken: bool = False
+    use_tiktoken: Optional[bool] = None
 
     def __post_init__(self):
         if self.n_local_heads == -1: