Address comments

Author: larryliu0820

CMakeLists.txt

@@ -13,7 +13,7 @@
 #
 cmake_minimum_required(VERSION 3.18)
 set(CMAKE_CXX_STANDARD 17)
-
+set(CMAKE_POLICY_VERSION_MINIMUM 3.5)
 project(Tokenizers)
 
 option(TOKENIZERS_BUILD_TEST "Build tests" OFF)

test/test_python_bindings.py

@@ -9,6 +9,7 @@
 Test script for PyTorch Tokenizers Python bindings
 """
 
+import os
 import sys
 import unittest
 
@@ -91,3 +92,38 @@ def test_version(self):
         """Test that version is available"""
         self.assertTrue(hasattr(pytorch_tokenizers, '__version__'))
         self.assertEqual(pytorch_tokenizers.__version__, "0.1.0")
+
+    def test_hf_tokenizer_encode_decode(self):
+        """Test HFTokenizer with test_hf_tokenizer.json to encode/decode 'Hello world!'"""
+        # Get the path to the test tokenizer file
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        tokenizer_path = os.path.join(test_dir, "resources", "test_hf_tokenizer.json")
+
+        # Create and load the tokenizer
+        hf_tokenizer = pytorch_tokenizers.HFTokenizer()
+        self.assertFalse(hf_tokenizer.is_loaded())
+
+        # Load the tokenizer from JSON file
+        hf_tokenizer.load(tokenizer_path)
+        self.assertTrue(hf_tokenizer.is_loaded())
+
+        # Test encoding "Hello world!"
+        text = "Hello world!"
+        encoded_tokens = hf_tokenizer.encode(text, 1, 0)  # bos=1, eos=0
+        self.assertIsInstance(encoded_tokens, list)
+        self.assertGreater(len(encoded_tokens), 0)
+
+        # Test decoding the encoded tokens
+        for token_id in encoded_tokens:
+            decoded_text = hf_tokenizer.decode(token_id)
+            self.assertIsInstance(decoded_text, str)
+
+        # Test that we can get vocab size
+        vocab_size = hf_tokenizer.vocab_size()
+        self.assertGreater(vocab_size, 0)
+
+        # Test BOS and EOS tokens
+        bos_token = hf_tokenizer.bos_tok()
+        eos_token = hf_tokenizer.eos_tok()
+        self.assertIsInstance(bos_token, int)
+        self.assertIsInstance(eos_token, int)