Fix Qwen2Tokenizer ignoring pretokenize_regex (Qwen3.6 / Qwen3.8 wrong token IDs)

src/transformers/models/qwen2/tokenization_qwen2.py

@@ -50,6 +50,10 @@ def __init__(
         **kwargs,
     ):
         self.add_prefix_space = add_prefix_space if add_prefix_space is not None else False
+        # Honour `pretokenize_regex` from tokenizer_config.json when present (Qwen3.6 / Qwen3.8
+        # publish a refined pattern with \p{M}). Fall back to the module constant otherwise so
+        # older Qwen2 / Qwen3 checkpoints that omit the key keep their published tokenization.
+        pretokenize_regex = kwargs.get("pretokenize_regex") or PRETOKENIZE_REGEX
         self._vocab = (
             vocab
             if vocab is not None
@@ -75,7 +79,7 @@ def __init__(
         self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
             [
                 pre_tokenizers.Split(
-                    Regex(PRETOKENIZE_REGEX),
+                    Regex(pretokenize_regex),
                     behavior="isolated",
                     invert=False,
                 ),

tests/models/qwen2/test_tokenization_qwen2.py

@@ -31,3 +31,32 @@ class Qwen2TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
     integration_expected_token_ids = [1986, 374, 264, 1273, 26525, 232, 198, 40, 572, 9223, 304, 220, 24, 17, 15, 15, 15, 11, 323, 419, 374, 31932, 963, 624, 105301, 88051, 116109, 20412, 198, 13048, 220, 21927, 198, 13048, 256, 21927, 271, 48426, 21927, 198, 44047, 397, 6023, 44047, 29, 18532, 198, 785, 2701, 914, 1265, 387, 10277, 20498, 25, 21927, 624, 3983, 220, 2603, 323, 220, 54684, 28319, 256, 220, 2603, 256, 220, 37033, 198, 18665, 1246, 525, 498, 3730]  # fmt: skip
     expected_tokens_from_ids = ['This', 'Ġis', 'Ġa', 'Ġtest', 'ĠðŁĺ', 'Ĭ', 'Ċ', 'I', 'Ġwas', 'Ġborn', 'Ġin', 'Ġ', '9', '2', '0', '0', '0', ',', 'Ġand', 'Ġthis', 'Ġis', 'Ġfals', 'é', '.Ċ', 'çĶŁæ´»çļĦ', '羣', 'è°Ľ', 'æĺ¯', 'Ċ', 'Hi', 'Ġ', 'ĠHello', 'Ċ', 'Hi', 'ĠĠ', 'ĠHello', 'ĊĊ', 'ĠĊĠĠĊ', 'ĠHello', 'Ċ', '<s', '>Ċ', 'hi', '<s', '>', 'there', 'Ċ', 'The', 'Ġfollowing', 'Ġstring', 'Ġshould', 'Ġbe', 'Ġproperly', 'Ġencoded', ':', 'ĠHello', '.Ċ', 'But', 'Ġ', 'ird', 'Ġand', 'Ġ', 'à¸Ľ', 'ี', 'ĠĠ', 'Ġ', 'ird', 'ĠĠ', 'Ġ', 'à¸Ķ', 'Ċ', 'Hey', 'Ġhow', 'Ġare', 'Ġyou', 'Ġdoing']  # fmt: skip
     integration_expected_decoded_text = "This is a test 😊\nI was born in 92000, and this is falsé.\n生活的真谛是\nHi  Hello\nHi   Hello\n\n \n  \n Hello\n<s>\nhi<s>there\nThe following string should be properly encoded: Hello.\nBut ird and ปี   ird   ด\nHey how are you doing"
+
+
+    def test_honors_pretokenize_regex_from_kwargs(self):
+        """Qwen3.6 / Qwen3.8 ship `pretokenize_regex` with `\\p{M}` in tokenizer_config.json.
+
+        Qwen2Tokenizer used to ignore that key and always apply the module-level
+        PRETOKENIZE_REGEX, so AutoTokenizer produced different IDs than the published
+        tokenizer.json for scripts that use combining marks (see #49066).
+        """
+        import json
+
+        from transformers.models.qwen2.tokenization_qwen2 import PRETOKENIZE_REGEX
+
+        refined = (
+            r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?[\p{L}\p{M}]+|\p{N}| ?[^\s\p{L}\p{M}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
+        )
+        self.assertNotEqual(refined, PRETOKENIZE_REGEX)
+        self.assertIn(r"\p{M}", refined)
+        self.assertNotIn(r"\p{M}", PRETOKENIZE_REGEX)
+
+        def split_pattern(tokenizer):
+            backend = json.loads(tokenizer.backend_tokenizer.to_str())
+            return backend["pre_tokenizer"]["pretokenizers"][0]["pattern"]["Regex"]
+
+        with_kwarg = Qwen2Tokenizer(pretokenize_regex=refined)
+        self.assertEqual(split_pattern(with_kwarg), refined)
+
+        without_kwarg = Qwen2Tokenizer()
+        self.assertEqual(split_pattern(without_kwarg), PRETOKENIZE_REGEX)