From 222a5d26768d47bb242931bde47ea27934f26986 Mon Sep 17 00:00:00 2001 From: "Tommy Yang (m0gician)" Date: Mon, 20 May 2024 01:25:14 +0800 Subject: [PATCH 1/8] Fix null byte `\x00` issue by switching to `numba.types.unicode_type` --- outlines/fsm/regex.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/outlines/fsm/regex.py b/outlines/fsm/regex.py index b68e318979..a396b8e9dd 100644 --- a/outlines/fsm/regex.py +++ b/outlines/fsm/regex.py @@ -87,14 +87,11 @@ def fsm_info(self): ((k, z) for k, v in self.trans_key_to_states.items() for z in v), dtype=np.dtype("int64, int64"), ) - alphabet_symbol_mapping_items = np.fromiter( - ( - it - for it in self.alphabet._symbol_mapping.items() - if it[0] != anything_else - ), - dtype=np.dtype("U2, int64"), - ) + alphabet_symbol_mapping_items = [ + (k, v) + for k, v in self.alphabet._symbol_mapping.items() + if k != anything_else + ] nb_finals = np.fromiter(self.finals, dtype=np.dtype("int64")) self.__dict__["_fsm_info"] = create_fsm_info( self.initial, @@ -110,7 +107,7 @@ def fsm_info(self): nb_int_list_type = numba.types.ListType(numba.int64) nb_int_pair_type = numba.types.UniTuple(numba.int64, 2) -nb_unichar_2_type = numba.types.UnicodeCharSeq(2) +nb_unicode_type = numba.types.unicode_type @numba.njit(cache=True) @@ -136,7 +133,7 @@ def create_fsm_info( # use 2-char strings so that we can represent incomplete utf-8 sequences # as 2-hex-digit pairs - alphabet_symbol_map = numba.typed.Dict.empty(nb_unichar_2_type, numba.int64) + alphabet_symbol_map = numba.typed.Dict.empty(nb_unicode_type, numba.int64) for symbol_and_trans_key in alphabet_symbol_mapping_items: alphabet_symbol_map[symbol_and_trans_key[0]] = symbol_and_trans_key[1] @@ -804,7 +801,7 @@ def reduced_vocabulary( raise RuntimeError( f"Cannot convert token `{token}` ({token_idx}) to bytes: {token_str}" ) - token_str = tuple(byte_symbol(b) for b in token_bytes) + token_str = "".join(byte_symbol(b) for b in token_bytes) vocabulary.setdefault(token_str, []).append(token_idx) else: @@ -813,15 +810,14 @@ def reduced_vocabulary( vocabulary_nb = numba.typed.List.empty_list( numba.types.Tuple( ( - nb_unichar_2_type[:], + nb_unicode_type, numba.int64[:], ) ) ) - for token_tuple, token_ids in vocabulary.items(): - token_tuple_np = np.fromiter(token_tuple, dtype=np.dtype("U2")) + for token_str, token_ids in vocabulary.items(): token_ids_np = np.fromiter(token_ids, dtype=np.dtype("int64")) - vocabulary_nb.append((token_tuple_np, token_ids_np)) + vocabulary_nb.append((token_str, token_ids_np)) return vocabulary_nb, empty_token_ids From 5ba4c2bf745cf4e2bb49ce0cd47b25fd39111efb Mon Sep 17 00:00:00 2001 From: Andrew Lapp Date: Sun, 26 May 2024 03:03:24 -0500 Subject: [PATCH 2/8] ensure byte fsm unicode_type compatibility by prefixing hex-bytes with \x00 --- outlines/fsm/regex.py | 54 +++++++++++++++++++++++++++++++---------- tests/fsm/test_regex.py | 53 ++++++++++++++++++++++++++++------------ 2 files changed, 79 insertions(+), 28 deletions(-) diff --git a/outlines/fsm/regex.py b/outlines/fsm/regex.py index a396b8e9dd..5ee526362d 100644 --- a/outlines/fsm/regex.py +++ b/outlines/fsm/regex.py @@ -196,7 +196,7 @@ def transition_trie_setdefault( def byte_symbol(byte: int) -> str: - return f"{byte:02X}" if byte >= 0x80 else chr(byte) + return f"\x00{byte:02X}" if byte >= 0x80 else chr(byte) def make_byte_level_fsm(fsm: FSM, keep_utf8=False) -> FSM: @@ -416,7 +416,7 @@ def _walk_fsm( alphabet_anything_value: int, fsm_initial: int, fsm_finals: Set[int], - input_string: Sequence[str], + input_string: str, start_state: int, full_match: bool = True, ) -> List[int]: @@ -424,7 +424,21 @@ def _walk_fsm( accepted_states: List[int] = numba.typed.List.empty_list(numba.int64) last_final_idx: int = numba.uint64(0) - for i, symbol in enumerate(input_string): + # Iterate over symbols (characters and null-prefixed two-hex-character bytes) + # By default, each symbol is a unicode character + # Except, if the character, input_string[i] == '\x00', then the next two + # in input_string characters are a hex representation of the byte + i = 0 + while i < len(input_string): + # if null-byte prefixed its a hex representation + # unless its the last character, then its a trailing null byte symbol + if input_string[i] == "\x00" and i != len(input_string) - 1: + symbol = input_string[i : i + 3] + i += 3 + else: + symbol = input_string[i] + i += 1 + trans_key = alphabet_symbol_mapping.get(symbol, alphabet_anything_value) new_state = fsm_transitions.get((state, trans_key)) @@ -438,11 +452,11 @@ def _walk_fsm( state = new_state if state in fsm_finals: - last_final_idx = numba.uint64(i + 1) + last_final_idx = numba.uint64(i) accepted_states.append(_nonoptional(state)) - if full_match and last_final_idx - 1 != i: + if full_match and last_final_idx != i: return numba.typed.List.empty_list(numba.int64) return accepted_states @@ -450,7 +464,7 @@ def _walk_fsm( def walk_fsm( fsm: BetterFSM, - input_string: Sequence[str], + input_string: str, start_state: int, full_match: bool = True, ) -> List[int]: @@ -464,7 +478,17 @@ def walk_fsm( alphabet_anything_value = fsm.alphabet.anything_value fsm_transitions = fsm.flat_transition_map - for i, symbol in enumerate(input_string): + # See _walk_fsm() explanation of symbol iteration + i = 0 + while i < len(input_string): + # if null-byte prefixed its a hex representation + # unless the input string itself is a null byte, then symbol is a lone null-byte + if input_string[i] == "\x00" and input_string != "\x00": + symbol = input_string[i : i + 3] + i += 3 + else: + symbol = input_string[i] + i += 1 trans_key = alphabet_symbol_mapping.get(symbol, alphabet_anything_value) new_state = fsm_transitions.get((state, trans_key)) @@ -478,11 +502,11 @@ def walk_fsm( state = new_state if state in fsm_finals: - last_final_idx = i + 1 + last_final_idx = i accepted_states.append(state) - if full_match and last_final_idx - 1 != i: + if full_match and last_final_idx != i: return [] return accepted_states @@ -652,7 +676,7 @@ def state_scan_tokens( alphabet_anything_value: int, fsm_initial: int, fsm_finals: Set[int], - vocabulary: List[Tuple[Sequence[str], Sequence[int]]], + vocabulary: List[Tuple[str, Sequence[int]]], start_state: int, ) -> Set[Tuple[int, int]]: res = set() @@ -669,7 +693,11 @@ def state_scan_tokens( False, ) - if state_seq is not None and len(state_seq) < len(token): + if token == "\x00": + token_length = 1 + else: + token_length = len(token) - 2 * token.count("\x00") + if state_seq is not None and len(state_seq) < token_length: continue for token_id in token_ids: @@ -680,7 +708,7 @@ def state_scan_tokens( def create_fsm_index_end_to_end( fsm_info: FSMInfo, - vocabulary: List[Tuple[Sequence[str], Sequence[int]]], + vocabulary: List[Tuple[str, Sequence[int]]], ) -> Dict[int, Set[Tuple[int, int]]]: """Create an FSM state-to-vocabulary map/index through end-to-end token parsing.""" @@ -768,7 +796,7 @@ def gpt2_unicode_to_bytes(): @lru_cache def reduced_vocabulary( tokenizer: "Tokenizer", -) -> Tuple[List[Tuple[Sequence[str], Sequence[int]]], Set[int]]: +) -> Tuple[List[Tuple[str, Sequence[int]]], Set[int]]: """Create a map from decoded vocabulary tokens to lists of equivalent token ids.""" empty_token_ids = set() vocabulary: Dict[Union[str, Tuple[str, ...]], List[int]] = {} diff --git a/tests/fsm/test_regex.py b/tests/fsm/test_regex.py index 2fc8a5384f..ce8ed6f8f6 100644 --- a/tests/fsm/test_regex.py +++ b/tests/fsm/test_regex.py @@ -25,7 +25,7 @@ def identity(s): def to_bytes(s): - return [chr(b) if b < 0x80 else f"{b:02X}" for b in s.encode("utf-8")] + return [chr(b) if b < 0x80 else f"\x00{b:02X}" for b in s.encode("utf-8")] def walk_fsm_numba( @@ -115,19 +115,27 @@ def test_walk_fsm_multi_bytes(function, transform): str_regex_fsm, _ = make_deterministic_fsm(regex_pattern.to_fsm().reduce()) regex_fsm = make_byte_level_better_fsm(str_regex_fsm, keep_utf8=True) - res = tuple(function(regex_fsm, transform("πŸ˜‚"), regex_fsm.initial, full_match=True)) + res = tuple( + function(regex_fsm, "".join(transform("πŸ˜‚")), regex_fsm.initial, full_match=True) + ) assert res[-1:] == (1,) res = tuple( - function(regex_fsm, transform("πŸ˜‚πŸ˜‚"), regex_fsm.initial, full_match=False) + function( + regex_fsm, "".join(transform("πŸ˜‚πŸ˜‚")), regex_fsm.initial, full_match=False + ) ) assert res[-1:] == (1,) - res = tuple(function(regex_fsm, transform("!"), regex_fsm.initial, full_match=True)) + res = tuple( + function(regex_fsm, "".join(transform("!")), regex_fsm.initial, full_match=True) + ) assert res == tuple() res = tuple( - function(regex_fsm, transform("πŸ˜‚πŸ˜‚"), regex_fsm.initial, full_match=True) + function( + regex_fsm, "".join(transform("πŸ˜‚πŸ˜‚")), regex_fsm.initial, full_match=True + ) ) assert res == tuple() @@ -304,15 +312,15 @@ def test_create_fsm_index_end_to_end(): vocabulary_nb = numba.typed.List.empty_list( numba.types.Tuple( ( - numba.types.UnicodeCharSeq(2)[:], + numba.types.unicode_type, numba.int64[:], ) ) ) for token_tuple, token_ids in vocabulary.items(): - token_tuple_np = np.fromiter(token_tuple, dtype=np.dtype("U2")) + token = "".join(token_tuple) token_ids_np = np.fromiter(token_ids, dtype=np.dtype("int64")) - vocabulary_nb.append((token_tuple_np, token_ids_np)) + vocabulary_nb.append((token, token_ids_np)) res = create_fsm_index_end_to_end(regex_fsm.fsm_info, vocabulary_nb) @@ -326,28 +334,34 @@ def test_create_fsm_index_end_to_end_multi_byte(): regex_fsm, _ = make_deterministic_fsm(regex_pattern.to_fsm().reduce()) byte_fsm = make_byte_level_better_fsm(regex_fsm, keep_utf8=True) + merge_symbols = lambda byte_hexs: "".join( + ["" + b if len(b) == 2 else b for b in byte_hexs] + ) + vocabulary = { "blah": numba.typed.List([0]), "😈a": numba.typed.List([1]), "πŸ˜‡": numba.typed.List([2]), "😍": numba.typed.List([3]), - ("F0", "9F", "98", "8D"): numba.typed.List([4]), # '😍' + merge_symbols(("F0", "9F", "98", "8D")): numba.typed.List([4]), # '😍' " 😍": numba.typed.List([5]), - (" ", "F0", "9F", "98", "8D"): numba.typed.List([6]), # ' 😍' - (" ", "F0", "9F", "98"): numba.typed.List([7]), # ' 😍' incomplete + merge_symbols((" ", "F0", "9F", "98", "8D")): numba.typed.List([6]), # ' 😍' + merge_symbols((" ", "F0", "9F", "98")): numba.typed.List( + [7] + ), # ' 😍' incomplete "": numba.typed.List([8]), } vocabulary_nb = numba.typed.List.empty_list( numba.types.Tuple( ( - numba.types.UnicodeCharSeq(2)[:], + numba.types.unicode_type, numba.int64[:], ) ) ) for token_tuple, token_ids in vocabulary.items(): - token_tuple_np = np.fromiter(token_tuple, dtype=np.dtype("U2")) + token_tuple_np = merge_symbols(token_tuple) token_ids_np = np.fromiter(token_ids, dtype=np.dtype("int64")) vocabulary_nb.append((token_tuple_np, token_ids_np)) @@ -356,7 +370,16 @@ def test_create_fsm_index_end_to_end_multi_byte(): assert res == {0: {(5, 3), (6, 3), (7, 7), (2, 2)}, 3: {(2, 3), (3, 3), (4, 3)}} -def test_create_fsm_index_tokenizer(): +@pytest.mark.parametrize( + "hf_tokenizer_uri", + [ + "gpt2", + "microsoft/phi-2", + "Qwen/Qwen1.5-0.5B-Chat", + "NousResearch/Hermes-2-Pro-Llama-3-8B", + ], +) +def test_create_fsm_index_tokenizer(hf_tokenizer_uri): # The combined regular expressions of a lexer state in a Python grammar regex_str = "(?:(?:[0-9](?:(?:_)?[0-9])*(?:e|E)(?:(?:\\+|\\-))?[0-9](?:(?:_)?[0-9])*|(?:[0-9](?:(?:_)?[0-9])*\\.(?:[0-9](?:(?:_)?[0-9])*)?|\\.[0-9](?:(?:_)?[0-9])*)(?:(?:e|E)(?:(?:\\+|\\-))?[0-9](?:(?:_)?[0-9])*)?)|[0-9](?:(?:_)?[0-9])*)(?:J|j)|(?:[0-9](?:(?:_)?[0-9])*(?:e|E)(?:(?:\\+|\\-))?[0-9](?:(?:_)?[0-9])*|(?:[0-9](?:(?:_)?[0-9])*\\.(?:[0-9](?:(?:_)?[0-9])*)?|\\.[0-9](?:(?:_)?[0-9])*)(?:(?:e|E)(?:(?:\\+|\\-))?[0-9](?:(?:_)?[0-9])*)?)|0(?:x|X)(?:(?:_)?(?:[0-9]|[a-f]|[A-F]))+|0(?:b|B)(?:(?:_)?[0-1])+|0(?:o|O)(?:(?:_)?[0-7])+|(?:(?i:([ubf]?r?|r[ubf])('([^\\\\']|.)*?'))|(?i:([ubf]?r?|r[ubf])(\"([^\\\"]|.)*?\")))|(?:(?:\r?\n[\t ]*|#[^\n]*))+|[1-9](?:(?:_)?[0-9])*|\\\\[\t \x0c]*\r?\n|continue|nonlocal|assert|global|import|lambda|return|async|await|break|class|False|match|raise|while|yield|case|from|None|pass|True|with|def|del|for|not|try|if|[^\\W\\d]\\w*|#[^\n]*|[\t \x0c]+|\\.\\.\\.|@|\\{|\\(|\\[|\\-|\\+|\\*|\\~" @@ -371,7 +394,7 @@ def test_create_fsm_index_tokenizer(): num_bytes_fsm_states = len(bytes_fsm.states) assert num_bytes_fsm_states == 235 - tokenizer = AutoTokenizer.from_pretrained("gpt2") + tokenizer = AutoTokenizer.from_pretrained(hf_tokenizer_uri) tokenizer = TransformerTokenizer(tokenizer) states_to_token_subsets, empty_token_ids = create_fsm_index_tokenizer( From 22d3ddd7cbfe5504f07129e117e314437533dc6f Mon Sep 17 00:00:00 2001 From: Andrew Lapp Date: Thu, 30 May 2024 17:38:11 -0500 Subject: [PATCH 3/8] index token -> transition key sequence for efficient _walk_fsm --- outlines/fsm/regex.py | 105 +++++++++++++----------- tests/fsm/test_regex.py | 174 +++++++++++++++++++++++++++++++++------- 2 files changed, 200 insertions(+), 79 deletions(-) diff --git a/outlines/fsm/regex.py b/outlines/fsm/regex.py index 5ee526362d..1d27a0872b 100644 --- a/outlines/fsm/regex.py +++ b/outlines/fsm/regex.py @@ -412,11 +412,9 @@ def make_deterministic_fsm(fsm: FSM) -> Tuple[BetterFSM, Dict[int, int]]: @numba.njit(nogil=True, cache=True) def _walk_fsm( fsm_transitions: Dict[Tuple[int, int], int], - alphabet_symbol_mapping: Dict[str, int], - alphabet_anything_value: int, fsm_initial: int, fsm_finals: Set[int], - input_string: str, + token_trans_key_seq: Sequence[int], start_state: int, full_match: bool = True, ) -> List[int]: @@ -424,23 +422,9 @@ def _walk_fsm( accepted_states: List[int] = numba.typed.List.empty_list(numba.int64) last_final_idx: int = numba.uint64(0) - # Iterate over symbols (characters and null-prefixed two-hex-character bytes) - # By default, each symbol is a unicode character - # Except, if the character, input_string[i] == '\x00', then the next two - # in input_string characters are a hex representation of the byte - i = 0 - while i < len(input_string): - # if null-byte prefixed its a hex representation - # unless its the last character, then its a trailing null byte symbol - if input_string[i] == "\x00" and i != len(input_string) - 1: - symbol = input_string[i : i + 3] - i += 3 - else: - symbol = input_string[i] - i += 1 - - trans_key = alphabet_symbol_mapping.get(symbol, alphabet_anything_value) - + # Iterate over token transition key sequence. The transition key + # sequence represents the FSM traversal rules of the tokens symbols. + for i, trans_key in enumerate(token_trans_key_seq): new_state = fsm_transitions.get((state, trans_key)) if new_state is None: @@ -452,11 +436,11 @@ def _walk_fsm( state = new_state if state in fsm_finals: - last_final_idx = numba.uint64(i) + last_final_idx = numba.uint64(i + 1) accepted_states.append(_nonoptional(state)) - if full_match and last_final_idx != i: + if full_match and last_final_idx - 1 != i: return numba.typed.List.empty_list(numba.int64) return accepted_states @@ -464,7 +448,7 @@ def _walk_fsm( def walk_fsm( fsm: BetterFSM, - input_string: str, + token_trans_key_seq: Sequence[int], start_state: int, full_match: bool = True, ) -> List[int]: @@ -474,23 +458,11 @@ def walk_fsm( accepted_states: List[int] = [] last_final_idx: int = 0 - alphabet_symbol_mapping = fsm.alphabet._symbol_mapping - alphabet_anything_value = fsm.alphabet.anything_value fsm_transitions = fsm.flat_transition_map - # See _walk_fsm() explanation of symbol iteration - i = 0 - while i < len(input_string): - # if null-byte prefixed its a hex representation - # unless the input string itself is a null byte, then symbol is a lone null-byte - if input_string[i] == "\x00" and input_string != "\x00": - symbol = input_string[i : i + 3] - i += 3 - else: - symbol = input_string[i] - i += 1 - trans_key = alphabet_symbol_mapping.get(symbol, alphabet_anything_value) - + # Iterate over token transition key sequence. The transition key + # sequence represents the FSM traversal rules of the tokens symbols. + for i, trans_key in enumerate(token_trans_key_seq): new_state = fsm_transitions.get((state, trans_key)) if new_state is None: @@ -502,11 +474,11 @@ def walk_fsm( state = new_state if state in fsm_finals: - last_final_idx = i + last_final_idx = i + 1 accepted_states.append(state) - if full_match and last_final_idx != i: + if full_match and last_final_idx - 1 != i: return [] return accepted_states @@ -677,27 +649,24 @@ def state_scan_tokens( fsm_initial: int, fsm_finals: Set[int], vocabulary: List[Tuple[str, Sequence[int]]], + token_trans_key_seqs: List[Sequence[int]], start_state: int, ) -> Set[Tuple[int, int]]: res = set() - for token, token_ids in vocabulary: + for (token, token_ids), token_trans_key_seq in zip( + vocabulary, token_trans_key_seqs + ): state_seq = _walk_fsm( fsm_transitions, - alphabet_symbol_mapping, - alphabet_anything_value, fsm_initial, fsm_finals, - token, + token_trans_key_seq, start_state, False, ) - if token == "\x00": - token_length = 1 - else: - token_length = len(token) - 2 * token.count("\x00") - if state_seq is not None and len(state_seq) < token_length: + if state_seq is not None and len(state_seq) < len(token_trans_key_seq): continue for token_id in token_ids: @@ -706,6 +675,37 @@ def state_scan_tokens( return res +@numba.njit(cache=True, nogil=True) +def get_tokens_trans_keys( + alphabet_symbol_mapping: Dict[str, int], + alphabet_anything_value: int, + vocabulary: List[Tuple[str, Sequence[int]]], +) -> List[Sequence[int]]: + tokens_trans_keys = numba.typed.List.empty_list(numba.int64[:]) + for token_str, _ in vocabulary: + trans_key_seq = [] + i = 0 + while i < len(token_str): + if token_str[i] == "\x00" and i != len(token_str) - 1: + symbol = token_str[i : i + 3] + i += 3 + else: + symbol = token_str[i] + i += 1 + + trans_key_seq.append( + alphabet_symbol_mapping.get(symbol, alphabet_anything_value) + ) + + trans_key_seq_array = np.empty(len(trans_key_seq), dtype=np.int64) + for j in range(len(trans_key_seq)): + trans_key_seq_array[j] = trans_key_seq[j] + + tokens_trans_keys.append(trans_key_seq_array) + + return tokens_trans_keys + + def create_fsm_index_end_to_end( fsm_info: FSMInfo, vocabulary: List[Tuple[str, Sequence[int]]], @@ -724,6 +724,12 @@ def create_fsm_index_end_to_end( desc="Compiling FSM index for all state transitions", ) + tokens_trans_key_seqs = get_tokens_trans_keys( + fsm_info.alphabet_symbol_mapping, + fsm_info.alphabet_anything_value, + vocabulary, + ) + while next_states: start_state = next_states.pop() @@ -734,6 +740,7 @@ def create_fsm_index_end_to_end( fsm_info.initial, fsm_info.finals, vocabulary, + tokens_trans_key_seqs, start_state, ) diff --git a/tests/fsm/test_regex.py b/tests/fsm/test_regex.py index ce8ed6f8f6..f1bf0f06a9 100644 --- a/tests/fsm/test_regex.py +++ b/tests/fsm/test_regex.py @@ -1,5 +1,3 @@ -from typing import Sequence - import interegular import numba import numpy as np @@ -12,9 +10,11 @@ create_fsm_index_tokenizer, fsm_union, get_sub_fsms_from_seq, + get_tokens_trans_keys, make_byte_level_better_fsm, make_byte_level_fsm, make_deterministic_fsm, + reduced_vocabulary, walk_fsm, ) from outlines.models.transformers import TransformerTokenizer @@ -28,19 +28,47 @@ def to_bytes(s): return [chr(b) if b < 0x80 else f"\x00{b:02X}" for b in s.encode("utf-8")] -def walk_fsm_numba( +def merge_symbols(byte_hexs): + return "".join(["\x00" + b if len(b) == 2 else b for b in byte_hexs]) + + +def token_str_to_trans_key(fsm, input_string): + vocabulary_nb = numba.typed.List.empty_list( + numba.types.Tuple((numba.types.unicode_type, numba.int64[:])) + ) + vocabulary_nb.append((input_string, np.fromiter([1], dtype=np.dtype("int64")))) + return get_tokens_trans_keys( + fsm.fsm_info.alphabet_symbol_mapping, + fsm.fsm_info.alphabet_anything_value, + vocabulary_nb, + )[0] + + +def walk_fsm_from_token_str( + fsm, + input_string: str, + start_state: int, + full_match: bool = True, +): + return walk_fsm( + fsm, + token_str_to_trans_key(fsm, input_string), + start_state, + full_match, + ) + + +def walk_fsm_from_token_str_numba( fsm, - input_string: Sequence[str], + input_string: str, start_state: int, full_match: bool = True, ): return _walk_fsm( fsm.fsm_info.transitions, - fsm.fsm_info.alphabet_symbol_mapping, - fsm.fsm_info.alphabet_anything_value, fsm.fsm_info.initial, fsm.fsm_info.finals, - input_string, + token_str_to_trans_key(fsm, input_string), start_state, full_match=full_match, ) @@ -49,8 +77,8 @@ def walk_fsm_numba( @pytest.mark.parametrize( "function", [ - walk_fsm, - walk_fsm_numba, + walk_fsm_from_token_str, + walk_fsm_from_token_str_numba, ], ) def test_walk_fsm(function): @@ -99,8 +127,8 @@ def test_walk_fsm(function): @pytest.mark.parametrize( "function", [ - walk_fsm, - walk_fsm_numba, + walk_fsm_from_token_str, + walk_fsm_from_token_str_numba, ], ) @pytest.mark.parametrize( @@ -116,25 +144,35 @@ def test_walk_fsm_multi_bytes(function, transform): regex_fsm = make_byte_level_better_fsm(str_regex_fsm, keep_utf8=True) res = tuple( - function(regex_fsm, "".join(transform("πŸ˜‚")), regex_fsm.initial, full_match=True) + function( + regex_fsm, merge_symbols(transform("πŸ˜‚")), regex_fsm.initial, full_match=True + ) ) assert res[-1:] == (1,) res = tuple( function( - regex_fsm, "".join(transform("πŸ˜‚πŸ˜‚")), regex_fsm.initial, full_match=False + regex_fsm, + merge_symbols(transform("πŸ˜‚πŸ˜‚")), + regex_fsm.initial, + full_match=False, ) ) assert res[-1:] == (1,) res = tuple( - function(regex_fsm, "".join(transform("!")), regex_fsm.initial, full_match=True) + function( + regex_fsm, merge_symbols(transform("!")), regex_fsm.initial, full_match=True + ) ) assert res == tuple() res = tuple( function( - regex_fsm, "".join(transform("πŸ˜‚πŸ˜‚")), regex_fsm.initial, full_match=True + regex_fsm, + merge_symbols(transform("πŸ˜‚πŸ˜‚")), + regex_fsm.initial, + full_match=True, ) ) assert res == tuple() @@ -202,14 +240,14 @@ def test_get_sub_fsms_from_seq(): assert fsm.accepts("+=") assert fsm.accepts("+") - state_seq = walk_fsm(fsm, "def", fsm.initial) + state_seq = walk_fsm_from_token_str(fsm, "def", fsm.initial) state_seq.insert(0, fsm.fsm_info.initial) res = list(get_sub_fsms_from_seq(state_seq, fsms_to_trans_finals)) assert res == [(0, False, True), (2, True, True)] # Make sure the old-to-new state map is correct - def_state_seq = walk_fsm(def_fsm, "def", fsm.initial) + def_state_seq = walk_fsm_from_token_str(def_fsm, "def", fsm.initial) def_state_seq.insert(0, fsm.fsm_info.initial) def_old_to_new_states = fsms_to_trans_finals[0][2] @@ -218,13 +256,13 @@ def test_get_sub_fsms_from_seq(): for old_state, new_state in zip(def_state_seq, state_seq) ) - state_seq = walk_fsm(fsm, "ef", fsm.initial) + state_seq = walk_fsm_from_token_str(fsm, "ef", fsm.initial) state_seq.insert(0, fsm.initial) res = list(get_sub_fsms_from_seq(state_seq, fsms_to_trans_finals)) assert res == [(2, True, True)] - name_state_seq = walk_fsm(name_fsm, "ef", fsm.initial) + name_state_seq = walk_fsm_from_token_str(name_fsm, "ef", fsm.initial) name_state_seq.insert(0, fsm.initial) name_old_to_new_states = fsms_to_trans_finals[2][2] @@ -233,13 +271,13 @@ def test_get_sub_fsms_from_seq(): for old_state, new_state in zip(name_state_seq, state_seq) ) - state_seq = walk_fsm(fsm, "match", fsm.initial) + state_seq = walk_fsm_from_token_str(fsm, "match", fsm.initial) state_seq.insert(0, fsm.initial) res = list(get_sub_fsms_from_seq(state_seq, fsms_to_trans_finals)) assert res == [(1, False, True), (2, True, True)] - match_state_seq = walk_fsm(match_fsm, "match", fsm.initial) + match_state_seq = walk_fsm_from_token_str(match_fsm, "match", fsm.initial) match_state_seq.insert(0, fsm.initial) match_old_to_new_states = fsms_to_trans_finals[1][2] @@ -248,25 +286,25 @@ def test_get_sub_fsms_from_seq(): for old_state, new_state in zip(match_state_seq, state_seq) ) - state_seq = walk_fsm(fsm, "defa", fsm.initial) + state_seq = walk_fsm_from_token_str(fsm, "defa", fsm.initial) state_seq.insert(0, fsm.initial) res = list(get_sub_fsms_from_seq(state_seq, fsms_to_trans_finals)) assert res == [(2, True, True)] - state_seq = walk_fsm(fsm, "de", fsm.initial) + state_seq = walk_fsm_from_token_str(fsm, "de", fsm.initial) state_seq.insert(0, fsm.initial) res = list(get_sub_fsms_from_seq(state_seq, fsms_to_trans_finals)) assert res == [(0, True, False), (2, True, True)] - state_seq = walk_fsm(fsm, "+", fsm.initial, False) + state_seq = walk_fsm_from_token_str(fsm, "+", fsm.initial, False) state_seq.insert(0, fsm.initial) res = list(get_sub_fsms_from_seq(state_seq, fsms_to_trans_finals)) assert res == [(3, True, False), (4, False, True)] - state_seq = walk_fsm(fsm, "+=", fsm.initial) + state_seq = walk_fsm_from_token_str(fsm, "+=", fsm.initial) state_seq.insert(0, fsm.initial) res = list(get_sub_fsms_from_seq(state_seq, fsms_to_trans_finals)) @@ -318,7 +356,7 @@ def test_create_fsm_index_end_to_end(): ) ) for token_tuple, token_ids in vocabulary.items(): - token = "".join(token_tuple) + token = merge_symbols(token_tuple) token_ids_np = np.fromiter(token_ids, dtype=np.dtype("int64")) vocabulary_nb.append((token, token_ids_np)) @@ -334,10 +372,6 @@ def test_create_fsm_index_end_to_end_multi_byte(): regex_fsm, _ = make_deterministic_fsm(regex_pattern.to_fsm().reduce()) byte_fsm = make_byte_level_better_fsm(regex_fsm, keep_utf8=True) - merge_symbols = lambda byte_hexs: "".join( - ["" + b if len(b) == 2 else b for b in byte_hexs] - ) - vocabulary = { "blah": numba.typed.List([0]), "😈a": numba.typed.List([1]), @@ -544,3 +578,83 @@ def build_regex(): ) profiler.dump_stats("line-profiler-build-json-regex.pkl") profiler.print_stats(output_unit=1e-3, summarize=True, stripzeros=True) + + +def test_token_trans_keys_identical(): + """assert two tokens w/ identical behavior wrt FSM have same trans key seq""" + + class MockTokenizer: + vocabulary = {"a": 1, "b": 2, "z": 3, "eos": 4} + special_tokens = {"eos"} + eos_token_id = 4 + + def convert_token_to_string(self, token): + return token + + tokenizer = MockTokenizer() + + pattern = r"z[ab]z" + regex_pattern = interegular.parse_pattern(pattern) + interegular_fsm = regex_pattern.to_fsm().reduce() + regex_fsm, _ = make_deterministic_fsm(interegular_fsm) + vocabulary, _ = reduced_vocabulary(tokenizer) + token_trans_keys = get_tokens_trans_keys( + regex_fsm.fsm_info.alphabet_symbol_mapping, + regex_fsm.fsm_info.alphabet_anything_value, + vocabulary, + ) + + token_str_trans_key_seq = { + token_str: trans_key_seq + for (token_str, _), trans_key_seq in zip(vocabulary, token_trans_keys) + } + # `a` and `b` both are workable, but `z` has distinct transition rules + assert interegular_fsm.accepts("zaz") + assert interegular_fsm.accepts("zbz") + assert (token_str_trans_key_seq["a"] == token_str_trans_key_seq["b"]).all() + assert not (token_str_trans_key_seq["a"] == token_str_trans_key_seq["z"]).all() + + +def test_token_trans_keys_walk_fsm(): + """assert _walk_fsm works using transition keys""" + + class MockTokenizer: + vocabulary = {"ab": 1, "ac": 2, "az": 3, "eos": 4} + special_tokens = {"eos"} + eos_token_id = 4 + + def convert_token_to_string(self, token): + return token + + tokenizer = MockTokenizer() + + pattern = r"a[bc]z" + regex_pattern = interegular.parse_pattern(pattern) + interegular_fsm = regex_pattern.to_fsm().reduce() + regex_fsm, _ = make_deterministic_fsm(interegular_fsm) + vocabulary, _ = reduced_vocabulary(tokenizer) + token_trans_keys = get_tokens_trans_keys( + regex_fsm.fsm_info.alphabet_symbol_mapping, + regex_fsm.fsm_info.alphabet_anything_value, + vocabulary, + ) + + token_str_trans_key_seq = { + token_str: trans_key_seq + for (token_str, _), trans_key_seq in zip(vocabulary, token_trans_keys) + } + + # verify initial state valid only for "ab" and "ac" using transition key seq + token_acceptance = {"ab": True, "ac": True, "az": False} + for token, should_accept in token_acceptance.items(): + token_trans_key_seq = token_str_trans_key_seq[token] + state_seq = _walk_fsm( + regex_fsm.fsm_info.transitions, + regex_fsm.fsm_info.initial, + regex_fsm.fsm_info.finals, + token_trans_key_seq, + regex_fsm.fsm_info.initial, + False, + ) + is_accepted = len(state_seq) >= len(token_trans_key_seq) + assert should_accept == is_accepted From ae9635c588c73338a37a978f97649041c3d2995d Mon Sep 17 00:00:00 2001 From: Andrew Lapp Date: Fri, 31 May 2024 11:51:44 -0500 Subject: [PATCH 4/8] pass token_transition_sequence to walk_fsm in parsing.py --- outlines/fsm/parsing.py | 9 +++++++- outlines/fsm/regex.py | 47 ++++++++++++++++++++++++--------------- tests/fsm/test_parsing.py | 14 +++++++----- 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/outlines/fsm/parsing.py b/outlines/fsm/parsing.py index 9ebc2af555..19deb975ed 100644 --- a/outlines/fsm/parsing.py +++ b/outlines/fsm/parsing.py @@ -38,6 +38,7 @@ from outlines.fsm.regex import ( fsm_union, get_sub_fsms_from_seq, + get_token_transitions, make_deterministic_fsm, walk_fsm, ) @@ -569,9 +570,15 @@ def match(self, text, pos, last_fsm_state_seq: Optional[Tuple[int, ...]] = None) text_part = text[start_pos:] + text_transitions = get_token_transitions( + self.fsm.fsm_info.alphabet_symbol_mapping, + self.fsm.fsm_info.alphabet_anything_value, + text_part, + ) + state_seq = walk_fsm( self.fsm, - text_part, + text_transitions, start_state, full_match=self.match_whole, ) diff --git a/outlines/fsm/regex.py b/outlines/fsm/regex.py index 1d27a0872b..6e2b81412b 100644 --- a/outlines/fsm/regex.py +++ b/outlines/fsm/regex.py @@ -675,6 +675,32 @@ def state_scan_tokens( return res +@numba.njit(cache=True, nogil=True) +def get_token_transitions( + alphabet_symbol_mapping: Dict[str, int], + alphabet_anything_value: int, + token_str: str, +) -> Sequence[int]: + trans_key_seq = [] + i = 0 + while i < len(token_str): + if token_str[i] == "\x00" and i != len(token_str) - 1: + symbol = token_str[i : i + 3] + i += 3 + else: + symbol = token_str[i] + i += 1 + + trans_key_seq.append( + alphabet_symbol_mapping.get(symbol, alphabet_anything_value) + ) + + trans_key_seq_array = np.empty(len(trans_key_seq), dtype=np.int64) + for j in range(len(trans_key_seq)): + trans_key_seq_array[j] = trans_key_seq[j] + return trans_key_seq_array + + @numba.njit(cache=True, nogil=True) def get_tokens_trans_keys( alphabet_symbol_mapping: Dict[str, int], @@ -683,24 +709,9 @@ def get_tokens_trans_keys( ) -> List[Sequence[int]]: tokens_trans_keys = numba.typed.List.empty_list(numba.int64[:]) for token_str, _ in vocabulary: - trans_key_seq = [] - i = 0 - while i < len(token_str): - if token_str[i] == "\x00" and i != len(token_str) - 1: - symbol = token_str[i : i + 3] - i += 3 - else: - symbol = token_str[i] - i += 1 - - trans_key_seq.append( - alphabet_symbol_mapping.get(symbol, alphabet_anything_value) - ) - - trans_key_seq_array = np.empty(len(trans_key_seq), dtype=np.int64) - for j in range(len(trans_key_seq)): - trans_key_seq_array[j] = trans_key_seq[j] - + trans_key_seq_array = get_token_transitions( + alphabet_symbol_mapping, alphabet_anything_value, token_str + ) tokens_trans_keys.append(trans_key_seq_array) return tokens_trans_keys diff --git a/tests/fsm/test_parsing.py b/tests/fsm/test_parsing.py index 4e093a994f..b624fddee0 100644 --- a/tests/fsm/test_parsing.py +++ b/tests/fsm/test_parsing.py @@ -9,7 +9,14 @@ from outlines.fsm.parsing import PartialLark, PartialPythonIndenter -def test_partial_parsing(): +@pytest.fixture +def cleanup_lark_import(): + yield + # Clean up lark.lark.LarkOptions._defaults + importlib.reload(lark.lark) + + +def test_partial_parsing(cleanup_lark_import): lp = PartialLark.open_from_package( "tests", "partial_python.lark", @@ -136,11 +143,8 @@ def test_partial_parsing(): assert len(parser_state.state_stack) == 4 assert parser_state.value_stack[-1].type == "LPAR" - # Clean up lark.lark.LarkOptions._defaults - importlib.reload(lark.lark) - -def test_sequential_parse_example(): +def test_sequential_parse_example(cleanup_lark_import): input_tokens = [ "x ", "= ", From a75972863e0525e5d777ee3424467fb53cb16db8 Mon Sep 17 00:00:00 2001 From: Andrew Lapp Date: Tue, 4 Jun 2024 17:02:40 -0500 Subject: [PATCH 5/8] add assertions for numba behavior re: UnicodeCharSeq / unicode_type --- tests/fsm/test_regex.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/fsm/test_regex.py b/tests/fsm/test_regex.py index f1bf0f06a9..1e14182b53 100644 --- a/tests/fsm/test_regex.py +++ b/tests/fsm/test_regex.py @@ -658,3 +658,32 @@ def convert_token_to_string(self, token): ) is_accepted = len(state_seq) >= len(token_trans_key_seq) assert should_accept == is_accepted + + +def test_numba_leading_null_byte_UnicodeCharSeq_remains_broken(): + """Assert numba UnicodeCharSeq w/ leading \x00 is still broken""" + # EXPLANATION: + # https://github.com/outlines-dev/outlines/pull/930#issuecomment-2143535968 + + # from https://github.com/numba/numba/issues/9542 + d = numba.typed.typeddict.Dict.empty(numba.types.UnicodeCharSeq(1), numba.int64) + d["δΈ€"] = 10 # \xe4\xb8\x80 + with pytest.raises(KeyError): + str(d) + + # most characters are fine, but "\x00" is converted to "" + l = np.fromiter(["\x99", "\x00"], dtype=np.dtype("U2")) + assert str(l[0]) == "\x99" # fine + assert str(l[1]) == "" # 1-byte null converted to 0-bytes + + +@pytest.mark.parametrize("input_key", ["δΈ€", "\x00"]) +def test_numba_leading_null_byte_unicode_type_sane(input_key): + """Assert numba unicode_type w/ leading \x00 is working""" + # EXPLANATION: + # https://github.com/outlines-dev/outlines/pull/930#issuecomment-2143535968 + + # from https://github.com/numba/numba/issues/9542 + d = numba.typed.typeddict.Dict.empty(numba.types.unicode_type, numba.int64) + d["δΈ€"] = 10 # \xe4\xb8\x80 + str(d) # assert successfully interprets From 1e3902776f9b2f58528f7939da55a72fc33c913b Mon Sep 17 00:00:00 2001 From: Andrew Lapp Date: Tue, 4 Jun 2024 17:26:18 -0500 Subject: [PATCH 6/8] Add docstrings to get_token/vocab_transition_keys, improve naming --- outlines/fsm/parsing.py | 4 ++-- outlines/fsm/regex.py | 43 ++++++++++++++++++++++++++--------------- tests/fsm/test_regex.py | 25 ++++++++++++------------ 3 files changed, 41 insertions(+), 31 deletions(-) diff --git a/outlines/fsm/parsing.py b/outlines/fsm/parsing.py index 19deb975ed..e4fa7b7640 100644 --- a/outlines/fsm/parsing.py +++ b/outlines/fsm/parsing.py @@ -38,7 +38,7 @@ from outlines.fsm.regex import ( fsm_union, get_sub_fsms_from_seq, - get_token_transitions, + get_token_transition_keys, make_deterministic_fsm, walk_fsm, ) @@ -570,7 +570,7 @@ def match(self, text, pos, last_fsm_state_seq: Optional[Tuple[int, ...]] = None) text_part = text[start_pos:] - text_transitions = get_token_transitions( + text_transitions = get_token_transition_keys( self.fsm.fsm_info.alphabet_symbol_mapping, self.fsm.fsm_info.alphabet_anything_value, text_part, diff --git a/outlines/fsm/regex.py b/outlines/fsm/regex.py index 6e2b81412b..e347a31cbd 100644 --- a/outlines/fsm/regex.py +++ b/outlines/fsm/regex.py @@ -649,24 +649,24 @@ def state_scan_tokens( fsm_initial: int, fsm_finals: Set[int], vocabulary: List[Tuple[str, Sequence[int]]], - token_trans_key_seqs: List[Sequence[int]], + vocabulary_transition_keys: List[Sequence[int]], start_state: int, ) -> Set[Tuple[int, int]]: res = set() - for (token, token_ids), token_trans_key_seq in zip( - vocabulary, token_trans_key_seqs + for (token, token_ids), token_transition_keys in zip( + vocabulary, vocabulary_transition_keys ): state_seq = _walk_fsm( fsm_transitions, fsm_initial, fsm_finals, - token_trans_key_seq, + token_transition_keys, start_state, False, ) - if state_seq is not None and len(state_seq) < len(token_trans_key_seq): + if state_seq is not None and len(state_seq) < len(token_transition_keys): continue for token_id in token_ids: @@ -676,12 +676,20 @@ def state_scan_tokens( @numba.njit(cache=True, nogil=True) -def get_token_transitions( +def get_token_transition_keys( alphabet_symbol_mapping: Dict[str, int], alphabet_anything_value: int, token_str: str, ) -> Sequence[int]: - trans_key_seq = [] + """ + Get the sequence of transition keys for an individual string + with respect to an FSMs alphabet symbol mapping + + This requires parsing the null-byte prefix rules of a byte-fsm: + - If two characters are prefixed by \x00, they are the grouped as a hex-byte + - Otherwise they are a standalone utf-8 character + """ + token_transition_keys = [] i = 0 while i < len(token_str): if token_str[i] == "\x00" and i != len(token_str) - 1: @@ -691,25 +699,28 @@ def get_token_transitions( symbol = token_str[i] i += 1 - trans_key_seq.append( + token_transition_keys.append( alphabet_symbol_mapping.get(symbol, alphabet_anything_value) ) - trans_key_seq_array = np.empty(len(trans_key_seq), dtype=np.int64) - for j in range(len(trans_key_seq)): - trans_key_seq_array[j] = trans_key_seq[j] - return trans_key_seq_array + tok_trans_array = np.empty(len(token_transition_keys), dtype=np.int64) + for j in range(len(token_transition_keys)): + tok_trans_array[j] = token_transition_keys[j] + return tok_trans_array @numba.njit(cache=True, nogil=True) -def get_tokens_trans_keys( +def get_vocabulary_transition_keys( alphabet_symbol_mapping: Dict[str, int], alphabet_anything_value: int, vocabulary: List[Tuple[str, Sequence[int]]], ) -> List[Sequence[int]]: + """ + Calculate the sequence transition keys for each token str within a vocabulary + """ tokens_trans_keys = numba.typed.List.empty_list(numba.int64[:]) for token_str, _ in vocabulary: - trans_key_seq_array = get_token_transitions( + trans_key_seq_array = get_token_transition_keys( alphabet_symbol_mapping, alphabet_anything_value, token_str ) tokens_trans_keys.append(trans_key_seq_array) @@ -735,7 +746,7 @@ def create_fsm_index_end_to_end( desc="Compiling FSM index for all state transitions", ) - tokens_trans_key_seqs = get_tokens_trans_keys( + vocabulary_transition_keys = get_vocabulary_transition_keys( fsm_info.alphabet_symbol_mapping, fsm_info.alphabet_anything_value, vocabulary, @@ -751,7 +762,7 @@ def create_fsm_index_end_to_end( fsm_info.initial, fsm_info.finals, vocabulary, - tokens_trans_key_seqs, + vocabulary_transition_keys, start_state, ) diff --git a/tests/fsm/test_regex.py b/tests/fsm/test_regex.py index 1e14182b53..943ee05fd1 100644 --- a/tests/fsm/test_regex.py +++ b/tests/fsm/test_regex.py @@ -10,7 +10,8 @@ create_fsm_index_tokenizer, fsm_union, get_sub_fsms_from_seq, - get_tokens_trans_keys, + get_token_transition_keys, + get_vocabulary_transition_keys, make_byte_level_better_fsm, make_byte_level_fsm, make_deterministic_fsm, @@ -33,15 +34,11 @@ def merge_symbols(byte_hexs): def token_str_to_trans_key(fsm, input_string): - vocabulary_nb = numba.typed.List.empty_list( - numba.types.Tuple((numba.types.unicode_type, numba.int64[:])) - ) - vocabulary_nb.append((input_string, np.fromiter([1], dtype=np.dtype("int64")))) - return get_tokens_trans_keys( + return get_token_transition_keys( fsm.fsm_info.alphabet_symbol_mapping, fsm.fsm_info.alphabet_anything_value, - vocabulary_nb, - )[0] + input_string, + ) def walk_fsm_from_token_str( @@ -598,21 +595,23 @@ def convert_token_to_string(self, token): interegular_fsm = regex_pattern.to_fsm().reduce() regex_fsm, _ = make_deterministic_fsm(interegular_fsm) vocabulary, _ = reduced_vocabulary(tokenizer) - token_trans_keys = get_tokens_trans_keys( + token_trans_keys = get_vocabulary_transition_keys( regex_fsm.fsm_info.alphabet_symbol_mapping, regex_fsm.fsm_info.alphabet_anything_value, vocabulary, ) - token_str_trans_key_seq = { + token_str_to_tranition_keys = { token_str: trans_key_seq for (token_str, _), trans_key_seq in zip(vocabulary, token_trans_keys) } # `a` and `b` both are workable, but `z` has distinct transition rules assert interegular_fsm.accepts("zaz") assert interegular_fsm.accepts("zbz") - assert (token_str_trans_key_seq["a"] == token_str_trans_key_seq["b"]).all() - assert not (token_str_trans_key_seq["a"] == token_str_trans_key_seq["z"]).all() + assert (token_str_to_tranition_keys["a"] == token_str_to_tranition_keys["b"]).all() + assert not ( + token_str_to_tranition_keys["a"] == token_str_to_tranition_keys["z"] + ).all() def test_token_trans_keys_walk_fsm(): @@ -633,7 +632,7 @@ def convert_token_to_string(self, token): interegular_fsm = regex_pattern.to_fsm().reduce() regex_fsm, _ = make_deterministic_fsm(interegular_fsm) vocabulary, _ = reduced_vocabulary(tokenizer) - token_trans_keys = get_tokens_trans_keys( + token_trans_keys = get_vocabulary_transition_keys( regex_fsm.fsm_info.alphabet_symbol_mapping, regex_fsm.fsm_info.alphabet_anything_value, vocabulary, From 4468180f0ce144b42a7fbae30f7ea8babb3c3dec Mon Sep 17 00:00:00 2001 From: Andrew Lapp Date: Tue, 4 Jun 2024 17:34:20 -0500 Subject: [PATCH 7/8] remove cleanup step redundant with cleanup_lark_import --- tests/fsm/test_parsing.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/fsm/test_parsing.py b/tests/fsm/test_parsing.py index b624fddee0..3f4c1ba420 100644 --- a/tests/fsm/test_parsing.py +++ b/tests/fsm/test_parsing.py @@ -204,6 +204,3 @@ def test_sequential_parse_example(cleanup_lark_import): if i + 1 == len(input_tokens): assert all(tk in next_vocab for tk in ["\n", "\nde", " ", " + 1"]) - - # Clean up lark.lark.LarkOptions._defaults - importlib.reload(lark.lark) From 501520a4ab0c028046160b326f2e151e8ca0b406 Mon Sep 17 00:00:00 2001 From: Andrew Lapp Date: Tue, 4 Jun 2024 17:49:16 -0500 Subject: [PATCH 8/8] Improve naming of token and vocab transition keys in regex.py --- outlines/fsm/regex.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/outlines/fsm/regex.py b/outlines/fsm/regex.py index e347a31cbd..0b35c978ff 100644 --- a/outlines/fsm/regex.py +++ b/outlines/fsm/regex.py @@ -414,7 +414,7 @@ def _walk_fsm( fsm_transitions: Dict[Tuple[int, int], int], fsm_initial: int, fsm_finals: Set[int], - token_trans_key_seq: Sequence[int], + token_transition_keys: Sequence[int], start_state: int, full_match: bool = True, ) -> List[int]: @@ -424,7 +424,7 @@ def _walk_fsm( # Iterate over token transition key sequence. The transition key # sequence represents the FSM traversal rules of the tokens symbols. - for i, trans_key in enumerate(token_trans_key_seq): + for i, trans_key in enumerate(token_transition_keys): new_state = fsm_transitions.get((state, trans_key)) if new_state is None: @@ -448,7 +448,7 @@ def _walk_fsm( def walk_fsm( fsm: BetterFSM, - token_trans_key_seq: Sequence[int], + token_transition_keys: Sequence[int], start_state: int, full_match: bool = True, ) -> List[int]: @@ -462,7 +462,7 @@ def walk_fsm( # Iterate over token transition key sequence. The transition key # sequence represents the FSM traversal rules of the tokens symbols. - for i, trans_key in enumerate(token_trans_key_seq): + for i, trans_key in enumerate(token_transition_keys): new_state = fsm_transitions.get((state, trans_key)) if new_state is None: @@ -703,10 +703,10 @@ def get_token_transition_keys( alphabet_symbol_mapping.get(symbol, alphabet_anything_value) ) - tok_trans_array = np.empty(len(token_transition_keys), dtype=np.int64) + token_transition_keys_array = np.empty(len(token_transition_keys), dtype=np.int64) for j in range(len(token_transition_keys)): - tok_trans_array[j] = token_transition_keys[j] - return tok_trans_array + token_transition_keys_array[j] = token_transition_keys[j] + return token_transition_keys_array @numba.njit(cache=True, nogil=True) @@ -718,14 +718,14 @@ def get_vocabulary_transition_keys( """ Calculate the sequence transition keys for each token str within a vocabulary """ - tokens_trans_keys = numba.typed.List.empty_list(numba.int64[:]) + vocab_transition_keys = numba.typed.List.empty_list(numba.int64[:]) for token_str, _ in vocabulary: - trans_key_seq_array = get_token_transition_keys( + token_transition_keys = get_token_transition_keys( alphabet_symbol_mapping, alphabet_anything_value, token_str ) - tokens_trans_keys.append(trans_key_seq_array) + vocab_transition_keys.append(token_transition_keys) - return tokens_trans_keys + return vocab_transition_keys def create_fsm_index_end_to_end(