From 2e5c8aeab0134b4265df4c8ac48a645bc0b3bad7 Mon Sep 17 00:00:00 2001 From: marcus Date: Fri, 24 Nov 2023 16:29:58 -0800 Subject: [PATCH 1/8] reserve space for codepoints --- llama.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index e599917a81eb1..08daaf8897b72 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6903,10 +6903,13 @@ struct llama_grammar_candidate { // pointer. If an invalid sequence is encountered, returns `llama_partial_utf8.n_remain == -1`. static std::pair, llama_partial_utf8> decode_utf8( const char * src, + size_t n_src, llama_partial_utf8 partial_start) { static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 }; const char * pos = src; std::vector code_points; + // common english strings have the same number of codepoints and bytes. + code_points.reserve(n_src); uint32_t value = partial_start.value; int n_remain = partial_start.n_remain; @@ -6957,6 +6960,13 @@ static std::pair, llama_partial_utf8> decode_utf8( return std::make_pair(std::move(code_points), llama_partial_utf8{ value, n_remain }); } +static std::pair, llama_partial_utf8> decode_utf8( + std::string src, + llama_partial_utf8 partial_start +) { + return decode_utf8(src.c_str(), src.size(), partial_start); +} + // returns true iff pos points to the end of one of the definitions of a rule static bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) { switch (pos->type) { @@ -7580,7 +7590,7 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c } else if (piece.empty() || piece[0] == 0) { candidates->data[i].logit = -INFINITY; } else { - candidates_decoded.push_back(decode_utf8(piece.c_str(), grammar->partial_utf8)); + candidates_decoded.push_back(decode_utf8(piece, grammar->partial_utf8)); candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second }); } } @@ -7787,7 +7797,7 @@ void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar const std::string piece = llama_token_to_piece(ctx, token); // Note terminating 0 in decoded string - const auto decoded = decode_utf8(piece.c_str(), grammar->partial_utf8); + const auto decoded = decode_utf8(piece, grammar->partial_utf8); const auto & code_points = decoded.first; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); From 9d3ba0bacdb6705ec2c8ad1ce21e1dfb6f6a2db8 Mon Sep 17 00:00:00 2001 From: marcus Date: Fri, 24 Nov 2023 17:27:18 -0800 Subject: [PATCH 2/8] improvement for the appended 0 --- llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index ec23485fda22c..f2b5967d791e9 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6425,8 +6425,8 @@ static std::pair, llama_partial_utf8> decode_utf8( static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 }; const char * pos = src; std::vector code_points; - // common english strings have the same number of codepoints and bytes. - code_points.reserve(n_src); + // common english strings have the same number of codepoints and bytes. `+ 1` for the terminating 0. + code_points.reserve(n_src + 1); uint32_t value = partial_start.value; int n_remain = partial_start.n_remain; From 5dd1f45e1d5c017af22c209ff4d6f35f2fa8d5e3 Mon Sep 17 00:00:00 2001 From: marcus Date: Mon, 4 Dec 2023 13:30:27 -0800 Subject: [PATCH 3/8] used precomputed token text for grammar sample --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index fd905ade7a73b..c16819a48bad8 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7361,7 +7361,7 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c for (size_t i = 0; i < candidates->size; ++i) { const llama_token id = candidates->data[i].id; - const std::string piece = llama_token_to_piece(ctx, id); + const std::string piece = ctx->model.vocab.id_to_token[id].text; if (id == eos) { if (!allow_eos) { candidates->data[i].logit = -INFINITY; From 7cd0d3232f73b1b7fd93d305a95b21cc868ff447 Mon Sep 17 00:00:00 2001 From: marcus Date: Mon, 4 Dec 2023 14:09:49 -0800 Subject: [PATCH 4/8] reserve canidates_decoded --- llama.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llama.cpp b/llama.cpp index c16819a48bad8..99b5c5b392e09 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7357,6 +7357,7 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c const llama_token eos = llama_token_eos(&ctx->model); std::vector, llama_partial_utf8>> candidates_decoded; + candidates_decoded.reserve(candidates->size); std::vector candidates_grammar; for (size_t i = 0; i < candidates->size; ++i) { From eb9d1fcd7dbef531a14783a0ff2fabbe4a2812ee Mon Sep 17 00:00:00 2001 From: marcus Date: Mon, 4 Dec 2023 14:10:11 -0800 Subject: [PATCH 5/8] reserve canidates_grammar --- llama.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llama.cpp b/llama.cpp index 99b5c5b392e09..ec4a7f94ccba8 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7359,6 +7359,7 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c std::vector, llama_partial_utf8>> candidates_decoded; candidates_decoded.reserve(candidates->size); std::vector candidates_grammar; + candidates_grammar.reserve(candidates->size); for (size_t i = 0; i < candidates->size; ++i) { const llama_token id = candidates->data[i].id; From 3773328080e6a139ee83198329a13cf4ff61d707 Mon Sep 17 00:00:00 2001 From: marcus Date: Mon, 4 Dec 2023 14:11:45 -0800 Subject: [PATCH 6/8] remove candidates_decoded --- llama.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llama.cpp b/llama.cpp index ec4a7f94ccba8..53f5bee28ae88 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7356,8 +7356,6 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c const llama_token eos = llama_token_eos(&ctx->model); - std::vector, llama_partial_utf8>> candidates_decoded; - candidates_decoded.reserve(candidates->size); std::vector candidates_grammar; candidates_grammar.reserve(candidates->size); @@ -7371,8 +7369,8 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c } else if (piece.empty() || piece[0] == 0) { candidates->data[i].logit = -INFINITY; } else { - candidates_decoded.push_back(decode_utf8(piece, grammar->partial_utf8)); - candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second }); + std::pair, llama_partial_utf8> decoded = decode_utf8(piece, grammar->partial_utf8); + candidates_grammar.push_back({ i, decoded.first.data(), decoded.second }); } } From 71596272c36769a98e23b164480fb5c61af8353f Mon Sep 17 00:00:00 2001 From: marcus Date: Mon, 4 Dec 2023 14:29:59 -0800 Subject: [PATCH 7/8] Revert "remove candidates_decoded" This reverts commit 3773328080e6a139ee83198329a13cf4ff61d707. --- llama.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 53f5bee28ae88..ec4a7f94ccba8 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7356,6 +7356,8 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c const llama_token eos = llama_token_eos(&ctx->model); + std::vector, llama_partial_utf8>> candidates_decoded; + candidates_decoded.reserve(candidates->size); std::vector candidates_grammar; candidates_grammar.reserve(candidates->size); @@ -7369,8 +7371,8 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c } else if (piece.empty() || piece[0] == 0) { candidates->data[i].logit = -INFINITY; } else { - std::pair, llama_partial_utf8> decoded = decode_utf8(piece, grammar->partial_utf8); - candidates_grammar.push_back({ i, decoded.first.data(), decoded.second }); + candidates_decoded.push_back(decode_utf8(piece, grammar->partial_utf8)); + candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second }); } } From b629ede6b389ecc3d45f85f6914e330b49ede528 Mon Sep 17 00:00:00 2001 From: marcus Date: Tue, 5 Dec 2023 10:58:24 -0800 Subject: [PATCH 8/8] changed decode_utf8 to take src by ref --- llama.cpp | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/llama.cpp b/llama.cpp index ec4a7f94ccba8..88cfa2a88cda2 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6654,14 +6654,13 @@ struct llama_grammar_candidate { // Decodes a UTF-8 string which may end in an incomplete sequence. Adds a terminating 0 for use as // pointer. If an invalid sequence is encountered, returns `llama_partial_utf8.n_remain == -1`. static std::pair, llama_partial_utf8> decode_utf8( - const char * src, - size_t n_src, + const std::string & src, llama_partial_utf8 partial_start) { static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 }; - const char * pos = src; + const char * pos = src.c_str(); std::vector code_points; // common english strings have the same number of codepoints and bytes. `+ 1` for the terminating 0. - code_points.reserve(n_src + 1); + code_points.reserve(src.size() + 1); uint32_t value = partial_start.value; int n_remain = partial_start.n_remain; @@ -6712,13 +6711,6 @@ static std::pair, llama_partial_utf8> decode_utf8( return std::make_pair(std::move(code_points), llama_partial_utf8{ value, n_remain }); } -static std::pair, llama_partial_utf8> decode_utf8( - std::string src, - llama_partial_utf8 partial_start -) { - return decode_utf8(src.c_str(), src.size(), partial_start); -} - // returns true iff pos points to the end of one of the definitions of a rule static bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) { switch (pos->type) { @@ -7363,7 +7355,7 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c for (size_t i = 0; i < candidates->size; ++i) { const llama_token id = candidates->data[i].id; - const std::string piece = ctx->model.vocab.id_to_token[id].text; + const std::string & piece = ctx->model.vocab.id_to_token[id].text; if (id == eos) { if (!allow_eos) { candidates->data[i].logit = -INFINITY; @@ -7575,7 +7567,7 @@ void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar GGML_ASSERT(false); } - const std::string piece = llama_token_to_piece(ctx, token); + const std::string & piece = ctx->model.vocab.id_to_token[token].text; // Note terminating 0 in decoded string const auto decoded = decode_utf8(piece, grammar->partial_utf8);