Skip to content

Commit d245cfc

Browse files
pcullitoncopybara-github
authored andcommitted
Internal change.
PiperOrigin-RevId: 732288967
1 parent 2bdf26d commit d245cfc

File tree

2 files changed

+8
-8
lines changed

2 files changed

+8
-8
lines changed

compression/shared.h

+6-7
Original file line numberDiff line numberDiff line change
@@ -180,17 +180,16 @@ constexpr bool IsSfpStream() {
180180
return hwy::IsSame<hwy::RemoveCvRef<Packed>, SfpStream>();
181181
}
182182

183-
template <typename Packed>
184-
constexpr bool IsNuqStream() {
185-
return hwy::IsSame<hwy::RemoveCvRef<Packed>, NuqStream>();
186-
}
187-
188183
// Instruction-tuned models require extra 'turn structure' tokens in prompts.
189-
enum class PromptWrapping { GEMMA_IT, GEMMA_PT, PALIGEMMA };
184+
enum class PromptWrapping {
185+
GEMMA_IT,
186+
GEMMA_PT,
187+
PALIGEMMA,
188+
};
190189

191190
inline bool EnumValid(PromptWrapping type) {
192191
return static_cast<int>(type) >= 0 &&
193-
static_cast<int>(type) <= static_cast<int>(PromptWrapping::PALIGEMMA);
192+
static_cast<int>(type) <= static_cast<int>(PromptWrapping::PALIGEMMA);
194193
}
195194

196195
// Tensor types for loading weights. Note that not all types are supported as

gemma/tokenizer.cc

+2-1
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,8 @@ std::vector<int> WrapAndTokenize(const GemmaTokenizer& tokenizer,
127127
}
128128

129129
// PaliGemma separator. The SEP token "\n" is always tokenized separately.
130-
if (info.wrapping == PromptWrapping::PALIGEMMA) {
130+
if (info.wrapping == PromptWrapping::PALIGEMMA
131+
) {
131132
std::vector<int> sep_tokens;
132133
HWY_ASSERT(tokenizer.Encode("\n", &sep_tokens));
133134
tokens.insert(tokens.end(), sep_tokens.begin(), sep_tokens.end());

0 commit comments

Comments
 (0)