Merge #295

295: Add null byte as hard context separator r=ManyTheFish a=LukasKalbertodt

# Pull Request

## Related issue
Fixes https://github.com/orgs/meilisearch/discussions/744

## What does this PR do?
Adds `\0` as context separator.

This allows one to use \0 as artificial separator, for example when concatting lots of small strings into a large string. See this discussion for context: https://github.com/orgs/meilisearch/discussions/744


## PR checklist
Please check if your PR fulfills the following requirements:
- [ ] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [ ] Have you read the contributing guidelines?
- [ ] Have you made sure that the title is accurate and descriptive of the changes?

Thank you so much for contributing to Meilisearch!


Co-authored-by: Lukas Kalbertodt <[email protected]>

Author: meili-bors[bot]

charabia/src/separators.rs

@@ -11,11 +11,11 @@
 /// - Zl Line Separator
 /// - Zp Paragraph Separator
 /// - Zs Space Separator
-/// plus ". ", ", " and ។ល។" (៘ decomposition) to categorize them as hard separators
+/// plus "\0", ". ", ", " and ។ល។" (៘ decomposition) to categorize them as hard separators
 /// and "`" to understand markdown formatted text
 #[rustfmt::skip]
 pub const DEFAULT_SEPARATORS: &[&str] = &[
-    ". ", ", ", "_", "‿", "⁀", "⁔", "︳", "︴", "﹍", "﹎", "﹏", "＿", "-", "֊", "־", "᐀", "᠆", "‐", "‒", "–",
+    "\0", ". ", ", ", "_", "‿", "⁀", "⁔", "︳", "︴", "﹍", "﹎", "﹏", "＿", "-", "֊", "־", "᐀", "᠆", "‐", "‒", "–",
     "—", "―", "⸗", "⸚", "⸺", "⸻", "⹀", "〜", "〰", "゠", "︱", "︲", "﹘", "﹣", "－", "𐺭", ")",
     "]", "}", "༻", "༽", "᚜", "⁆", "⁾", "₎", "⌉", "⌋", "〉", "❩", "❫", "❭", "❯", "❱", "❳", "❵", "⟆",
     "⟧", "⟩", "⟫", "⟭", "⟯", "⦄", "⦆", "⦈", "⦊", "⦌", "⦎", "⦐", "⦒", "⦔", "⦖", "⦘", "⧙", "⧛", "⧽",
@@ -64,6 +64,7 @@ pub const DEFAULT_SEPARATORS: &[&str] = &[
 
 #[rustfmt::skip]
 pub const CONTEXT_SEPARATORS: &[&str] = &[
+    "\0", // Null byte, can be used as artificial separator
     "᠆", // Mongolian Todo Soft Hyphen, mark the end of a paragraph.
     "᚛", "᚜", // Oghams, mark start and end of text
     "!", ". ", ", ", ";", "?", "¡", "§", "¶", "¿", ";", // Latin