Merge #1859

1859: v0.29 r=maryamsulemani97 a=guimachiavelli

This is a staging PR for all changes related to Meilisearch v0.29.

Please avoid making changes directly to this PR; instead, create new child branches based off this one.

Closes meilisearch/integration-guides#213, #1854, #1853, #1852, #1851, #1840, #1839, #1838, #1837, #1846


Co-authored-by: gui machiavelli <[email protected]>
Co-authored-by: Maryam Sulemani <[email protected]>
Co-authored-by: gui machiavelli <[email protected]>
Co-authored-by: Maryam <[email protected]>
Co-authored-by: Clémentine Urquizar <[email protected]>

Author: 4 people

.code-samples.meilisearch.yaml

@@ -986,3 +986,19 @@ getting_started_pagination: |-
     --data-binary '{
       "maxTotalHits": 500
     }'
+search_parameter_guide_matching_strategy_1: |
+  curl \
+    -X POST 'http://localhost:7700/indexes/movies/search' \
+    -H 'Content-Type: application/json' \
+    --data-binary '{
+      "q": "big fat liar",
+      "matchingStrategy": "last"
+    }'      
+search_parameter_guide_matching_strategy_2: |
+  curl \
+    -X POST 'http://localhost:7700/indexes/movies/search' \
+    -H 'Content-Type: application/json' \
+    --data-binary '{
+      "q": "big fat liar",
+      "matchingStrategy": "all"
+    }'    

.vuepress/config.js

@@ -1,6 +1,6 @@
 const ogprefix = 'og: http://ogp.me/ns#'
 module.exports = {
-  title: 'Meilisearch Documentation v0.28',
+  title: 'Meilisearch Documentation v0.29',
   description: 'Open source Instant Search Engine',
   theme: 'default-prefers-color-scheme',
   themeConfig: {
@@ -271,21 +271,6 @@ module.exports = {
             },
           ],
         },
-        {
-          title: '🧪 Experimental',
-          collapsable: false,
-          path: '/learn/experimental/overview.html',
-          children: [
-            {
-              title: 'Overview',
-              path: '/learn/experimental/overview',
-            },
-            {
-              title: 'Auto-batching',
-              path: '/learn/experimental/auto-batching',
-            },
-          ],
-        },
         {
           title: '👐 Contributing',
           path: '/learn/contributing/overview.html',

.vuepress/public/_redirects

@@ -196,3 +196,6 @@
 
 # Rename indexation to indexing
 /learn/advanced/indexation.html                     /learn/advanced/indexing.html
+
+# Remove autobatching
+/learn/experimental/auto-batching.html              /learn/core_concepts/documents.html

.vuepress/public/postman/meilisearch-collection.json

@@ -1,7 +1,7 @@
 {
 	"info": {
-		"_postman_id": "453f3754-4b37-4654-8fe3-b2b3aef24048",
-		"name": "Meilisearch v0.28",
+		"_postman_id": "5ad97bb3-840b-40dc-9012-482a0c2c52a5",
+		"name": "Meilisearch v0.29",
 		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
 		"_exporter_id": "8898306"
 	},
@@ -79,7 +79,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "[\n  { \"id\": 2,    \"title\": \"Pride and Prejudice\",                    \"author\": \"Jane Austin\",              \"genre\": \"romance\",    \"price\": 3.5 },\n  { \"id\": 456,  \"title\": \"Le Petit Prince\",                        \"author\": \"Antoine de Saint-Exupéry\", \"genre\": \"adventure\" , \"price\": 10.0 },\n  { \"id\": 1,    \"title\": \"Alice In Wonderland\",                    \"author\": \"Lewis Carroll\",            \"genre\": \"fantasy\",    \"price\": 25.99 },\n  { \"id\": 1344, \"title\": \"The Hobbit\",                             \"author\": \"J. R. R. Tolkien\",         \"genre\": \"fantasy\" },\n  { \"id\": 4,    \"title\": \"Harry Potter and the Half-Blood Prince\", \"author\": \"J. K. Rowling\",            \"genre\": \"fantasy\" },\n  { \"id\": 42,   \"title\": \"The Hitchhiker's Guide to the Galaxy\",   \"author\": \"Douglas Adams\" }\n]",
+							"raw": "[\n  { \"id\": 2,    \"title\": \"Pride and Prejudice\",                    \"author\": \"Jane Austen\",              \"genre\": \"romance\",    \"price\": 3.5 },\n  { \"id\": 456,  \"title\": \"Le Petit Prince\",                        \"author\": \"Antoine de Saint-Exupéry\", \"genre\": \"adventure\" , \"price\": 10.0 },\n  { \"id\": 1,    \"title\": \"Alice In Wonderland\",                    \"author\": \"Lewis Carroll\",            \"genre\": \"fantasy\",    \"price\": 25.99 },\n  { \"id\": 1344, \"title\": \"The Hobbit\",                             \"author\": \"J. R. R. Tolkien\",         \"genre\": \"fantasy\" },\n  { \"id\": 4,    \"title\": \"Harry Potter and the Half-Blood Prince\", \"author\": \"J. K. Rowling\",            \"genre\": \"fantasy\" },\n  { \"id\": 42,   \"title\": \"The Hitchhiker's Guide to the Galaxy\",   \"author\": \"Douglas Adams\" }\n]",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -309,6 +309,11 @@
 									"key": "highlightPostTag",
 									"value": "</mark>",
 									"disabled": true
+								},
+								{
+									"key": "matchingStrategy",
+									"value": "all",
+									"disabled": true
 								}
 							]
 						}

.vuepress/public/sample-template.yaml

@@ -152,3 +152,5 @@ synonyms_guide_1: |-
 getting_started_faceting: |-
 getting_started_pagination: |-
 getting_started_front_end_integration_md: |-
+search_parameter_guide_matching_strategy_1: |-
+search_parameter_guide_matching_strategy_2: |-

learn/advanced/filtering_and_faceted_search.md

@@ -102,10 +102,14 @@ The [`GET` route of the search endpoint](/reference/api/search.md#search-in-an-i
 
 String expressions combine conditions using the following filter operators and parentheses:
 
-- `NOT` only returns documents that do not satisfy a condition : `NOT genres = horror`
+- `NOT` returns all documents that do not satisfy a condition. The expression `NOT genres = horror` returns all documents whose `genres` do not contain `horror` and all documents missing a `genres` field
 - `AND` operates by connecting two conditions and only returns documents that satisfy both of them: `genres = horror AND director = 'Jordan Peele'`
 - `OR` connects two conditions and returns results that satisfy at least one of them: `genres = horror OR genres = comedy`
 - `TO` is equivalent to `>= AND <=`. The expression `release_date 795484800 TO 972129600` translates to `release_date >= 795484800 AND release_date <= 972129600`
+- `IN [valueA, valueB, …, valueN]` selects all documents whose chosen field contains at least one of the specified values. The expression `genres IN [horror, comedy]` returns all documents whose `genres` includes either `horror`, `comedy`, or both
+- `EXISTS` checks for the existence of a field. Fields with empty or null values still count as existing. The expression `release_date NOT EXISTS` returns all documents without a `release_date`
+
+When creating an expression with a field name or value identical to a filter operator such as `AND` or `NOT`, you must wrap it in quotation marks: `title = "NOT" OR title = "AND"`.
 
 ::: tip
 String expressions are read left to right. `NOT` takes precedence over `AND` and `AND` takes precedence over `OR`. You can use parentheses to ensure expressions are correctly parsed.
@@ -220,6 +224,12 @@ You can use this filter when searching for `Planet of the Apes`:
 
 <CodeSamples id="filtering_guide_3" />
 
+`NOT director = "Tim Burton"` will include both documents that do not contain `"Tim Burton"` in its `director` field and documents without a `director` field. To return only documents that have a `director` field, expand the filter expression with the `EXISTS` operator:
+
+```SQL
+rating >= 3 AND (NOT director = "Tim Burton" AND director EXISTS)
+```
+
 ## Filtering with `_geoRadius`
 
 If your documents contain `_geo` data, you can use the `_geoRadius` built-in filter rule to filter results according to their geographic position.

learn/advanced/indexing.md

@@ -28,16 +28,14 @@ Multi-threading is unfortunately not possible in machines with only one processo
 
 ## Improving indexing performance
 
-If you encounter performance issues during the indexing we recommend trying the following points:
+If you encounter performance issues during indexing, we recommend trying the following:
 
 - Make sure you are using the latest [stable version of Meilisearch](https://github.com/meilisearch/meilisearch/releases). New releases often include performance improvements that can significantly increase indexing speed.
 
-- indexing is a memory-intensive and multi-threaded operation. This means **the more memory and processor cores available, the faster Meilisearch will index new documents**. When trying to improve indexing speed, using a machine with more processor cores is more effective than increasing RAM.
+- Indexing is a memory-intensive and multi-threaded operation. This means **the more memory and processor cores available, the faster Meilisearch will index new documents**. When trying to improve indexing speed, using a machine with more processor cores is more effective than increasing RAM.
 
 - **Bigger HTTP payloads are processed more quickly than smaller payloads**. For example, adding the same 100,000 documents in two batches of 50,000 documents will be quicker than adding them in four batches of 25,000 documents. By default, Meilisearch sets the maximum payload size to 100MB, but [you can change this value if necessary](/learn/configuration/instance_options.md#payload-limit-size). That said, **the bigger the payload is, the higher the memory consumption will be**. An instance may crash if it requires more RAM than is currently available in a machine.
 
-  - If you want to speed up indexing but don't wish to batch documents manually, consider giving our [experimental auto-batcher](/learn/experimental/auto-batching.md) a try.
-
 - **Meilisearch should not be your main database**. The more documents you add, the longer will indexing and search take, so you should only index documents you want to retrieve when searching.
 
 - By default, all document fields are searchable. We strongly recommend changing this by [updating the `searchableAttributes` list](/reference/api/settings.md#update-searchable-attributes) so it only contains fields you want to search in. The fewer fields Meilisearch needs to index, the faster is the indexing process.

learn/advanced/storage.md

@@ -71,3 +71,9 @@ These metrics are highly dependent on the machine that is running Meilisearch. R
 It is important to note that **there is no reliable way to predict the final size of a database**. This is true for just about any search engine on the market—we're just the only ones saying it out loud.
 
 Database size is affected by a large number of criteria, including settings, relevancy rules, use of facets, the number of different languages present, and more.
+
+## Soft deletion
+
+Meilisearch renders deleted documents inaccessible to all users but does not immediately remove them from the database. This is a common optimization technique called soft deletion. Soft deleted documents are permanently deleted during a later update, depending on your index size and the available disk space. It might be important to check how soft deletion interacts with data retention legislation relevant to your application.
+
+Soft deletion also affects document updates: when you update a document, Meilisearch removes the current record and creates a new document with updated data.

learn/advanced/tokenization.md

@@ -21,5 +21,6 @@ Pipelines include many language-specific operations. Currently, we have four pip
 2. A specialized Chinese pipeline using [Jieba](https://github.com/messense/jieba-rs)
 3. A specialized Japanese pipeline using [Lindera](https://github.com/lindera-morphology/lindera)
 4. A specialized Hebrew pipeline based off the default Meilisearch pipeline. Uses [Niqqud](https://docs.rs/niqqud/latest/niqqud/) for normalization
+5. A specialized Thai pipeline using [dictionary-based](https://github.com/PyThaiNLP/nlpo3) segmentation
 
 For more details, check out the [tokenizer contribution guide](https://github.com/meilisearch/charabia/blob/main/CONTRIBUTING.md).

learn/configuration/instance_options.md

@@ -106,6 +106,19 @@ If no master key is provided in a `development` environment, all routes will be
 
 [Learn more about Meilisearch's use of security keys.](/learn/security/master_api_keys.md)
 
+### Disable auto-batching
+
+::: warning
+🚩 This is a CLI flag and does not take any values. Assigning a value will throw an error. 🚩
+:::
+
+**Environment variable**: `MEILI_DISABLE_AUTO_BATCHING`
+**CLI option**: `--disable-auto-batching`
+
+Deactivates auto-batching when provided.
+
+[Learn more about auto-batching.](/learn/core_concepts/documents.md#auto-batching)
+
 ### Disable analytics
 
 ::: warning

learn/core_concepts/documents.md

@@ -124,3 +124,19 @@ Since CSV does not support arrays or nested objects, `cast` cannot be converted
 ::: note
 If you don't specify the data type for an attribute, it will default to `:string`.
 :::
+
+### Auto-batching
+
+Auto-batching combines consecutive document addition requests into a single batch and processes them together. This significantly speeds up the indexing process.
+
+Meilisearch batches document addition requests when they:
+
+- Target the same index
+- Have the same update method (i.e., [POST](/reference/api/documents.md#add-or-replace-documents) or [PUT](/reference/api/documents.md#add-or-update-documents))
+- Are immediately consecutive
+
+Tasks within the same batch share the same values for `startedAt`, `finishedAt`, and `duration`.
+
+If a task fails due to an invalid document, it will be removed from the batch. The rest of the batch will still process normally. If an [`internal`](/reference/api/overview.md#errors) error occurs, the whole batch will fail and all tasks within it will share the same `error` object.
+
+You can deactivate auto-batching using the `--disable-auto-batching` command-line flag or the `MEILI_DISABLE_AUTO_BATCHING` environment variable. This is useful in cases where you want to avoid any potential bugs in the feature or reduce visibility latency. When auto-batching is disabled, the whole queue takes longer to process, but each individual task will be processed earlier (until a certain number of processed tasks).

learn/experimental/auto-batching.md

@@ -51,7 +51,7 @@ These commands launch the **latest stable release** of Meilisearch.
 
 ```bash
 # Fetch the latest version of Meilisearch image from DockerHub
-docker pull getmeili/meilisearch:v0.28
+docker pull getmeili/meilisearch:v0.29
 
 # Launch Meilisearch in development mode with a master key
 docker run -it --rm \
@@ -181,7 +181,9 @@ Meilisearch stores data in the form of discrete records, called [documents](/lea
 Meilisearch currently only accepts data in JSON, NDJSON, and CSV formats. You can read more about this in our [documents guide](/learn/core_concepts/documents.md#dataset-format).
 :::
 
-The previous command added documents from `movies.json` to a new index called `movies`. After adding documents, you should receive a response like this:
+The previous command added documents from `movies.json` to a new index called `movies`.
+
+By default, Meilisearch combines consecutive document requests into a single batch and processes them together. This process is called [auto-batching](/learn/core_concepts/documents.md#auto-batching), and it significantly speeds up indexing. After adding documents, you should receive a response like this:
 
 ```json
 {

learn/experimental/overview.md

@@ -6,6 +6,7 @@ Meilisearch is multilingual, featuring optimized support for:
 - Chinese
 - Japanese
 - Hebrew
+- Thai
 
 We aim to provide global language support, and your feedback helps us move closer to that goal. If you notice inconsistencies in your search results or the way your documents are processed, please [open an issue in the Meilisearch repository](https://github.com/meilisearch/meilisearch/issues/new/choose).
 
@@ -29,6 +30,7 @@ Under the hood, Meilisearch relies on tokenizers that identify the most importan
 - A pipeline specifically tailored for Chinese
 - A pipeline specifically tailored for Japanese
 - A pipeline specifically tailored for Hebrew
+- A pipeline specifically tailored for Thai
 
 ### My language does not use whitespace to separate words. Can I still use Meilisearch?
 

learn/getting_started/quick_start.md

@@ -20,7 +20,7 @@ _Meilisearch helps the Rust community find crates on [crates.meilisearch.com](ht
 - **Blazing fast** (answers < 50 milliseconds): Priority is given to fast answers for a smooth search experience.
 - [Search as you type](/learn/what_is_meilisearch/features.md#search-as-you-type): Results are updated on each keystroke. To make this possible, we use [prefix-search](/learn/advanced/prefix.md#prefix-search).
 - [Typo tolerance](/learn/what_is_meilisearch/features.md#typo-tolerant): Understands typos and misspellings.
-- [Tokenization](/learn/advanced/tokenization.md) in **English**, **Chinese**, and **all languages that uses space as a word divider**.
+- [Tokenization](/learn/advanced/tokenization.md) in **English**, **Chinese**, and **all languages that use space as a word divider**.
 - **Return the whole document**: The entire document is returned upon search.
 - **Highly customizable search and indexing**:
   - [Custom ranking](/learn/core_concepts/relevancy.md): Customize the relevancy of the search engine and the ranking of the search results.