Make token deduplication optional for the repo indexer

custom/conf/app.ini.sample

@@ -296,6 +296,7 @@ REPO_INDEXER_ENABLED = false
 REPO_INDEXER_PATH = indexers/repos.bleve
 UPDATE_BUFFER_LEN = 20
 MAX_FILE_SIZE = 1048576
+REPO_INDEXER_DEDUPLICATE = true
 
 [admin]
 ; Disallow regular (non-admin) users from creating organizations.

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -180,6 +180,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
 - `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
 - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
+- `REPO_INDEXER_DEDUPLICATE`: **true** Deduplicate terms from the index, reducing the index size while limiting the search capabilities.
 
 ## Security (`security`)
 

modules/indexer/repo.go

@@ -103,14 +103,18 @@ func createRepoIndexer(path string, latestVersion int) error {
 	textFieldMapping.IncludeInAll = false
 	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
 
+	tokenFilters := []string{unicodeNormalizeName, lowercase.Name}
+	if setting.Indexer.RepoIndexerDeduplicate {
+		tokenFilters = append(tokenFilters, unique.Name)
+	}
 	mapping := bleve.NewIndexMapping()
 	if err = addUnicodeNormalizeTokenFilter(mapping); err != nil {
 		return err
 	} else if err = mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
 		"type":          custom.Name,
 		"char_filters":  []string{},
 		"tokenizer":     unicode.Name,
-		"token_filters": []string{unicodeNormalizeName, lowercase.Name, unique.Name},
+		"token_filters": tokenFilters,
 	}); err != nil {
 		return err
 	}

modules/setting/indexer.go

@@ -19,16 +19,17 @@ const (
 var (
 	// Indexer settings
 	Indexer = struct {
-		IssueType             string
-		IssuePath             string
-		RepoIndexerEnabled    bool
-		RepoPath              string
-		UpdateQueueLength     int
-		MaxIndexerFileSize    int64
-		IssueQueueType        string
-		IssueQueueDir         string
-		IssueQueueConnStr     string
-		IssueQueueBatchNumber int
+		IssueType              string
+		IssuePath              string
+		RepoIndexerEnabled     bool
+		RepoPath               string
+		UpdateQueueLength      int
+		MaxIndexerFileSize     int64
+		RepoIndexerDeduplicate bool
+		IssueQueueType         string
+		IssueQueueDir          string
+		IssueQueueConnStr      string
+		IssueQueueBatchNumber  int
 	}{
 		IssueType:             "bleve",
 		IssuePath:             "indexers/issues.bleve",
@@ -51,6 +52,7 @@ func newIndexerService() {
 	if !filepath.IsAbs(Indexer.RepoPath) {
 		Indexer.RepoPath = path.Join(AppWorkPath, Indexer.RepoPath)
 	}
+	Indexer.RepoIndexerDeduplicate = sec.Key("REPO_INDEXER_DEDUPLICATE").MustBool(true)
 	Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
 	Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024)
 	Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType)