Skip to content

Commit 1262ff6

Browse files
65436543
authored
Refactor code_indexer to use an SearchOptions struct for PerformSearch (#29724)
similar to how it's already done for the issue_indexer --- *Sponsored by Kithara Software GmbH*
1 parent e0ea381 commit 1262ff6

File tree

9 files changed

+86
-39
lines changed

9 files changed

+86
-39
lines changed

‎modules/indexer/code/bleve/bleve.go

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro
142142
return err
143143
}
144144
if size, err = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64); err != nil {
145-
return fmt.Errorf("Misformatted git cat-file output: %w", err)
145+
return fmt.Errorf("misformatted git cat-file output: %w", err)
146146
}
147147
}
148148

@@ -233,26 +233,26 @@ func (b *Indexer) Delete(_ context.Context, repoID int64) error {
233233

234234
// Search searches for files in the specified repo.
235235
// Returns the matching file-paths
236-
func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
236+
func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
237237
var (
238238
indexerQuery query.Query
239239
keywordQuery query.Query
240240
)
241241

242-
if isFuzzy {
243-
phraseQuery := bleve.NewMatchPhraseQuery(keyword)
242+
if opts.IsKeywordFuzzy {
243+
phraseQuery := bleve.NewMatchPhraseQuery(opts.Keyword)
244244
phraseQuery.FieldVal = "Content"
245245
phraseQuery.Analyzer = repoIndexerAnalyzer
246246
keywordQuery = phraseQuery
247247
} else {
248-
prefixQuery := bleve.NewPrefixQuery(keyword)
248+
prefixQuery := bleve.NewPrefixQuery(opts.Keyword)
249249
prefixQuery.FieldVal = "Content"
250250
keywordQuery = prefixQuery
251251
}
252252

253-
if len(repoIDs) > 0 {
254-
repoQueries := make([]query.Query, 0, len(repoIDs))
255-
for _, repoID := range repoIDs {
253+
if len(opts.RepoIDs) > 0 {
254+
repoQueries := make([]query.Query, 0, len(opts.RepoIDs))
255+
for _, repoID := range opts.RepoIDs {
256256
repoQueries = append(repoQueries, inner_bleve.NumericEqualityQuery(repoID, "RepoID"))
257257
}
258258

@@ -266,8 +266,8 @@ func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword
266266

267267
// Save for reuse without language filter
268268
facetQuery := indexerQuery
269-
if len(language) > 0 {
270-
languageQuery := bleve.NewMatchQuery(language)
269+
if len(opts.Language) > 0 {
270+
languageQuery := bleve.NewMatchQuery(opts.Language)
271271
languageQuery.FieldVal = "Language"
272272
languageQuery.Analyzer = analyzer_keyword.Name
273273

@@ -277,12 +277,12 @@ func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword
277277
)
278278
}
279279

280-
from := (page - 1) * pageSize
280+
from, pageSize := opts.GetSkipTake()
281281
searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
282282
searchRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"}
283283
searchRequest.IncludeLocations = true
284284

285-
if len(language) == 0 {
285+
if len(opts.Language) == 0 {
286286
searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
287287
}
288288

@@ -326,7 +326,7 @@ func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword
326326
}
327327

328328
searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10)
329-
if len(language) > 0 {
329+
if len(opts.Language) > 0 {
330330
// Use separate query to go get all language counts
331331
facetRequest := bleve.NewSearchRequestOptions(facetQuery, 1, 0, false)
332332
facetRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"}

‎modules/indexer/code/elasticsearch/elasticsearch.go

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -281,35 +281,31 @@ func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLan
281281
}
282282

283283
// Search searches for codes and language stats by given conditions.
284-
func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
284+
func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
285285
searchType := esMultiMatchTypePhrasePrefix
286-
if isFuzzy {
286+
if opts.IsKeywordFuzzy {
287287
searchType = esMultiMatchTypeBestFields
288288
}
289289

290-
kwQuery := elastic.NewMultiMatchQuery(keyword, "content").Type(searchType)
290+
kwQuery := elastic.NewMultiMatchQuery(opts.Keyword, "content").Type(searchType)
291291
query := elastic.NewBoolQuery()
292292
query = query.Must(kwQuery)
293-
if len(repoIDs) > 0 {
294-
repoStrs := make([]any, 0, len(repoIDs))
295-
for _, repoID := range repoIDs {
293+
if len(opts.RepoIDs) > 0 {
294+
repoStrs := make([]any, 0, len(opts.RepoIDs))
295+
for _, repoID := range opts.RepoIDs {
296296
repoStrs = append(repoStrs, repoID)
297297
}
298298
repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...)
299299
query = query.Must(repoQuery)
300300
}
301301

302302
var (
303-
start int
304-
kw = "<em>" + keyword + "</em>"
305-
aggregation = elastic.NewTermsAggregation().Field("language").Size(10).OrderByCountDesc()
303+
start, pageSize = opts.GetSkipTake()
304+
kw = "<em>" + opts.Keyword + "</em>"
305+
aggregation = elastic.NewTermsAggregation().Field("language").Size(10).OrderByCountDesc()
306306
)
307307

308-
if page > 0 {
309-
start = (page - 1) * pageSize
310-
}
311-
312-
if len(language) == 0 {
308+
if len(opts.Language) == 0 {
313309
searchResult, err := b.inner.Client.Search().
314310
Index(b.inner.VersionedIndexName()).
315311
Aggregation("language", aggregation).
@@ -330,7 +326,7 @@ func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword
330326
return convertResult(searchResult, kw, pageSize)
331327
}
332328

333-
langQuery := elastic.NewMatchQuery("language", language)
329+
langQuery := elastic.NewMatchQuery("language", opts.Language)
334330
countResult, err := b.inner.Client.Search().
335331
Index(b.inner.VersionedIndexName()).
336332
Aggregation("language", aggregation).

‎modules/indexer/code/git.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision s
3232

3333
needGenesis := len(status.CommitSha) == 0
3434
if !needGenesis {
35-
hasAncestorCmd := git.NewCommand(ctx, "merge-base").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision)
35+
hasAncestorCmd := git.NewCommand(ctx, "merge-base").AddDynamicArguments(status.CommitSha, revision)
3636
stdout, _, _ := hasAncestorCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
3737
needGenesis = len(stdout) == 0
3838
}

‎modules/indexer/code/indexer_test.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"os"
99
"testing"
1010

11+
"code.gitea.io/gitea/models/db"
1112
"code.gitea.io/gitea/models/unittest"
1213
"code.gitea.io/gitea/modules/git"
1314
"code.gitea.io/gitea/modules/indexer/code/bleve"
@@ -70,7 +71,15 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
7071

7172
for _, kw := range keywords {
7273
t.Run(kw.Keyword, func(t *testing.T) {
73-
total, res, langs, err := indexer.Search(context.TODO(), kw.RepoIDs, "", kw.Keyword, 1, 10, true)
74+
total, res, langs, err := indexer.Search(context.TODO(), &internal.SearchOptions{
75+
RepoIDs: kw.RepoIDs,
76+
Keyword: kw.Keyword,
77+
Paginator: &db.ListOptions{
78+
Page: 1,
79+
PageSize: 10,
80+
},
81+
IsKeywordFuzzy: true,
82+
})
7483
assert.NoError(t, err)
7584
assert.Len(t, kw.IDs, int(total))
7685
assert.Len(t, langs, kw.Langs)

‎modules/indexer/code/internal/indexer.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"context"
88
"fmt"
99

10+
"code.gitea.io/gitea/models/db"
1011
repo_model "code.gitea.io/gitea/models/repo"
1112
"code.gitea.io/gitea/modules/indexer/internal"
1213
)
@@ -16,7 +17,17 @@ type Indexer interface {
1617
internal.Indexer
1718
Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error
1819
Delete(ctx context.Context, repoID int64) error
19-
Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
20+
Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error)
21+
}
22+
23+
type SearchOptions struct {
24+
RepoIDs []int64
25+
Keyword string
26+
Language string
27+
28+
IsKeywordFuzzy bool
29+
30+
db.Paginator
2031
}
2132

2233
// NewDummyIndexer returns a dummy indexer
@@ -38,6 +49,6 @@ func (d *dummyIndexer) Delete(ctx context.Context, repoID int64) error {
3849
return fmt.Errorf("indexer is not ready")
3950
}
4051

41-
func (d *dummyIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
52+
func (d *dummyIndexer) Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) {
4253
return 0, nil, nil, fmt.Errorf("indexer is not ready")
4354
}

‎modules/indexer/code/search.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ type ResultLine struct {
3232

3333
type SearchResultLanguages = internal.SearchResultLanguages
3434

35+
type SearchOptions = internal.SearchOptions
36+
3537
func indices(content string, selectionStartIndex, selectionEndIndex int) (int, int) {
3638
startIndex := selectionStartIndex
3739
numLinesBefore := 0
@@ -125,12 +127,12 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res
125127

126128
// PerformSearch perform a search on a repository
127129
// if isFuzzy is true set the Damerau-Levenshtein distance from 0 to 2
128-
func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int, []*Result, []*internal.SearchResultLanguages, error) {
129-
if len(keyword) == 0 {
130+
func PerformSearch(ctx context.Context, opts *SearchOptions) (int, []*Result, []*SearchResultLanguages, error) {
131+
if opts == nil || len(opts.Keyword) == 0 {
130132
return 0, nil, nil, nil
131133
}
132134

133-
total, results, resultLanguages, err := (*globalIndexer.Load()).Search(ctx, repoIDs, language, keyword, page, pageSize, isFuzzy)
135+
total, results, resultLanguages, err := (*globalIndexer.Load()).Search(ctx, opts)
134136
if err != nil {
135137
return 0, nil, nil, err
136138
}

‎routers/web/explore/code.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package explore
66
import (
77
"net/http"
88

9+
"code.gitea.io/gitea/models/db"
910
repo_model "code.gitea.io/gitea/models/repo"
1011
"code.gitea.io/gitea/modules/base"
1112
code_indexer "code.gitea.io/gitea/modules/indexer/code"
@@ -76,7 +77,16 @@ func Code(ctx *context.Context) {
7677
)
7778

7879
if (len(repoIDs) > 0) || isAdmin {
79-
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isFuzzy)
80+
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
81+
RepoIDs: repoIDs,
82+
Keyword: keyword,
83+
IsKeywordFuzzy: isFuzzy,
84+
Language: language,
85+
Paginator: &db.ListOptions{
86+
Page: page,
87+
PageSize: setting.UI.RepoSearchPagingNum,
88+
},
89+
})
8090
if err != nil {
8191
if code_indexer.IsAvailable(ctx) {
8292
ctx.ServerError("SearchResults", err)

‎routers/web/repo/search.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package repo
66
import (
77
"net/http"
88

9+
"code.gitea.io/gitea/models/db"
910
"code.gitea.io/gitea/modules/base"
1011
code_indexer "code.gitea.io/gitea/modules/indexer/code"
1112
"code.gitea.io/gitea/modules/setting"
@@ -41,8 +42,16 @@ func Search(ctx *context.Context) {
4142
page = 1
4243
}
4344

44-
total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch(ctx, []int64{ctx.Repo.Repository.ID},
45-
language, keyword, page, setting.UI.RepoSearchPagingNum, isFuzzy)
45+
total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
46+
RepoIDs: []int64{ctx.Repo.Repository.ID},
47+
Keyword: keyword,
48+
IsKeywordFuzzy: isFuzzy,
49+
Language: language,
50+
Paginator: &db.ListOptions{
51+
Page: page,
52+
PageSize: setting.UI.RepoSearchPagingNum,
53+
},
54+
})
4655
if err != nil {
4756
if code_indexer.IsAvailable(ctx) {
4857
ctx.ServerError("SearchResults", err)

‎routers/web/user/code.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package user
66
import (
77
"net/http"
88

9+
"code.gitea.io/gitea/models/db"
910
repo_model "code.gitea.io/gitea/models/repo"
1011
"code.gitea.io/gitea/modules/base"
1112
code_indexer "code.gitea.io/gitea/modules/indexer/code"
@@ -74,7 +75,16 @@ func CodeSearch(ctx *context.Context) {
7475
)
7576

7677
if len(repoIDs) > 0 {
77-
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isFuzzy)
78+
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
79+
RepoIDs: repoIDs,
80+
Keyword: keyword,
81+
IsKeywordFuzzy: isFuzzy,
82+
Language: language,
83+
Paginator: &db.ListOptions{
84+
Page: page,
85+
PageSize: setting.UI.RepoSearchPagingNum,
86+
},
87+
})
7888
if err != nil {
7989
if code_indexer.IsAvailable(ctx) {
8090
ctx.ServerError("SearchResults", err)

0 commit comments

Comments
 (0)