feat(code search): replace fuzzy search with union search for indexer (#6947)

Fuzzy searching for code has been known to be problematic #5264 and in my personal opinion isn't very useful.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6947
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
Co-authored-by: Shiny Nematoda <snematoda.751k2@aleeas.com>
Co-committed-by: Shiny Nematoda <snematoda.751k2@aleeas.com>
This commit is contained in:
Shiny Nematoda 2025-03-11 21:22:51 +00:00 committed by Gusted
parent cb46a036aa
commit 3816db68aa
10 changed files with 105 additions and 86 deletions

View file

@ -28,10 +28,10 @@ type GrepResult struct {
HighlightedRanges [][3]int HighlightedRanges [][3]int
} }
type grepMode int type GrepMode int
const ( const (
FixedGrepMode grepMode = iota FixedGrepMode GrepMode = iota
FixedAnyGrepMode FixedAnyGrepMode
RegExpGrepMode RegExpGrepMode
) )
@ -43,7 +43,7 @@ type GrepOptions struct {
MaxResultLimit int MaxResultLimit int
MatchesPerFile int // >= git 2.38 MatchesPerFile int // >= git 2.38
ContextLineNumber int ContextLineNumber int
Mode grepMode Mode GrepMode
Filename string Filename string
} }

View file

@ -40,10 +40,6 @@ import (
const ( const (
unicodeNormalizeName = "unicodeNormalize" unicodeNormalizeName = "unicodeNormalize"
maxBatchSize = 16 maxBatchSize = 16
// fuzzyDenominator determines the levenshtein distance per each character of a keyword
fuzzyDenominator = 4
// see https://github.com/blevesearch/bleve/issues/1563#issuecomment-786822311
maxFuzziness = 2
) )
func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error { func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
@ -260,12 +256,14 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
keywordQuery query.Query keywordQuery query.Query
) )
phraseQuery := bleve.NewMatchPhraseQuery(opts.Keyword) if opts.Mode == internal.CodeSearchModeUnion {
phraseQuery.FieldVal = "Content" query := bleve.NewDisjunctionQuery()
phraseQuery.Analyzer = repoIndexerAnalyzer for _, field := range strings.Fields(opts.Keyword) {
keywordQuery = phraseQuery query.AddQuery(inner_bleve.MatchPhraseQuery(field, "Content", repoIndexerAnalyzer, 0))
if opts.IsKeywordFuzzy { }
phraseQuery.Fuzziness = min(maxFuzziness, len(opts.Keyword)/fuzzyDenominator) keywordQuery = query
} else {
keywordQuery = inner_bleve.MatchPhraseQuery(opts.Keyword, "Content", repoIndexerAnalyzer, 0)
} }
if len(opts.RepoIDs) > 0 { if len(opts.RepoIDs) > 0 {
@ -325,13 +323,16 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
for i, hit := range result.Hits { for i, hit := range result.Hits {
startIndex, endIndex := -1, -1 startIndex, endIndex := -1, -1
for _, locations := range hit.Locations["Content"] { for _, locations := range hit.Locations["Content"] {
if startIndex != -1 && endIndex != -1 {
break
}
location := locations[0] location := locations[0]
locationStart := int(location.Start) locationStart := int(location.Start)
locationEnd := int(location.End) locationEnd := int(location.End)
if startIndex < 0 || locationStart < startIndex { if startIndex < 0 || locationStart < startIndex {
startIndex = locationStart startIndex = locationStart
} }
if endIndex < 0 || locationEnd > endIndex { if endIndex < 0 && locationEnd > endIndex {
endIndex = locationEnd endIndex = locationEnd
} }
} }

View file

@ -33,8 +33,8 @@ const (
esRepoIndexerLatestVersion = 2 esRepoIndexerLatestVersion = 2
// multi-match-types, currently only 2 types are used // multi-match-types, currently only 2 types are used
// Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
esMultiMatchTypeBestFields = "best_fields" esMultiMatchTypeBestFields = "best_fields"
esMultiMatchTypePhrasePrefix = "phrase_prefix" esMultiMatchTypePhrase = "phrase"
) )
var _ internal.Indexer = &Indexer{} var _ internal.Indexer = &Indexer{}
@ -334,8 +334,8 @@ func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLan
// Search searches for codes and language stats by given conditions. // Search searches for codes and language stats by given conditions.
func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
searchType := esMultiMatchTypePhrasePrefix searchType := esMultiMatchTypePhrase
if opts.IsKeywordFuzzy { if opts.Mode == internal.CodeSearchModeUnion {
searchType = esMultiMatchTypeBestFields searchType = esMultiMatchTypeBestFields
} }

View file

@ -100,8 +100,8 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
Page: 1, Page: 1,
PageSize: 10, PageSize: 10,
}, },
Filename: kw.Filename, Filename: kw.Filename,
IsKeywordFuzzy: true, Mode: SearchModeUnion,
}) })
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, kw.IDs, int(total)) assert.Len(t, kw.IDs, int(total))

View file

@ -20,13 +20,27 @@ type Indexer interface {
Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error)
} }
type CodeSearchMode int
const (
CodeSearchModeExact CodeSearchMode = iota
CodeSearchModeUnion
)
func (mode CodeSearchMode) String() string {
if mode == CodeSearchModeUnion {
return "union"
}
return "exact"
}
type SearchOptions struct { type SearchOptions struct {
RepoIDs []int64 RepoIDs []int64
Keyword string Keyword string
Language string Language string
Filename string Filename string
IsKeywordFuzzy bool Mode CodeSearchMode
db.Paginator db.Paginator
} }

View file

@ -35,7 +35,14 @@ type SearchResultLanguages = internal.SearchResultLanguages
type SearchOptions = internal.SearchOptions type SearchOptions = internal.SearchOptions
var CodeSearchOptions = [2]string{"exact", "fuzzy"} var CodeSearchOptions = [2]string{"exact", "union"}
type SearchMode = internal.CodeSearchMode
const (
SearchModeExact = internal.CodeSearchModeExact
SearchModeUnion = internal.CodeSearchModeUnion
)
func indices(content string, selectionStartIndex, selectionEndIndex int) (int, int) { func indices(content string, selectionStartIndex, selectionEndIndex int) (int, int) {
startIndex := selectionStartIndex startIndex := selectionStartIndex
@ -206,7 +213,6 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res
} }
// PerformSearch perform a search on a repository // PerformSearch perform a search on a repository
// if isFuzzy is true set the Damerau-Levenshtein distance from 0 to 2
func PerformSearch(ctx context.Context, opts *SearchOptions) (int, []*Result, []*SearchResultLanguages, error) { func PerformSearch(ctx context.Context, opts *SearchOptions) (int, []*Result, []*SearchResultLanguages, error) {
if opts == nil || len(opts.Keyword) == 0 { if opts == nil || len(opts.Keyword) == 0 {
return 0, nil, nil, nil return 0, nil, nil, nil

View file

@ -37,19 +37,17 @@ func Code(ctx *context.Context) {
keyword := ctx.FormTrim("q") keyword := ctx.FormTrim("q")
path := ctx.FormTrim("path") path := ctx.FormTrim("path")
isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) mode := code_indexer.SearchModeExact
if mode := ctx.FormTrim("mode"); len(mode) > 0 { if m := ctx.FormTrim("mode"); m == "union" ||
isFuzzy = mode == "fuzzy" m == "fuzzy" ||
ctx.FormBool("fuzzy") {
mode = code_indexer.SearchModeUnion
} }
ctx.Data["Keyword"] = keyword ctx.Data["Keyword"] = keyword
ctx.Data["Language"] = language ctx.Data["Language"] = language
ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"} ctx.Data["CodeSearchOptions"] = code_indexer.CodeSearchOptions
if isFuzzy { ctx.Data["CodeSearchMode"] = mode.String()
ctx.Data["CodeSearchMode"] = "fuzzy"
} else {
ctx.Data["CodeSearchMode"] = "exact"
}
ctx.Data["PageIsViewCode"] = true ctx.Data["PageIsViewCode"] = true
if keyword == "" { if keyword == "" {
@ -88,11 +86,11 @@ func Code(ctx *context.Context) {
if (len(repoIDs) > 0) || isAdmin { if (len(repoIDs) > 0) || isAdmin {
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{ total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
RepoIDs: repoIDs, RepoIDs: repoIDs,
Keyword: keyword, Keyword: keyword,
IsKeywordFuzzy: isFuzzy, Mode: mode,
Language: language, Language: language,
Filename: path, Filename: path,
Paginator: &db.ListOptions{ Paginator: &db.ListOptions{
Page: page, Page: page,
PageSize: setting.UI.RepoSearchPagingNum, PageSize: setting.UI.RepoSearchPagingNum,

View file

@ -21,14 +21,14 @@ type searchMode int
const ( const (
ExactSearchMode searchMode = iota ExactSearchMode searchMode = iota
FuzzySearchMode UnionSearchMode
RegExpSearchMode RegExpSearchMode
) )
func searchModeFromString(s string) searchMode { func searchModeFromString(s string) searchMode {
switch s { switch s {
case "fuzzy", "union": case "fuzzy", "union":
return FuzzySearchMode return UnionSearchMode
case "regexp": case "regexp":
return RegExpSearchMode return RegExpSearchMode
default: default:
@ -40,8 +40,8 @@ func (m searchMode) String() string {
switch m { switch m {
case ExactSearchMode: case ExactSearchMode:
return "exact" return "exact"
case FuzzySearchMode: case UnionSearchMode:
return "fuzzy" return "union"
case RegExpSearchMode: case RegExpSearchMode:
return "regexp" return "regexp"
default: default:
@ -49,6 +49,24 @@ func (m searchMode) String() string {
} }
} }
func (m searchMode) ToIndexer() code_indexer.SearchMode {
if m == ExactSearchMode {
return code_indexer.SearchModeExact
}
return code_indexer.SearchModeUnion
}
func (m searchMode) ToGitGrep() git.GrepMode {
switch m {
case RegExpSearchMode:
return git.RegExpGrepMode
case UnionSearchMode:
return git.FixedAnyGrepMode
default:
return git.FixedGrepMode
}
}
// Search render repository search page // Search render repository search page
func Search(ctx *context.Context) { func Search(ctx *context.Context) {
language := ctx.FormTrim("l") language := ctx.FormTrim("l")
@ -59,7 +77,7 @@ func Search(ctx *context.Context) {
if modeStr := ctx.FormString("mode"); len(modeStr) > 0 { if modeStr := ctx.FormString("mode"); len(modeStr) > 0 {
mode = searchModeFromString(modeStr) mode = searchModeFromString(modeStr)
} else if ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) { // for backward compatibility in links } else if ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) { // for backward compatibility in links
mode = FuzzySearchMode mode = UnionSearchMode
} }
ctx.Data["Keyword"] = keyword ctx.Data["Keyword"] = keyword
@ -90,11 +108,11 @@ func Search(ctx *context.Context) {
if setting.Indexer.RepoIndexerEnabled { if setting.Indexer.RepoIndexerEnabled {
var err error var err error
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{ total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
RepoIDs: []int64{ctx.Repo.Repository.ID}, RepoIDs: []int64{ctx.Repo.Repository.ID},
Keyword: keyword, Keyword: keyword,
IsKeywordFuzzy: mode == FuzzySearchMode, Mode: mode.ToIndexer(),
Language: language, Language: language,
Filename: path, Filename: path,
Paginator: &db.ListOptions{ Paginator: &db.ListOptions{
Page: page, Page: page,
PageSize: setting.UI.RepoSearchPagingNum, PageSize: setting.UI.RepoSearchPagingNum,
@ -110,19 +128,12 @@ func Search(ctx *context.Context) {
ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable(ctx) ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable(ctx)
} }
} else { } else {
grepOpt := git.GrepOptions{ res, err := git.GrepSearch(ctx, ctx.Repo.GitRepo, keyword, git.GrepOptions{
ContextLineNumber: 1, ContextLineNumber: 1,
RefName: ctx.Repo.RefName, RefName: ctx.Repo.RefName,
Filename: path, Filename: path,
} Mode: mode.ToGitGrep(),
switch mode { })
case FuzzySearchMode:
grepOpt.Mode = git.FixedAnyGrepMode
ctx.Data["CodeSearchMode"] = "union"
case RegExpSearchMode:
grepOpt.Mode = git.RegExpGrepMode
}
res, err := git.GrepSearch(ctx, ctx.Repo.GitRepo, keyword, grepOpt)
if err != nil { if err != nil {
ctx.ServerError("GrepSearch", err) ctx.ServerError("GrepSearch", err)
return return

View file

@ -41,19 +41,17 @@ func CodeSearch(ctx *context.Context) {
keyword := ctx.FormTrim("q") keyword := ctx.FormTrim("q")
path := ctx.FormTrim("path") path := ctx.FormTrim("path")
isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) mode := code_indexer.SearchModeExact
if mode := ctx.FormTrim("mode"); len(mode) > 0 { if m := ctx.FormTrim("mode"); m == "union" ||
isFuzzy = mode == "fuzzy" m == "fuzzy" ||
ctx.FormBool("fuzzy") {
mode = code_indexer.SearchModeUnion
} }
ctx.Data["Keyword"] = keyword ctx.Data["Keyword"] = keyword
ctx.Data["Language"] = language ctx.Data["Language"] = language
ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"} ctx.Data["CodeSearchOptions"] = code_indexer.CodeSearchOptions
if isFuzzy { ctx.Data["CodeSearchMode"] = mode.String()
ctx.Data["CodeSearchMode"] = "fuzzy"
} else {
ctx.Data["CodeSearchMode"] = "exact"
}
ctx.Data["IsCodePage"] = true ctx.Data["IsCodePage"] = true
if keyword == "" { if keyword == "" {
@ -85,11 +83,11 @@ func CodeSearch(ctx *context.Context) {
if len(repoIDs) > 0 { if len(repoIDs) > 0 {
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{ total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
RepoIDs: repoIDs, RepoIDs: repoIDs,
Keyword: keyword, Keyword: keyword,
IsKeywordFuzzy: isFuzzy, Mode: mode,
Language: language, Language: language,
Filename: path, Filename: path,
Paginator: &db.ListOptions{ Paginator: &db.ListOptions{
Page: page, Page: page,
PageSize: setting.UI.RepoSearchPagingNum, PageSize: setting.UI.RepoSearchPagingNum,

View file

@ -82,19 +82,10 @@ func testSearchRepo(t *testing.T, indexer bool) {
testSearch(t, "/user2/glob/search?q=loren&page=1", []string{"a.txt"}, indexer) testSearch(t, "/user2/glob/search?q=loren&page=1", []string{"a.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=loren&page=1&mode=exact", []string{"a.txt"}, indexer) testSearch(t, "/user2/glob/search?q=loren&page=1&mode=exact", []string{"a.txt"}, indexer)
if indexer { // union search: Union/OR of all the keywords
// fuzzy search: matches both file3 (x/b.txt) and file1 (a.txt) testSearch(t, "/user2/glob/search?q=file3+file1&mode=union&page=1", []string{"a.txt", "x/b.txt"}, indexer)
// when indexer is enabled testSearch(t, "/user2/glob/search?q=file4&mode=union&page=1", []string{}, indexer)
testSearch(t, "/user2/glob/search?q=file3&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer) testSearch(t, "/user2/glob/search?q=file5&mode=union&page=1", []string{}, indexer)
testSearch(t, "/user2/glob/search?q=file4&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file5&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
} else {
// fuzzy search: Union/OR of all the keywords
// when indexer is disabled
testSearch(t, "/user2/glob/search?q=file3+file1&mode=union&page=1", []string{"a.txt", "x/b.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file4&mode=union&page=1", []string{}, indexer)
testSearch(t, "/user2/glob/search?q=file5&mode=union&page=1", []string{}, indexer)
}
testSearch(t, "/user2/glob/search?q=file3&page=1&mode=exact", []string{"x/b.txt"}, indexer) testSearch(t, "/user2/glob/search?q=file3&page=1&mode=exact", []string{"x/b.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file4&page=1&mode=exact", []string{}, indexer) testSearch(t, "/user2/glob/search?q=file4&page=1&mode=exact", []string{}, indexer)
@ -121,11 +112,11 @@ func testSearch(t *testing.T, url string, expected []string, indexer bool) {
}) })
if indexer { if indexer {
assert.EqualValues(t, []string{"exact", "fuzzy"}, dropdownOptions) assert.EqualValues(t, []string{"exact", "union"}, dropdownOptions)
} else { } else {
assert.EqualValues(t, []string{"exact", "union", "regexp"}, dropdownOptions) assert.EqualValues(t, []string{"exact", "union", "regexp"}, dropdownOptions)
} }
filenames := resultFilenames(t, doc) filenames := resultFilenames(t, doc)
assert.EqualValues(t, expected, filenames) assert.ElementsMatch(t, expected, filenames)
} }