diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 5552a9deb0..3dcc9fe3a4 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -156,25 +156,32 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( var queries []query.Query if options.Keyword != "" { - if options.IsFuzzyKeyword { - fuzziness := 1 - if kl := len(options.Keyword); kl > 3 { - fuzziness = 2 - } else if kl < 2 { - fuzziness = 0 - } - queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ - inner_bleve.MatchQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness), - inner_bleve.MatchQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness), - inner_bleve.MatchQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness), - }...)) - } else { - queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ - inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, 0), - inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, 0), - inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, 0), - }...)) + tokens, err := options.Tokens() + if err != nil { + return nil, err } + q := bleve.NewBooleanQuery() + for _, token := range tokens { + fuzziness := 0 + if token.Fuzzy { + // TODO: replace with "auto" after bleve update + fuzziness = min(len(token.Term)/4, 2) + } + innerQ := bleve.NewDisjunctionQuery( + inner_bleve.MatchPhraseQuery(token.Term, "title", issueIndexerAnalyzer, fuzziness), + inner_bleve.MatchPhraseQuery(token.Term, "content", issueIndexerAnalyzer, fuzziness), + inner_bleve.MatchPhraseQuery(token.Term, "comments", issueIndexerAnalyzer, fuzziness)) + + switch token.Kind { + case internal.BoolOptMust: + q.AddMust(innerQ) + case internal.BoolOptShould: + q.AddShould(innerQ) + case internal.BoolOptNot: + q.AddMustNot(innerQ) + } + } + queries = append(queries, q) } if len(options.RepoIDs) > 0 || options.AllPublic { diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 24e1ac8855..221ae6dd2f 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -23,6 +23,10 @@ const ( // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types esMultiMatchTypeBestFields = "best_fields" esMultiMatchTypePhrasePrefix = "phrase_prefix" + + // fuzziness options + // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/common-options.html#fuzziness + esFuzzyAuto = "AUTO" ) var _ internal.Indexer = &Indexer{} @@ -145,12 +149,30 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query := elastic.NewBoolQuery() if options.Keyword != "" { - searchType := esMultiMatchTypePhrasePrefix - if options.IsFuzzyKeyword { - searchType = esMultiMatchTypeBestFields + q := elastic.NewBoolQuery() + tokens, err := options.Tokens() + if err != nil { + return nil, err } + for _, token := range tokens { + innerQ := elastic.NewMultiMatchQuery(token.Term, "title", "content", "comments") + if token.Fuzzy { + // If the term is not a phrase use fuzziness set to AUTO + innerQ = innerQ.Type(esMultiMatchTypeBestFields).Fuzziness(esFuzzyAuto) + } else { + innerQ = innerQ.Type(esMultiMatchTypePhrasePrefix) + } - query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(searchType)) + switch token.Kind { + case internal.BoolOptMust: + q.Must(innerQ) + case internal.BoolOptShould: + q.Should(innerQ) + case internal.BoolOptNot: + q.MustNot(innerQ) + } + } + query.Must(q) } if len(options.RepoIDs) > 0 { diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index dda2b7a5c1..0751060afc 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -74,8 +74,6 @@ type SearchResult struct { type SearchOptions struct { Keyword string // keyword to search - IsFuzzyKeyword bool // if false the levenshtein distance is 0 - RepoIDs []int64 // repository IDs which the issues belong to AllPublic bool // if include all public repositories diff --git a/modules/indexer/issues/internal/qstring.go b/modules/indexer/issues/internal/qstring.go new file mode 100644 index 0000000000..fdb89b09e9 --- /dev/null +++ b/modules/indexer/issues/internal/qstring.go @@ -0,0 +1,112 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package internal + +import ( + "io" + "strings" +) + +type BoolOpt int + +const ( + BoolOptMust BoolOpt = iota + BoolOptShould + BoolOptNot +) + +type Token struct { + Term string + Kind BoolOpt + Fuzzy bool +} + +type Tokenizer struct { + in *strings.Reader +} + +func (t *Tokenizer) next() (tk Token, err error) { + var ( + sb strings.Builder + r rune + ) + tk.Kind = BoolOptShould + tk.Fuzzy = true + + // skip all leading white space + for { + if r, _, err = t.in.ReadRune(); err == nil && r == ' ' { + //nolint:staticcheck,wastedassign // SA4006 the variable is used after the loop + r, _, err = t.in.ReadRune() + continue + } + break + } + if err != nil { + return tk, err + } + + // check for +/- op, increment to the next rune in both cases + switch r { + case '+': + tk.Kind = BoolOptMust + r, _, err = t.in.ReadRune() + case '-': + tk.Kind = BoolOptNot + r, _, err = t.in.ReadRune() + } + if err != nil { + return tk, err + } + + // parse the string, escaping special characters + for esc := false; err == nil; r, _, err = t.in.ReadRune() { + if esc { + if !strings.ContainsRune("+-\\\"", r) { + sb.WriteRune('\\') + } + sb.WriteRune(r) + esc = false + continue + } + switch r { + case '\\': + esc = true + case '"': + if !tk.Fuzzy { + goto nextEnd + } + tk.Fuzzy = false + case ' ', '\t': + if tk.Fuzzy { + goto nextEnd + } + sb.WriteRune(r) + default: + sb.WriteRune(r) + } + } +nextEnd: + + tk.Term = sb.String() + if err == io.EOF { + err = nil + } // do not consider EOF as an error at the end + return tk, err +} + +// Tokenize the keyword +func (o *SearchOptions) Tokens() (tokens []Token, err error) { + in := strings.NewReader(o.Keyword) + it := Tokenizer{in: in} + + for token, err := it.next(); err == nil; token, err = it.next() { + tokens = append(tokens, token) + } + if err != nil && err != io.EOF { + return nil, err + } + + return tokens, nil +} diff --git a/modules/indexer/issues/internal/qstring_test.go b/modules/indexer/issues/internal/qstring_test.go new file mode 100644 index 0000000000..a911b86e2f --- /dev/null +++ b/modules/indexer/issues/internal/qstring_test.go @@ -0,0 +1,171 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package internal + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testIssueQueryStringOpt struct { + Keyword string + Results []Token +} + +var testOpts = []testIssueQueryStringOpt{ + { + Keyword: "Hello", + Results: []Token{ + { + Term: "Hello", + Fuzzy: true, + Kind: BoolOptShould, + }, + }, + }, + { + Keyword: "Hello World", + Results: []Token{ + { + Term: "Hello", + Fuzzy: true, + Kind: BoolOptShould, + }, + { + Term: "World", + Fuzzy: true, + Kind: BoolOptShould, + }, + }, + }, + { + Keyword: "+Hello +World", + Results: []Token{ + { + Term: "Hello", + Fuzzy: true, + Kind: BoolOptMust, + }, + { + Term: "World", + Fuzzy: true, + Kind: BoolOptMust, + }, + }, + }, + { + Keyword: "+Hello World", + Results: []Token{ + { + Term: "Hello", + Fuzzy: true, + Kind: BoolOptMust, + }, + { + Term: "World", + Fuzzy: true, + Kind: BoolOptShould, + }, + }, + }, + { + Keyword: "+Hello -World", + Results: []Token{ + { + Term: "Hello", + Fuzzy: true, + Kind: BoolOptMust, + }, + { + Term: "World", + Fuzzy: true, + Kind: BoolOptNot, + }, + }, + }, + { + Keyword: "\"Hello World\"", + Results: []Token{ + { + Term: "Hello World", + Fuzzy: false, + Kind: BoolOptShould, + }, + }, + }, + { + Keyword: "+\"Hello World\"", + Results: []Token{ + { + Term: "Hello World", + Fuzzy: false, + Kind: BoolOptMust, + }, + }, + }, + { + Keyword: "-\"Hello World\"", + Results: []Token{ + { + Term: "Hello World", + Fuzzy: false, + Kind: BoolOptNot, + }, + }, + }, + { + Keyword: "\"+Hello -World\"", + Results: []Token{ + { + Term: "+Hello -World", + Fuzzy: false, + Kind: BoolOptShould, + }, + }, + }, + { + Keyword: "\\+Hello", // \+Hello => +Hello + Results: []Token{ + { + Term: "+Hello", + Fuzzy: true, + Kind: BoolOptShould, + }, + }, + }, + { + Keyword: "\\\\Hello", // \\Hello => \Hello + Results: []Token{ + { + Term: "\\Hello", + Fuzzy: true, + Kind: BoolOptShould, + }, + }, + }, + { + Keyword: "\\\"Hello", // \"Hello => "Hello + Results: []Token{ + { + Term: "\"Hello", + Fuzzy: true, + Kind: BoolOptShould, + }, + }, + }, +} + +func TestIssueQueryString(t *testing.T) { + var opt SearchOptions + for _, res := range testOpts { + t.Run(opt.Keyword, func(t *testing.T) { + opt.Keyword = res.Keyword + tokens, err := opt.Tokens() + require.NoError(t, err) + assert.Equal(t, res.Results, tokens) + }) + } +} diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index e8e6a4e7d1..0763cd1297 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -131,6 +131,20 @@ var cases = []*testIndexerCase{ ExpectedIDs: []int64{1002, 1001, 1000}, ExpectedTotal: 3, }, + { + Name: "Keyword Exclude", + ExtraData: []*internal.IndexerData{ + {ID: 1000, Title: "hi hello world"}, + {ID: 1001, Content: "hi hello world"}, + {ID: 1002, Comments: []string{"hello", "hello world"}}, + }, + SearchOptions: &internal.SearchOptions{ + Keyword: "hello world -hi", + SortBy: internal.SortByCreatedDesc, + }, + ExpectedIDs: []int64{1002}, + ExpectedTotal: 1, + }, { Name: "Keyword Fuzzy", ExtraData: []*internal.IndexerData{ @@ -139,9 +153,8 @@ var cases = []*testIndexerCase{ {ID: 1002, Comments: []string{"hi", "hello world"}}, }, SearchOptions: &internal.SearchOptions{ - Keyword: "hello world", - SortBy: internal.SortByCreatedDesc, - IsFuzzyKeyword: true, + Keyword: "hello world", + SortBy: internal.SortByCreatedDesc, }, ExpectedIDs: []int64{1002, 1001, 1000}, ExpectedTotal: 3, diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 7c291198f1..951f3c8bfb 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -232,20 +232,36 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( limit = 1 } - keyword := options.Keyword - if !options.IsFuzzyKeyword { - // to make it non fuzzy ("typo tolerance" in meilisearch terms), we have to quote the keyword(s) - // https://www.meilisearch.com/docs/reference/api/search#phrase-search - keyword = doubleQuoteKeyword(keyword) + var keywords []string + if options.Keyword != "" { + tokens, err := options.Tokens() + if err != nil { + return nil, err + } + for _, token := range tokens { + if !token.Fuzzy { + // to make it a phrase search, we have to quote the keyword(s) + // https://www.meilisearch.com/docs/reference/api/search#phrase-search + token.Term = doubleQuoteKeyword(token.Term) + } + + // internal.BoolOptShould (Default, requires no modifications) + // internal.BoolOptMust (Not supported by meilisearch) + if token.Kind == internal.BoolOptNot { + token.Term = "-" + token.Term + } + keywords = append(keywords, token.Term) + } } - searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(keyword, &meilisearch.SearchRequest{ - Filter: query.Statement(), - Limit: int64(limit), - Offset: int64(skip), - Sort: sortBy, - MatchingStrategy: meilisearch.All, - }) + searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()). + Search(strings.Join(keywords, " "), &meilisearch.SearchRequest{ + Filter: query.Statement(), + Limit: int64(limit), + Offset: int64(skip), + Sort: sortBy, + MatchingStrategy: meilisearch.All, + }) if err != nil { return nil, err } diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 5f9b952119..b4c673ffd7 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -204,8 +204,6 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption opt keyword = "" } - isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) - var mileIDs []int64 if milestoneID > 0 || milestoneID == db.NoConditionID { // -1 to get those issues which have no any milestone assigned mileIDs = []int64{milestoneID} @@ -226,7 +224,7 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption opt IssueIDs: nil, } if keyword != "" { - allIssueIDs, err := issueIDsFromSearch(ctx, keyword, isFuzzy, statsOpts) + allIssueIDs, err := issueIDsFromSearch(ctx, keyword, statsOpts) if err != nil { if issue_indexer.IsAvailable(ctx) { ctx.ServerError("issueIDsFromSearch", err) @@ -294,7 +292,7 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption opt var issues issues_model.IssueList { - ids, err := issueIDsFromSearch(ctx, keyword, isFuzzy, &issues_model.IssuesOptions{ + ids, err := issueIDsFromSearch(ctx, keyword, &issues_model.IssuesOptions{ Paginator: &db.ListOptions{ Page: pager.Paginater.Current(), PageSize: setting.UI.IssuePagingNum, @@ -458,16 +456,16 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption opt ctx.Data["OpenCount"] = issueStats.OpenCount ctx.Data["ClosedCount"] = issueStats.ClosedCount ctx.Data["AllCount"] = issueStats.AllCount - linkStr := "?q=%s&type=%s&sort=%s&state=%s&labels=%s&milestone=%d&project=%d&assignee=%d&poster=%d&fuzzy=%t&archived=%t" + linkStr := "?q=%s&type=%s&sort=%s&state=%s&labels=%s&milestone=%d&project=%d&assignee=%d&poster=%d&archived=%t" ctx.Data["AllStatesLink"] = fmt.Sprintf(linkStr, url.QueryEscape(keyword), url.QueryEscape(viewType), url.QueryEscape(sortType), "all", url.QueryEscape(selectLabels), - milestoneID, projectID, assigneeID, posterID, isFuzzy, archived) + milestoneID, projectID, assigneeID, posterID, archived) ctx.Data["OpenLink"] = fmt.Sprintf(linkStr, url.QueryEscape(keyword), url.QueryEscape(viewType), url.QueryEscape(sortType), "open", url.QueryEscape(selectLabels), - milestoneID, projectID, assigneeID, posterID, isFuzzy, archived) + milestoneID, projectID, assigneeID, posterID, archived) ctx.Data["ClosedLink"] = fmt.Sprintf(linkStr, url.QueryEscape(keyword), url.QueryEscape(viewType), url.QueryEscape(sortType), "closed", url.QueryEscape(selectLabels), - milestoneID, projectID, assigneeID, posterID, isFuzzy, archived) + milestoneID, projectID, assigneeID, posterID, archived) ctx.Data["SelLabelIDs"] = labelIDs ctx.Data["SelectLabels"] = selectLabels ctx.Data["ViewType"] = viewType @@ -476,7 +474,6 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption opt ctx.Data["ProjectID"] = projectID ctx.Data["AssigneeID"] = assigneeID ctx.Data["PosterID"] = posterID - ctx.Data["IsFuzzy"] = isFuzzy ctx.Data["Keyword"] = keyword ctx.Data["IsShowClosed"] = isShowClosed switch { @@ -499,17 +496,12 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption opt pager.AddParam(ctx, "assignee", "AssigneeID") pager.AddParam(ctx, "poster", "PosterID") pager.AddParam(ctx, "archived", "ShowArchivedLabels") - pager.AddParam(ctx, "fuzzy", "IsFuzzy") ctx.Data["Page"] = pager } -func issueIDsFromSearch(ctx *context.Context, keyword string, fuzzy bool, opts *issues_model.IssuesOptions) ([]int64, error) { - ids, _, err := issue_indexer.SearchIssues(ctx, issue_indexer.ToSearchOptions(keyword, opts).Copy( - func(o *issue_indexer.SearchOptions) { - o.IsFuzzyKeyword = fuzzy - }, - )) +func issueIDsFromSearch(ctx *context.Context, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) { + ids, _, err := issue_indexer.SearchIssues(ctx, issue_indexer.ToSearchOptions(keyword, opts)) if err != nil { return nil, fmt.Errorf("SearchIssues: %w", err) } diff --git a/routers/web/user/home.go b/routers/web/user/home.go index d67af29071..2c98da1383 100644 --- a/routers/web/user/home.go +++ b/routers/web/user/home.go @@ -463,8 +463,6 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { User: ctx.Doer, } - isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) - // Search all repositories which // // As user: @@ -594,9 +592,7 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { // USING FINAL STATE OF opts FOR A QUERY. var issues issues_model.IssueList { - issueIDs, _, err := issue_indexer.SearchIssues(ctx, issue_indexer.ToSearchOptions(keyword, opts).Copy( - func(o *issue_indexer.SearchOptions) { o.IsFuzzyKeyword = isFuzzy }, - )) + issueIDs, _, err := issue_indexer.SearchIssues(ctx, issue_indexer.ToSearchOptions(keyword, opts)) if err != nil { ctx.ServerError("issueIDsFromSearch", err) return @@ -622,9 +618,7 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { // ------------------------------- // Fill stats to post to ctx.Data. // ------------------------------- - issueStats, err := getUserIssueStats(ctx, ctxUser, filterMode, issue_indexer.ToSearchOptions(keyword, opts).Copy( - func(o *issue_indexer.SearchOptions) { o.IsFuzzyKeyword = isFuzzy }, - )) + issueStats, err := getUserIssueStats(ctx, ctxUser, filterMode, issue_indexer.ToSearchOptions(keyword, opts)) if err != nil { ctx.ServerError("getUserIssueStats", err) return @@ -679,7 +673,6 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { ctx.Data["IsShowClosed"] = isShowClosed ctx.Data["SelectLabels"] = selectedLabels ctx.Data["PageIsOrgIssues"] = org != nil - ctx.Data["IsFuzzy"] = isFuzzy if isShowClosed { ctx.Data["State"] = "closed" @@ -695,7 +688,6 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { pager.AddParam(ctx, "labels", "SelectLabels") pager.AddParam(ctx, "milestone", "MilestoneID") pager.AddParam(ctx, "assignee", "AssigneeID") - pager.AddParam(ctx, "fuzzy", "IsFuzzy") ctx.Data["Page"] = pager ctx.HTML(http.StatusOK, tplIssues) diff --git a/templates/repo/issue/filter_list.tmpl b/templates/repo/issue/filter_list.tmpl index 0e1d8f8036..f59c6d67ce 100644 --- a/templates/repo/issue/filter_list.tmpl +++ b/templates/repo/issue/filter_list.tmpl @@ -14,13 +14,13 @@
- {{ctx.Locale.Tr "repo.issues.filter_milestone_all"}} - {{ctx.Locale.Tr "repo.issues.filter_milestone_none"}} + {{ctx.Locale.Tr "repo.issues.filter_milestone_all"}} + {{ctx.Locale.Tr "repo.issues.filter_milestone_none"}} {{if .OpenMilestones}}