femsci/forgejo
femsci/forgejo
1
0
Fork 0
forked from NYANDEV/forgejo
Code Pull requests Activity

Add open/closed field support for issue index (#25708)

Browse source 
A couple of notes:
* Future changes should refactor arguments into a struct
* This filtering only is supported by meilisearch right now
* Issue index number is bumped which will cause a re-index
This commit is contained in:
techknowlogick 2023-07-07 13:10:13 -04:00 committed by GitHub
parent 7586b5815a
commit cb01b8691d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 51 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
modules/indexer/issues/bleve/bleve.go
View file

@ -138,7 +138,7 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error {
// Search searches for issues by given conditions. // Search searches for issues by given conditions.
// Returns the matching issue IDs // Returns the matching issue IDs
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) { func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) {
var repoQueriesP []*query.NumericRangeQuery var repoQueriesP []*query.NumericRangeQuery
for _, repoID := range repoIDs { for _, repoID := range repoIDs {
repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "repo_id")) repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "repo_id"))

2
modules/indexer/issues/bleve/bleve_test.go
View file

@ -77,7 +77,7 @@ func TestBleveIndexAndSearch(t *testing.T) {
} }
for _, kw := range keywords { for _, kw := range keywords {
res, err := indexer.Search(context.TODO(), kw.Keyword, []int64{2}, 10, 0) res, err := indexer.Search(context.TODO(), kw.Keyword, []int64{2}, 10, 0, "")
assert.NoError(t, err) assert.NoError(t, err)
ids := make([]int64, 0, len(res.Hits)) ids := make([]int64, 0, len(res.Hits))

2
modules/indexer/issues/db/db.go
View file

@ -36,7 +36,7 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error {
} }
// Search searches for issues // Search searches for issues
func (i *Indexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) { func (i *Indexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) {
total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start) total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start)
if err != nil { if err != nil {
return nil, err return nil, err

2
modules/indexer/issues/elasticsearch/elasticsearch.go
View file

@ -140,7 +140,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error {
// Search searches for issues by given conditions. // Search searches for issues by given conditions.
// Returns the matching issue IDs // Returns the matching issue IDs
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) { func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) {
kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments") kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
query := elastic.NewBoolQuery() query := elastic.NewBoolQuery()
query = query.Must(kwQuery) query = query.Must(kwQuery)

20
modules/indexer/issues/indexer.go
View file

@ -242,12 +242,18 @@ func UpdateIssueIndexer(issue *issues_model.Issue) {
comments = append(comments, comment.Content) comments = append(comments, comment.Content)
} }
} }
issueType := "issue"
if issue.IsPull {
issueType = "pull"
}
indexerData := &internal.IndexerData{ indexerData := &internal.IndexerData{
ID: issue.ID, ID: issue.ID,
RepoID: issue.RepoID, RepoID: issue.RepoID,
Title: issue.Title, State: string(issue.State()),
Content: issue.Content, IssueType: issueType,
Comments: comments, Title: issue.Title,
Content: issue.Content,
Comments: comments,
} }
log.Debug("Adding to channel: %v", indexerData) log.Debug("Adding to channel: %v", indexerData)
if err := issueIndexerQueue.Push(indexerData); err != nil { if err := issueIndexerQueue.Push(indexerData); err != nil {
@ -278,10 +284,10 @@ func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) {
// SearchIssuesByKeyword search issue ids by keywords and repo id // SearchIssuesByKeyword search issue ids by keywords and repo id
// WARNNING: You have to ensure user have permission to visit repoIDs' issues // WARNNING: You have to ensure user have permission to visit repoIDs' issues
func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) { func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword, state string) ([]int64, error) {
var issueIDs []int64 var issueIDs []int64
indexer := *globalIndexer.Load() indexer := *globalIndexer.Load()
res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0) res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0, state)
if err != nil { if err != nil {
return nil, err return nil, err
} }

16
modules/indexer/issues/indexer_test.go
View file

@ -50,19 +50,19 @@ func TestBleveSearchIssues(t *testing.T) {
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2") ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2", "")
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, []int64{2}, ids) assert.EqualValues(t, []int64{2}, ids)
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first") ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first", "")
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids) assert.EqualValues(t, []int64{1}, ids)
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for") ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for", "")
assert.NoError(t, err) assert.NoError(t, err)
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good") ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good", "")
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids) assert.EqualValues(t, []int64{1}, ids)
} }
@ -73,19 +73,19 @@ func TestDBSearchIssues(t *testing.T) {
setting.Indexer.IssueType = "db" setting.Indexer.IssueType = "db"
InitIssueIndexer(true) InitIssueIndexer(true)
ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2") ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2", "")
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, []int64{2}, ids) assert.EqualValues(t, []int64{2}, ids)
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first") ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first", "")
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids) assert.EqualValues(t, []int64{1}, ids)
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for") ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for", "")
assert.NoError(t, err) assert.NoError(t, err)
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good") ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good", "")
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids) assert.EqualValues(t, []int64{1}, ids)
} }

4
modules/indexer/issues/internal/indexer.go
View file

@ -15,7 +15,7 @@ type Indexer interface {
internal.Indexer internal.Indexer
Index(ctx context.Context, issue []*IndexerData) error Index(ctx context.Context, issue []*IndexerData) error
Delete(ctx context.Context, ids ...int64) error Delete(ctx context.Context, ids ...int64) error
Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*SearchResult, error)
} }
// NewDummyIndexer returns a dummy indexer // NewDummyIndexer returns a dummy indexer
@ -37,6 +37,6 @@ func (d *dummyIndexer) Delete(ctx context.Context, ids ...int64) error {
return fmt.Errorf("indexer is not ready") return fmt.Errorf("indexer is not ready")
} }
func (d *dummyIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) { func (d *dummyIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*SearchResult, error) {
return nil, fmt.Errorf("indexer is not ready") return nil, fmt.Errorf("indexer is not ready")
} }

16
modules/indexer/issues/internal/model.go
View file

@ -5,13 +5,15 @@ package internal
// IndexerData data stored in the issue indexer // IndexerData data stored in the issue indexer
type IndexerData struct { type IndexerData struct {
ID int64 `json:"id"` ID int64 `json:"id"`
RepoID int64 `json:"repo_id"` RepoID int64 `json:"repo_id"`
Title string `json:"title"` State string `json:"state"` // open, closed, all
Content string `json:"content"` IssueType string `json:"type"` // issue or pull
Comments []string `json:"comments"` Title string `json:"title"`
IsDelete bool `json:"is_delete"` Content string `json:"content"`
IDs []int64 `json:"ids"` Comments []string `json:"comments"`
IsDelete bool `json:"is_delete"`
IDs []int64 `json:"ids"`
} }
// Match represents on search result // Match represents on search result

11
modules/indexer/issues/meilisearch/meilisearch.go
View file

@ -16,7 +16,7 @@ import (
) )
const ( const (
issueIndexerLatestVersion = 0 issueIndexerLatestVersion = 1
) )
var _ internal.Indexer = &Indexer{} var _ internal.Indexer = &Indexer{}
@ -70,12 +70,19 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error {
// Search searches for issues by given conditions. // Search searches for issues by given conditions.
// Returns the matching issue IDs // Returns the matching issue IDs
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) { func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) {
repoFilters := make([]string, 0, len(repoIDs)) repoFilters := make([]string, 0, len(repoIDs))
for _, repoID := range repoIDs { for _, repoID := range repoIDs {
repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10)) repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10))
} }
filter := strings.Join(repoFilters, " OR ") filter := strings.Join(repoFilters, " OR ")
if state == "open" || state == "closed" {
if filter != "" {
filter = "(" + filter + ") AND state = " + state
} else {
filter = "state = " + state
}
}
searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(keyword, &meilisearch.SearchRequest{ searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(keyword, &meilisearch.SearchRequest{
Filter: filter, Filter: filter,
Limit: int64(limit), Limit: int64(limit),

4
routers/api/v1/repo/issue.go
View file

@ -195,7 +195,7 @@ func SearchIssues(ctx *context.APIContext) {
} }
var issueIDs []int64 var issueIDs []int64
if len(keyword) > 0 && len(repoIDs) > 0 { if len(keyword) > 0 && len(repoIDs) > 0 {
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil { if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err) ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
return return
} }
@ -394,7 +394,7 @@ func ListIssues(ctx *context.APIContext) {
var issueIDs []int64 var issueIDs []int64
var labelIDs []int64 var labelIDs []int64
if len(keyword) > 0 { if len(keyword) > 0 {
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword) issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state"))
if err != nil { if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err) ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
return return

6
routers/web/repo/issue.go
View file

@ -189,7 +189,7 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
var issueIDs []int64 var issueIDs []int64
if len(keyword) > 0 { if len(keyword) > 0 {
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword) issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword, ctx.FormString("state"))
if err != nil { if err != nil {
if issue_indexer.IsAvailable(ctx) { if issue_indexer.IsAvailable(ctx) {
ctx.ServerError("issueIndexer.Search", err) ctx.ServerError("issueIndexer.Search", err)
@ -2466,7 +2466,7 @@ func SearchIssues(ctx *context.Context) {
} }
var issueIDs []int64 var issueIDs []int64
if len(keyword) > 0 && len(repoIDs) > 0 { if len(keyword) > 0 && len(repoIDs) > 0 {
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil { if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err.Error()) ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err.Error())
return return
} }
@ -2614,7 +2614,7 @@ func ListIssues(ctx *context.Context) {
var issueIDs []int64 var issueIDs []int64
var labelIDs []int64 var labelIDs []int64
if len(keyword) > 0 { if len(keyword) > 0 {
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword) issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state"))
if err != nil { if err != nil {
ctx.Error(http.StatusInternalServerError, err.Error()) ctx.Error(http.StatusInternalServerError, err.Error())
return return

2
routers/web/user/home.go
View file

@ -725,7 +725,7 @@ func issueIDsFromSearch(ctx *context.Context, ctxUser *user_model.User, keyword
if err != nil { if err != nil {
return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %w", err) return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %w", err)
} }
issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword) issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword, ctx.FormString("state"))
if err != nil { if err != nil {
return nil, fmt.Errorf("SearchIssuesByKeyword: %w", err) return nil, fmt.Errorf("SearchIssuesByKeyword: %w", err)
} }