Merge pull request 'feat: support regexp in git-grep search' (#4968) from yoctozepto/git-grep-regexp into forgejo

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4968
Reviewed-by: Shiny Nematoda <snematoda@noreply.codeberg.org>

modules/git/grep.go

@@ -27,12 +27,20 @@ type GrepResult struct {
 	HighlightedRanges [][3]int
 }
 
+type grepMode int
+
+const (
+	FixedGrepMode grepMode = iota
+	FixedAnyGrepMode
+	RegExpGrepMode
+)
+
 type GrepOptions struct {
 	RefName           string
 	MaxResultLimit    int
 	MatchesPerFile    int
 	ContextLineNumber int
-	IsFuzzy           bool
+	Mode              grepMode
 	PathSpec          []setting.Glob
 }
 
@@ -74,12 +82,20 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
 	var results []*GrepResult
 	// -I skips binary files
 	cmd := NewCommand(ctx, "grep",
-		"-I", "--null", "--break", "--heading", "--column",
+		"-I", "--null", "--break", "--heading",
-		"--fixed-strings", "--line-number", "--ignore-case", "--full-name")
+		"--line-number", "--ignore-case", "--full-name")
+	if opts.Mode == RegExpGrepMode {
+		// No `--column` -- regexp mode does not support highlighting in the
+		// current implementation as the length of the match is unknown from
+		// `grep` but required for highlighting.
+		cmd.AddArguments("--perl-regexp")
+	} else {
+		cmd.AddArguments("--fixed-strings", "--column")
+	}
 	cmd.AddOptionValues("--context", fmt.Sprint(opts.ContextLineNumber))
 	cmd.AddOptionValues("--max-count", fmt.Sprint(opts.MatchesPerFile))
 	words := []string{search}
-	if opts.IsFuzzy {
+	if opts.Mode == FixedAnyGrepMode {
 		words = strings.Fields(search)
 	}
 	for _, word := range words {
@@ -148,6 +164,7 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
 				if lineNum, lineCode, ok := strings.Cut(line, "\x00"); ok {
 					lineNumInt, _ := strconv.Atoi(lineNum)
 					res.LineNumbers = append(res.LineNumbers, lineNumInt)
+					// We support highlighting only when `--column` parameter is used.
 					if lineCol, lineCode2, ok := strings.Cut(lineCode, "\x00"); ok {
 						lineColInt, _ := strconv.Atoi(lineCol)
 						start := lineColInt - 1

modules/git/grep_test.go

@@ -201,3 +201,34 @@ func TestGrepRefs(t *testing.T) {
 	assert.Len(t, res, 1)
 	assert.Equal(t, "A", res[0].LineCodes[0])
 }
+
+func TestGrepCanHazRegexOnDemand(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	err := InitRepository(DefaultContext, tmpDir, false, Sha1ObjectFormat.Name())
+	require.NoError(t, err)
+
+	gitRepo, err := openRepositoryWithDefaultContext(tmpDir)
+	require.NoError(t, err)
+	defer gitRepo.Close()
+
+	require.NoError(t, os.WriteFile(path.Join(tmpDir, "matching"), []byte("It's a match!"), 0o666))
+	require.NoError(t, os.WriteFile(path.Join(tmpDir, "not-matching"), []byte("Orisitamatch?"), 0o666))
+
+	err = AddChanges(tmpDir, true)
+	require.NoError(t, err)
+
+	err = CommitChanges(tmpDir, CommitChangesOptions{Message: "Add fixtures for regexp test"})
+	require.NoError(t, err)
+
+	// should find nothing by default...
+	res, err := GrepSearch(context.Background(), gitRepo, "\\bmatch\\b", GrepOptions{})
+	require.NoError(t, err)
+	assert.Empty(t, res)
+
+	// ... unless configured explicitly
+	res, err = GrepSearch(context.Background(), gitRepo, "\\bmatch\\b", GrepOptions{Mode: RegExpGrepMode})
+	require.NoError(t, err)
+	assert.Len(t, res, 1)
+	assert.Equal(t, "matching", res[0].Filename)
+}

options/locale/locale_en-US.ini

@@ -173,6 +173,8 @@ union = Union
 union_tooltip = Include results that match any of the whitespace seperated keywords
 exact = Exact
 exact_tooltip = Include only results that match the exact search term
+regexp = RegExp
+regexp_tooltip = Interpret the search term as a regular expression
 repo_kind = Search repos...
 user_kind = Search users...
 org_kind = Search orgs...

routers/web/explore/code.go

@@ -36,10 +36,18 @@ func Code(ctx *context.Context) {
 	keyword := ctx.FormTrim("q")
 
 	isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true)
+	if mode := ctx.FormTrim("mode"); len(mode) > 0 {
+		isFuzzy = mode == "fuzzy"
+	}
 
 	ctx.Data["Keyword"] = keyword
 	ctx.Data["Language"] = language
-	ctx.Data["IsFuzzy"] = isFuzzy
+	ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"}
+	if isFuzzy {
+		ctx.Data["CodeSearchMode"] = "fuzzy"
+	} else {
+		ctx.Data["CodeSearchMode"] = "exact"
+	}
 	ctx.Data["PageIsViewCode"] = true
 
 	if keyword == "" {

routers/web/repo/search.go

@@ -17,16 +17,53 @@ import (
 
 const tplSearch base.TplName = "repo/search"
 
+type searchMode int
+
+const (
+	ExactSearchMode searchMode = iota
+	FuzzySearchMode
+	RegExpSearchMode
+)
+
+func searchModeFromString(s string) searchMode {
+	switch s {
+	case "fuzzy", "union":
+		return FuzzySearchMode
+	case "regexp":
+		return RegExpSearchMode
+	default:
+		return ExactSearchMode
+	}
+}
+
+func (m searchMode) String() string {
+	switch m {
+	case ExactSearchMode:
+		return "exact"
+	case FuzzySearchMode:
+		return "fuzzy"
+	case RegExpSearchMode:
+		return "regexp"
+	default:
+		panic("cannot happen")
+	}
+}
+
 // Search render repository search page
 func Search(ctx *context.Context) {
 	language := ctx.FormTrim("l")
 	keyword := ctx.FormTrim("q")
 
-	isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true)
+	mode := ExactSearchMode
+	if modeStr := ctx.FormString("mode"); len(modeStr) > 0 {
+		mode = searchModeFromString(modeStr)
+	} else if ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) { // for backward compatibility in links
+		mode = FuzzySearchMode
+	}
 
 	ctx.Data["Keyword"] = keyword
 	ctx.Data["Language"] = language
-	ctx.Data["IsFuzzy"] = isFuzzy
+	ctx.Data["CodeSearchMode"] = mode.String()
 	ctx.Data["PageIsViewCode"] = true
 
 	if keyword == "" {
@@ -47,7 +84,7 @@ func Search(ctx *context.Context) {
 		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
 			RepoIDs:        []int64{ctx.Repo.Repository.ID},
 			Keyword:        keyword,
-			IsKeywordFuzzy: isFuzzy,
+			IsKeywordFuzzy: mode == FuzzySearchMode,
 			Language:       language,
 			Paginator: &db.ListOptions{
 				Page:     page,
@@ -63,12 +100,20 @@ func Search(ctx *context.Context) {
 		} else {
 			ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable(ctx)
 		}
+		ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"}
 	} else {
-		res, err := git.GrepSearch(ctx, ctx.Repo.GitRepo, keyword, git.GrepOptions{
+		grepOpt := git.GrepOptions{
 			ContextLineNumber: 1,
-			IsFuzzy:           isFuzzy,
 			RefName:           ctx.Repo.RefName,
-		})
+		}
+		switch mode {
+		case FuzzySearchMode:
+			grepOpt.Mode = git.FixedAnyGrepMode
+			ctx.Data["CodeSearchMode"] = "union"
+		case RegExpSearchMode:
+			grepOpt.Mode = git.RegExpGrepMode
+		}
+		res, err := git.GrepSearch(ctx, ctx.Repo.GitRepo, keyword, grepOpt)
 		if err != nil {
 			ctx.ServerError("GrepSearch", err)
 			return
@@ -88,6 +133,7 @@ func Search(ctx *context.Context) {
 				Lines: code_indexer.HighlightSearchResultCode(r.Filename, r.LineNumbers, r.HighlightedRanges, strings.Join(r.LineCodes, "\n")),
 			})
 		}
+		ctx.Data["CodeSearchOptions"] = []string{"exact", "union", "regexp"}
 	}
 
 	ctx.Data["CodeIndexerDisabled"] = !setting.Indexer.RepoIndexerEnabled

routers/web/user/code.go

@@ -41,10 +41,18 @@ func CodeSearch(ctx *context.Context) {
 	keyword := ctx.FormTrim("q")
 
 	isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true)
+	if mode := ctx.FormTrim("mode"); len(mode) > 0 {
+		isFuzzy = mode == "fuzzy"
+	}
 
 	ctx.Data["Keyword"] = keyword
 	ctx.Data["Language"] = language
-	ctx.Data["IsFuzzy"] = isFuzzy
+	ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"}
+	if isFuzzy {
+		ctx.Data["CodeSearchMode"] = "fuzzy"
+	} else {
+		ctx.Data["CodeSearchMode"] = "exact"
+	}
 	ctx.Data["IsCodePage"] = true
 
 	if keyword == "" {

services/wiki/wiki.go

@@ -417,7 +417,7 @@ func SearchWikiContents(ctx context.Context, repo *repo_model.Repository, keywor
 
 	return git.GrepSearch(ctx, gitRepo, keyword, git.GrepOptions{
 		ContextLineNumber: 0,
-		IsFuzzy:           true,
+		Mode:              git.FixedAnyGrepMode,
 		RefName:           repo.GetWikiBranchName(),
 		MaxResultLimit:    10,
 		MatchesPerFile:    3,

templates/repo/search.tmpl

@@ -5,7 +5,7 @@
 		{{if $.CodeIndexerDisabled}}
 			{{$branchURLPrefix := printf "%s/search/branch/" $.RepoLink}}
 			{{$tagURLPrefix := printf "%s/search/tag/" $.RepoLink}}
-			{{$suffix := printf "?q=%s&fuzzy=%t" (.Keyword|QueryEscape) .IsFuzzy}}
+			{{$suffix := printf "?q=%s&mode=%s" (.Keyword|QueryEscape) .CodeSearchMode}}
 			{{template "repo/branch_dropdown" dict "root" . "ContainerClasses" "tw-mb-3" "branchURLPrefix" $branchURLPrefix "branchURLSuffix" $suffix "tagURLPrefix" $tagURLPrefix "tagURLSuffix" $suffix}}
 		{{end}}
 		{{template "shared/search/code/search" .}}

templates/shared/search/code/results.tmpl

@@ -1,7 +1,7 @@
 <div class="flex-text-block tw-flex-wrap">
 	{{range $term := .SearchResultLanguages}}
 	<a class="ui {{if eq $.Language $term.Language}}primary{{end}} basic label tw-m-0"
-		href="?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}&fuzzy={{$.IsFuzzy}}">
+		href="?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}&mode={{$.CodeSearchMode}}">
 		<i class="color-icon tw-mr-2" style="background-color: {{$term.Color}}"></i>
 		{{$term.Language}}
 		<div class="detail">{{$term.Count}}</div>

templates/shared/search/code/search.tmpl

@@ -1,11 +1,11 @@
 <form class="ui form ignore-dirty">
-	{{template "shared/search/combo_fuzzy"
+	{{template "shared/search/combo_multi"
 		dict
 			"Value" .Keyword
 			"Disabled" .CodeIndexerUnavailable
-			"IsFuzzy" .IsFuzzy
 			"Placeholder" (ctx.Locale.Tr "search.code_kind")
-			"CodeIndexerDisabled" $.CodeIndexerDisabled}}
+			"Selected" $.CodeSearchMode
+			"Options" $.CodeSearchOptions}}
 </form>
 <div class="divider"></div>
 <div class="ui user list">

templates/shared/search/combo_fuzzy.tmpl

@@ -2,14 +2,12 @@
 {{/* Disabled (optional) - if search field/button has to be disabled */}}
 {{/* Placeholder (optional) - placeholder text to be used */}}
 {{/* IsFuzzy - state of the fuzzy/union search toggle */}}
-{{/* CodeIndexerDisabled (optional) - if the performed search is done using git-grep */}}
 {{/* Tooltip (optional) - a tooltip to be displayed on button hover */}}
 <div class="ui small fluid action input">
 	{{template "shared/search/input" dict "Value" .Value "Disabled" .Disabled "Placeholder" .Placeholder}}
 	{{template "shared/search/fuzzy"
 		dict
 			"Disabled" .Disabled
-			"IsFuzzy" .IsFuzzy
+			"IsFuzzy" .IsFuzzy}}
-			"CodeIndexerDisabled" .CodeIndexerDisabled}}
 	{{template "shared/search/button" dict "Disabled" .Disabled "Tooltip" .Tooltip}}
 </div>

templates/shared/search/combo_multi.tmpl

@@ -0,0 +1,24 @@
+{{/* Value - value of the search field (for search results page) */}}
+{{/* Disabled (optional) - if search field/button has to be disabled */}}
+{{/* Placeholder (optional) - placeholder text to be used */}}
+{{/* Selected - the currently selected option */}}
+{{/* Options - options available to choose from */}}
+{{/* Tooltip (optional) - a tooltip to be displayed on button hover */}}
+<div class="ui small fluid action input">
+	{{template "shared/search/input" dict "Value" .Value "Disabled" .Disabled "Placeholder" .Placeholder}}
+	<div class="ui small dropdown selection {{if .Disabled}} disabled{{end}}" data-tooltip-content="{{ctx.Locale.Tr "search.type_tooltip"}}">
+		<div class="text">
+			{{ctx.Locale.Tr (printf "search.%s" .Selected)}}
+		</div>
+		<div class="menu" data-test-tag="fuzzy-dropdown">
+			{{range $opt := .Options}}
+				{{$isActive := eq $.Selected $opt}}
+				<label class="{{if $isActive}}active {{end}}item" data-value="{{$opt}}" data-tooltip-content="{{ctx.Locale.Tr (printf "search.%s_tooltip" $opt)}}">
+					<input hidden type="radio" name="mode" value="{{$opt}}"{{if $isActive}} checked{{end}}/>
+					{{ctx.Locale.Tr (printf "search.%s" $opt)}}
+				</label>
+			{{end}}
+		</div>
+	</div>
+	{{template "shared/search/button" dict "Disabled" .Disabled "Tooltip" .Tooltip}}
+</div>

templates/shared/search/fuzzy.tmpl

@@ -1,21 +1,15 @@
 {{/* Disabled (optional) - if dropdown has to be disabled */}}
 {{/* IsFuzzy - state of the fuzzy search toggle */}}
-<div class="ui small dropdown selection {{if .Disabled}} disabled{{end}}" data-tooltip-content="{{ctx.Locale.Tr "search.type_tooltip"}}" data-test-tag="fuzzy-dropdown">
+<div class="ui small dropdown selection {{if .Disabled}} disabled{{end}}" data-tooltip-content="{{ctx.Locale.Tr "search.type_tooltip"}}">
-	{{$fuzzyType := "fuzzy"}}
-	{{if .CodeIndexerDisabled}}
-		{{$fuzzyType = "union"}}
-	{{end}}
 	<input name="fuzzy" type="hidden"{{if .Disabled}} disabled{{end}} value="{{.IsFuzzy}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
-	<div class="text">{{/*
+	<div class="text">{{if .IsFuzzy}}{{/*
-		if code indexer is disabled display fuzzy as union
+		*/}}{{ctx.Locale.Tr "search.fuzzy"}}{{/*
-	*/}}{{if .IsFuzzy}}{{/*
-		*/}}{{ctx.Locale.Tr (printf "search.%s" $fuzzyType)}}{{/*
 	*/}}{{else}}{{/*
 		*/}}{{ctx.Locale.Tr "search.exact"}}{{/*
 	*/}}{{end}}</div>
 	<div class="menu">
-		<div class="item" data-value="true" data-tooltip-content="{{ctx.Locale.Tr (printf "search.%s_tooltip" $fuzzyType)}}">{{/*
+		<div class="item" data-value="true" data-tooltip-content="{{ctx.Locale.Tr "search.fuzzy_tooltip"}}">{{/*
-		*/}}{{ctx.Locale.Tr (printf "search.%s" $fuzzyType)}}</div>
+		*/}}{{ctx.Locale.Tr "search.fuzzy"}}</div>
 		<div class="item" data-value="false" data-tooltip-content="{{ctx.Locale.Tr "search.exact_tooltip"}}">{{ctx.Locale.Tr "search.exact"}}</div>
 	</div>
 </div>

tests/integration/repo_search_test.go

@@ -79,25 +79,25 @@ func testSearchRepo(t *testing.T, indexer bool) {
 	}
 
 	testSearch(t, "/user2/glob/search?q=loren&page=1", []string{"a.txt"}, indexer)
-	testSearch(t, "/user2/glob/search?q=loren&page=1&fuzzy=false", []string{"a.txt"}, indexer)
+	testSearch(t, "/user2/glob/search?q=loren&page=1&mode=exact", []string{"a.txt"}, indexer)
 
 	if indexer {
 		// fuzzy search: matches both file3 (x/b.txt) and file1 (a.txt)
 		// when indexer is enabled
-		testSearch(t, "/user2/glob/search?q=file3&page=1", []string{"x/b.txt", "a.txt"}, indexer)
+		testSearch(t, "/user2/glob/search?q=file3&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
-		testSearch(t, "/user2/glob/search?q=file4&page=1", []string{"x/b.txt", "a.txt"}, indexer)
+		testSearch(t, "/user2/glob/search?q=file4&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
-		testSearch(t, "/user2/glob/search?q=file5&page=1", []string{"x/b.txt", "a.txt"}, indexer)
+		testSearch(t, "/user2/glob/search?q=file5&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
 	} else {
 		// fuzzy search: Union/OR of all the keywords
 		// when indexer is disabled
-		testSearch(t, "/user2/glob/search?q=file3+file1&page=1", []string{"a.txt", "x/b.txt"}, indexer)
+		testSearch(t, "/user2/glob/search?q=file3+file1&mode=union&page=1", []string{"a.txt", "x/b.txt"}, indexer)
-		testSearch(t, "/user2/glob/search?q=file4&page=1", []string{}, indexer)
+		testSearch(t, "/user2/glob/search?q=file4&mode=union&page=1", []string{}, indexer)
-		testSearch(t, "/user2/glob/search?q=file5&page=1", []string{}, indexer)
+		testSearch(t, "/user2/glob/search?q=file5&mode=union&page=1", []string{}, indexer)
 	}
 
-	testSearch(t, "/user2/glob/search?q=file3&page=1&fuzzy=false", []string{"x/b.txt"}, indexer)
+	testSearch(t, "/user2/glob/search?q=file3&page=1&mode=exact", []string{"x/b.txt"}, indexer)
-	testSearch(t, "/user2/glob/search?q=file4&page=1&fuzzy=false", []string{}, indexer)
+	testSearch(t, "/user2/glob/search?q=file4&page=1&mode=exact", []string{}, indexer)
-	testSearch(t, "/user2/glob/search?q=file5&page=1&fuzzy=false", []string{}, indexer)
+	testSearch(t, "/user2/glob/search?q=file5&page=1&mode=exact", []string{}, indexer)
 }
 
 func testSearch(t *testing.T, url string, expected []string, indexer bool) {
@@ -113,21 +113,19 @@ func testSearch(t *testing.T, url string, expected []string, indexer bool) {
 	branchDropdown := container.Find(".js-branch-tag-selector")
 	assert.EqualValues(t, indexer, len(branchDropdown.Nodes) == 0)
 
-	// if indexer is disabled "fuzzy" should be displayed as "union"
+	dropdownOptions := container.
-	expectedFuzzy := "Fuzzy"
+		Find(".menu[data-test-tag=fuzzy-dropdown]").
-	if !indexer {
+		Find("input[type=radio][name=mode]").
-		expectedFuzzy = "Union"
+		Map(func(_ int, sel *goquery.Selection) string {
-	}
+			attr, exists := sel.Attr("value")
+			assert.True(t, exists)
+			return attr
+		})
 
-	fuzzyDropdown := container.Find(".ui.dropdown[data-test-tag=fuzzy-dropdown]")
+	if indexer {
-	actualFuzzyText := fuzzyDropdown.Find(".menu .item[data-value=true]").First().Text()
+		assert.EqualValues(t, []string{"exact", "fuzzy"}, dropdownOptions)
-	assert.EqualValues(t, expectedFuzzy, actualFuzzyText)
+	} else {
-
+		assert.EqualValues(t, []string{"exact", "union", "regexp"}, dropdownOptions)
-	if fuzzyDropdown.
-		Find("input[name=fuzzy][value=true]").
-		Length() != 0 {
-		actualFuzzyText = fuzzyDropdown.Find("div.text").First().Text()
-		assert.EqualValues(t, expectedFuzzy, actualFuzzyText)
 	}
 
 	filenames := resultFilenames(t, doc)