Update git module (#2074)

vendor/code.gitea.io/git/tree_entry.go

@@ -7,6 +7,8 @@ package git
 import (
 	"fmt"
 	"path"
+	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -145,159 +147,112 @@ func (tes Entries) Sort() {
 	sort.Sort(tes)
 }
 
-// getCommitInfoState transient state for getting commit info for entries
+type commitInfo struct {
-type getCommitInfoState struct {
+	entryName string
-	entries        map[string]*TreeEntry // map from filepath to entry
+	infos     []interface{}
-	commits        map[string]*Commit    // map from filepath to commit
+	err       error
-	lastCommitHash string
-	lastCommit     *Commit
-	treePath       string
-	headCommit     *Commit
-	nextSearchSize int // next number of commits to search for
 }
 
-func initGetCommitInfoState(entries Entries, headCommit *Commit, treePath string) *getCommitInfoState {
+// GetCommitsInfo takes advantages of concurrency to speed up getting information
-	entriesByPath := make(map[string]*TreeEntry, len(entries))
+// of all commits that are corresponding to these entries. This method will automatically
-	for _, entry := range entries {
+// choose the right number of goroutine (concurrency) to use related of the host CPU.
-		entriesByPath[path.Join(treePath, entry.Name())] = entry
-	}
-	if treePath = path.Clean(treePath); treePath == "." {
-		treePath = ""
-	}
-	return &getCommitInfoState{
-		entries:        entriesByPath,
-		commits:        make(map[string]*Commit, len(entriesByPath)),
-		treePath:       treePath,
-		headCommit:     headCommit,
-		nextSearchSize: 16,
-	}
-}
-
-// GetCommitsInfo gets information of all commits that are corresponding to these entries
 func (tes Entries) GetCommitsInfo(commit *Commit, treePath string) ([][]interface{}, error) {
-	state := initGetCommitInfoState(tes, commit, treePath)
+	return tes.GetCommitsInfoWithCustomConcurrency(commit, treePath, 0)
-	if err := getCommitsInfo(state); err != nil {
+}
+
+// GetCommitsInfoWithCustomConcurrency takes advantages of concurrency to speed up getting information
+// of all commits that are corresponding to these entries. If the given maxConcurrency is negative or
+// equal to zero:  the right number of goroutine (concurrency) to use will be chosen related of the
+// host CPU.
+func (tes Entries) GetCommitsInfoWithCustomConcurrency(commit *Commit, treePath string, maxConcurrency int) ([][]interface{}, error) {
+	if len(tes) == 0 {
+		return nil, nil
+	}
+
+	if maxConcurrency <= 0 {
+		maxConcurrency = runtime.NumCPU()
+	}
+
+	// Length of taskChan determines how many goroutines (subprocesses) can run at the same time.
+	// The length of revChan should be same as taskChan so goroutines whoever finished job can
+	// exit as early as possible, only store data inside channel.
+	taskChan := make(chan bool, maxConcurrency)
+	revChan := make(chan commitInfo, maxConcurrency)
+	doneChan := make(chan error)
+
+	// Receive loop will exit when it collects same number of data pieces as tree entries.
+	// It notifies doneChan before exits or notify early with possible error.
+	infoMap := make(map[string][]interface{}, len(tes))
+	go func() {
+		i := 0
+		for info := range revChan {
+			if info.err != nil {
+				doneChan <- info.err
+				return
+			}
+
+			infoMap[info.entryName] = info.infos
+			i++
+			if i == len(tes) {
+				break
+			}
+		}
+		doneChan <- nil
+	}()
+
+	for i := range tes {
+		// When taskChan is idle (or has empty slots), put operation will not block.
+		// However when taskChan is full, code will block and wait any running goroutines to finish.
+		taskChan <- true
+
+		if tes[i].Type != ObjectCommit {
+			go func(i int) {
+				cinfo := commitInfo{entryName: tes[i].Name()}
+				c, err := commit.GetCommitByPath(filepath.Join(treePath, tes[i].Name()))
+				if err != nil {
+					cinfo.err = fmt.Errorf("GetCommitByPath (%s/%s): %v", treePath, tes[i].Name(), err)
+				} else {
+					cinfo.infos = []interface{}{tes[i], c}
+				}
+				revChan <- cinfo
+				<-taskChan // Clear one slot from taskChan to allow new goroutines to start.
+			}(i)
+			continue
+		}
+
+		// Handle submodule
+		go func(i int) {
+			cinfo := commitInfo{entryName: tes[i].Name()}
+			sm, err := commit.GetSubModule(path.Join(treePath, tes[i].Name()))
+			if err != nil && !IsErrNotExist(err) {
+				cinfo.err = fmt.Errorf("GetSubModule (%s/%s): %v", treePath, tes[i].Name(), err)
+				revChan <- cinfo
+				return
+			}
+
+			smURL := ""
+			if sm != nil {
+				smURL = sm.URL
+			}
+
+			c, err := commit.GetCommitByPath(filepath.Join(treePath, tes[i].Name()))
+			if err != nil {
+				cinfo.err = fmt.Errorf("GetCommitByPath (%s/%s): %v", treePath, tes[i].Name(), err)
+			} else {
+				cinfo.infos = []interface{}{tes[i], NewSubModuleFile(c, smURL, tes[i].ID.String())}
+			}
+			revChan <- cinfo
+			<-taskChan
+		}(i)
+	}
+
+	if err := <-doneChan; err != nil {
 		return nil, err
 	}
 
 	commitsInfo := make([][]interface{}, len(tes))
-	for i, entry := range tes {
+	for i := 0; i < len(tes); i++ {
-		commit = state.commits[path.Join(treePath, entry.Name())]
+		commitsInfo[i] = infoMap[tes[i].Name()]
-		switch entry.Type {
-		case ObjectCommit:
-			subModuleURL := ""
-			if subModule, err := state.headCommit.GetSubModule(entry.Name()); err != nil {
-				return nil, err
-			} else if subModule != nil {
-				subModuleURL = subModule.URL
-			}
-			subModuleFile := NewSubModuleFile(commit, subModuleURL, entry.ID.String())
-			commitsInfo[i] = []interface{}{entry, subModuleFile}
-		default:
-			commitsInfo[i] = []interface{}{entry, commit}
-		}
 	}
 	return commitsInfo, nil
 }
-
-func (state *getCommitInfoState) nextCommit(hash string) {
-	state.lastCommitHash = hash
-	state.lastCommit = nil
-}
-
-func (state *getCommitInfoState) commit() (*Commit, error) {
-	var err error
-	if state.lastCommit == nil {
-		state.lastCommit, err = state.headCommit.repo.GetCommit(state.lastCommitHash)
-	}
-	return state.lastCommit, err
-}
-
-func (state *getCommitInfoState) update(entryPath string) error {
-	var entryNameStartIndex int
-	if len(state.treePath) > 0 {
-		entryNameStartIndex = len(state.treePath) + 1
-	}
-
-	if index := strings.IndexByte(entryPath[entryNameStartIndex:], '/'); index >= 0 {
-		entryPath = entryPath[:entryNameStartIndex+index]
-	}
-
-	if _, ok := state.entries[entryPath]; !ok {
-		return nil
-	} else if _, ok := state.commits[entryPath]; ok {
-		return nil
-	}
-
-	var err error
-	state.commits[entryPath], err = state.commit()
-	return err
-}
-
-func getCommitsInfo(state *getCommitInfoState) error {
-	for len(state.entries) > len(state.commits) {
-		if err := getNextCommitInfos(state); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func getNextCommitInfos(state *getCommitInfoState) error {
-	logOutput, err := logCommand(state.lastCommitHash, state).RunInDir(state.headCommit.repo.Path)
-	if err != nil {
-		return err
-	}
-	lines := strings.Split(logOutput, "\n")
-	i := 0
-	for i < len(lines) {
-		state.nextCommit(lines[i])
-		i++
-		for ; i < len(lines); i++ {
-			entryPath := lines[i]
-			if entryPath == "" {
-				break
-			}
-			if entryPath[0] == '"' {
-				entryPath, err = strconv.Unquote(entryPath)
-				if err != nil {
-					return fmt.Errorf("Unquote: %v", err)
-				}
-			}
-			if err = state.update(entryPath); err != nil {
-				return err
-			}
-		}
-		i++ // skip blank line
-		if len(state.entries) == len(state.commits) {
-			break
-		}
-	}
-	return nil
-}
-
-func logCommand(exclusiveStartHash string, state *getCommitInfoState) *Command {
-	var commitHash string
-	if len(exclusiveStartHash) == 0 {
-		commitHash = state.headCommit.ID.String()
-	} else {
-		commitHash = exclusiveStartHash + "^"
-	}
-	var command *Command
-	numRemainingEntries := len(state.entries) - len(state.commits)
-	if numRemainingEntries < 32 {
-		searchSize := (numRemainingEntries + 1) / 2
-		command = NewCommand("log", prettyLogFormat, "--name-only",
-			"-"+strconv.Itoa(searchSize), commitHash, "--")
-		for entryPath := range state.entries {
-			if _, ok := state.commits[entryPath]; !ok {
-				command.AddArguments(entryPath)
-			}
-		}
-	} else {
-		command = NewCommand("log", prettyLogFormat, "--name-only",
-			"-"+strconv.Itoa(state.nextSearchSize), commitHash, "--", state.treePath)
-	}
-	state.nextSearchSize += state.nextSearchSize
-	return command
-}

vendor/vendor.json

@@ -3,10 +3,10 @@
 	"ignore": "test appengine",
 	"package": [
 		{
-			"checksumSHA1": "Ju4zZF8u/DPrZYEEY40rogh3hyQ=",
+			"checksumSHA1": "bjwnhCgzLl8EtrjX+vsz7POSaWw=",
 			"path": "code.gitea.io/git",
-			"revision": "51eca9e92242b93a0510edd19f1db6fc11ca1028",
+			"revision": "7c4fc4e5ca3aab313bbe68b4b19ca253830bc60f",
-			"revisionTime": "2017-06-21T01:06:07Z"
+			"revisionTime": "2017-06-28T06:06:37Z"
 		},
 		{
 			"checksumSHA1": "nLhT+bLMj8uLICP+EZbrdoQe6mM=",