forked from NYANDEV/forgejo
Add migrate from OneDev (#16356)
* Use context to simplify logic. * Added migration from OneDev. This PR adds [OneDev](https://code.onedev.io/) as migration source. Supported: - [x] Milestones - [x] Issues - [x] Pull Requests - [x] Comments - [x] Reviews - [x] Labels
This commit is contained in:
parent
2d1935acc7
commit
cee5f7c5e2
24 changed files with 1093 additions and 92 deletions
619
modules/migrations/onedev.go
Normal file
619
modules/migrations/onedev.go
Normal file
|
@ -0,0 +1,619 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/migrations/base"
|
||||
"code.gitea.io/gitea/modules/structs"
|
||||
)
|
||||
|
||||
var (
|
||||
_ base.Downloader = &OneDevDownloader{}
|
||||
_ base.DownloaderFactory = &OneDevDownloaderFactory{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterDownloaderFactory(&OneDevDownloaderFactory{})
|
||||
}
|
||||
|
||||
// OneDevDownloaderFactory defines a downloader factory
|
||||
type OneDevDownloaderFactory struct {
|
||||
}
|
||||
|
||||
// New returns a downloader related to this factory according MigrateOptions
|
||||
func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
|
||||
u, err := url.Parse(opts.CloneAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
repoName := ""
|
||||
|
||||
fields := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||
if len(fields) == 2 && fields[0] == "projects" {
|
||||
repoName = fields[1]
|
||||
} else if len(fields) == 1 {
|
||||
repoName = fields[0]
|
||||
} else {
|
||||
return nil, fmt.Errorf("invalid path: %s", u.Path)
|
||||
}
|
||||
|
||||
u.Path = ""
|
||||
u.Fragment = ""
|
||||
|
||||
log.Trace("Create onedev downloader. BaseURL: %v RepoName: %s", u, repoName)
|
||||
|
||||
return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoName), nil
|
||||
}
|
||||
|
||||
// GitServiceType returns the type of git service
|
||||
func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType {
|
||||
return structs.OneDevService
|
||||
}
|
||||
|
||||
type onedevUser struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Email string `json:"email"`
|
||||
}
|
||||
|
||||
// OneDevDownloader implements a Downloader interface to get repository informations
|
||||
// from OneDev
|
||||
type OneDevDownloader struct {
|
||||
base.NullDownloader
|
||||
ctx context.Context
|
||||
client *http.Client
|
||||
baseURL *url.URL
|
||||
repoName string
|
||||
repoID int64
|
||||
maxIssueIndex int64
|
||||
userMap map[int64]*onedevUser
|
||||
milestoneMap map[int64]string
|
||||
}
|
||||
|
||||
// SetContext set context
|
||||
func (d *OneDevDownloader) SetContext(ctx context.Context) {
|
||||
d.ctx = ctx
|
||||
}
|
||||
|
||||
// NewOneDevDownloader creates a new downloader
|
||||
func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader {
|
||||
var downloader = &OneDevDownloader{
|
||||
ctx: ctx,
|
||||
baseURL: baseURL,
|
||||
repoName: repoName,
|
||||
client: &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: func(req *http.Request) (*url.URL, error) {
|
||||
if len(username) > 0 && len(password) > 0 {
|
||||
req.SetBasicAuth(username, password)
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
},
|
||||
},
|
||||
userMap: make(map[int64]*onedevUser),
|
||||
milestoneMap: make(map[int64]string),
|
||||
}
|
||||
|
||||
return downloader
|
||||
}
|
||||
|
||||
func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error {
|
||||
u, err := d.baseURL.Parse(endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if parameter != nil {
|
||||
query := u.Query()
|
||||
for k, v := range parameter {
|
||||
query.Set(k, v)
|
||||
}
|
||||
u.RawQuery = query.Encode()
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := d.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
return decoder.Decode(&result)
|
||||
}
|
||||
|
||||
// GetRepoInfo returns repository information
|
||||
func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) {
|
||||
info := make([]struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
}, 0, 1)
|
||||
|
||||
err := d.callAPI(
|
||||
"/api/projects",
|
||||
map[string]string{
|
||||
"query": `"Name" is "` + d.repoName + `"`,
|
||||
"offset": "0",
|
||||
"count": "1",
|
||||
},
|
||||
&info,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(info) != 1 {
|
||||
return nil, fmt.Errorf("Project %s not found", d.repoName)
|
||||
}
|
||||
|
||||
d.repoID = info[0].ID
|
||||
|
||||
cloneURL, err := d.baseURL.Parse(info[0].Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
originalURL, err := d.baseURL.Parse("/projects/" + info[0].Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &base.Repository{
|
||||
Name: info[0].Name,
|
||||
Description: info[0].Description,
|
||||
CloneURL: cloneURL.String(),
|
||||
OriginalURL: originalURL.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetMilestones returns milestones
|
||||
func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) {
|
||||
rawMilestones := make([]struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
DueDate *time.Time `json:"dueDate"`
|
||||
Closed bool `json:"closed"`
|
||||
}, 0, 100)
|
||||
|
||||
endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID)
|
||||
|
||||
var milestones = make([]*base.Milestone, 0, 100)
|
||||
offset := 0
|
||||
for {
|
||||
err := d.callAPI(
|
||||
endpoint,
|
||||
map[string]string{
|
||||
"offset": strconv.Itoa(offset),
|
||||
"count": "100",
|
||||
},
|
||||
&rawMilestones,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rawMilestones) == 0 {
|
||||
break
|
||||
}
|
||||
offset += 100
|
||||
|
||||
for _, milestone := range rawMilestones {
|
||||
d.milestoneMap[milestone.ID] = milestone.Name
|
||||
closed := milestone.DueDate
|
||||
if !milestone.Closed {
|
||||
closed = nil
|
||||
}
|
||||
|
||||
milestones = append(milestones, &base.Milestone{
|
||||
Title: milestone.Name,
|
||||
Description: milestone.Description,
|
||||
Deadline: milestone.DueDate,
|
||||
Closed: closed,
|
||||
})
|
||||
}
|
||||
}
|
||||
return milestones, nil
|
||||
}
|
||||
|
||||
// GetLabels returns labels
|
||||
func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) {
|
||||
return []*base.Label{
|
||||
{
|
||||
Name: "Bug",
|
||||
Color: "f64e60",
|
||||
},
|
||||
{
|
||||
Name: "Build Failure",
|
||||
Color: "f64e60",
|
||||
},
|
||||
{
|
||||
Name: "Discussion",
|
||||
Color: "8950fc",
|
||||
},
|
||||
{
|
||||
Name: "Improvement",
|
||||
Color: "1bc5bd",
|
||||
},
|
||||
{
|
||||
Name: "New Feature",
|
||||
Color: "1bc5bd",
|
||||
},
|
||||
{
|
||||
Name: "Support Request",
|
||||
Color: "8950fc",
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type onedevIssueContext struct {
|
||||
foreignID int64
|
||||
localID int64
|
||||
IsPullRequest bool
|
||||
}
|
||||
|
||||
func (c onedevIssueContext) LocalID() int64 {
|
||||
return c.localID
|
||||
}
|
||||
|
||||
func (c onedevIssueContext) ForeignID() int64 {
|
||||
return c.foreignID
|
||||
}
|
||||
|
||||
// GetIssues returns issues
|
||||
func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
|
||||
rawIssues := make([]struct {
|
||||
ID int64 `json:"id"`
|
||||
Number int64 `json:"number"`
|
||||
State string `json:"state"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
MilestoneID int64 `json:"milestoneId"`
|
||||
SubmitterID int64 `json:"submitterId"`
|
||||
SubmitDate time.Time `json:"submitDate"`
|
||||
}, 0, perPage)
|
||||
|
||||
err := d.callAPI(
|
||||
"/api/issues",
|
||||
map[string]string{
|
||||
"query": `"Project" is "` + d.repoName + `"`,
|
||||
"offset": strconv.Itoa((page - 1) * perPage),
|
||||
"count": strconv.Itoa(perPage),
|
||||
},
|
||||
&rawIssues,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
issues := make([]*base.Issue, 0, len(rawIssues))
|
||||
for _, issue := range rawIssues {
|
||||
fields := make([]struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}, 0, 10)
|
||||
err := d.callAPI(
|
||||
fmt.Sprintf("/api/issues/%d/fields", issue.ID),
|
||||
nil,
|
||||
&fields,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
var label *base.Label
|
||||
for _, field := range fields {
|
||||
if field.Name == "Type" {
|
||||
label = &base.Label{Name: field.Value}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
state := strings.ToLower(issue.State)
|
||||
if state == "released" {
|
||||
state = "closed"
|
||||
}
|
||||
poster := d.tryGetUser(issue.SubmitterID)
|
||||
issues = append(issues, &base.Issue{
|
||||
Title: issue.Title,
|
||||
Number: issue.Number,
|
||||
PosterName: poster.Name,
|
||||
PosterEmail: poster.Email,
|
||||
Content: issue.Description,
|
||||
Milestone: d.milestoneMap[issue.MilestoneID],
|
||||
State: state,
|
||||
Created: issue.SubmitDate,
|
||||
Updated: issue.SubmitDate,
|
||||
Labels: []*base.Label{label},
|
||||
Context: onedevIssueContext{
|
||||
foreignID: issue.ID,
|
||||
localID: issue.Number,
|
||||
IsPullRequest: false,
|
||||
},
|
||||
})
|
||||
|
||||
if d.maxIssueIndex < issue.Number {
|
||||
d.maxIssueIndex = issue.Number
|
||||
}
|
||||
}
|
||||
|
||||
return issues, len(issues) == 0, nil
|
||||
}
|
||||
|
||||
// GetComments returns comments
|
||||
func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
|
||||
context, ok := opts.Context.(onedevIssueContext)
|
||||
if !ok {
|
||||
return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context)
|
||||
}
|
||||
|
||||
rawComments := make([]struct {
|
||||
Date time.Time `json:"date"`
|
||||
UserID int64 `json:"userId"`
|
||||
Content string `json:"content"`
|
||||
}, 0, 100)
|
||||
|
||||
var endpoint string
|
||||
if context.IsPullRequest {
|
||||
endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", context.ForeignID())
|
||||
} else {
|
||||
endpoint = fmt.Sprintf("/api/issues/%d/comments", context.ForeignID())
|
||||
}
|
||||
|
||||
err := d.callAPI(
|
||||
endpoint,
|
||||
nil,
|
||||
&rawComments,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
rawChanges := make([]struct {
|
||||
Date time.Time `json:"date"`
|
||||
UserID int64 `json:"userId"`
|
||||
Data map[string]interface{} `json:"data"`
|
||||
}, 0, 100)
|
||||
|
||||
if context.IsPullRequest {
|
||||
endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", context.ForeignID())
|
||||
} else {
|
||||
endpoint = fmt.Sprintf("/api/issues/%d/changes", context.ForeignID())
|
||||
}
|
||||
|
||||
err = d.callAPI(
|
||||
endpoint,
|
||||
nil,
|
||||
&rawChanges,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges))
|
||||
for _, comment := range rawComments {
|
||||
if len(comment.Content) == 0 {
|
||||
continue
|
||||
}
|
||||
poster := d.tryGetUser(comment.UserID)
|
||||
comments = append(comments, &base.Comment{
|
||||
IssueIndex: context.LocalID(),
|
||||
PosterID: poster.ID,
|
||||
PosterName: poster.Name,
|
||||
PosterEmail: poster.Email,
|
||||
Content: comment.Content,
|
||||
Created: comment.Date,
|
||||
Updated: comment.Date,
|
||||
})
|
||||
}
|
||||
for _, change := range rawChanges {
|
||||
contentV, ok := change.Data["content"]
|
||||
if !ok {
|
||||
contentV, ok = change.Data["comment"]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
content, ok := contentV.(string)
|
||||
if !ok || len(content) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
poster := d.tryGetUser(change.UserID)
|
||||
comments = append(comments, &base.Comment{
|
||||
IssueIndex: context.LocalID(),
|
||||
PosterID: poster.ID,
|
||||
PosterName: poster.Name,
|
||||
PosterEmail: poster.Email,
|
||||
Content: content,
|
||||
Created: change.Date,
|
||||
Updated: change.Date,
|
||||
})
|
||||
}
|
||||
|
||||
return comments, true, nil
|
||||
}
|
||||
|
||||
// GetPullRequests returns pull requests
|
||||
func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
|
||||
rawPullRequests := make([]struct {
|
||||
ID int64 `json:"id"`
|
||||
Number int64 `json:"number"`
|
||||
Title string `json:"title"`
|
||||
SubmitterID int64 `json:"submitterId"`
|
||||
SubmitDate time.Time `json:"submitDate"`
|
||||
Description string `json:"description"`
|
||||
TargetBranch string `json:"targetBranch"`
|
||||
SourceBranch string `json:"sourceBranch"`
|
||||
BaseCommitHash string `json:"baseCommitHash"`
|
||||
CloseInfo *struct {
|
||||
Date *time.Time `json:"date"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
}, 0, perPage)
|
||||
|
||||
err := d.callAPI(
|
||||
"/api/pull-requests",
|
||||
map[string]string{
|
||||
"query": `"Target Project" is "` + d.repoName + `"`,
|
||||
"offset": strconv.Itoa((page - 1) * perPage),
|
||||
"count": strconv.Itoa(perPage),
|
||||
},
|
||||
&rawPullRequests,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests))
|
||||
for _, pr := range rawPullRequests {
|
||||
var mergePreview struct {
|
||||
TargetHeadCommitHash string `json:"targetHeadCommitHash"`
|
||||
HeadCommitHash string `json:"headCommitHash"`
|
||||
MergeStrategy string `json:"mergeStrategy"`
|
||||
MergeCommitHash string `json:"mergeCommitHash"`
|
||||
}
|
||||
err := d.callAPI(
|
||||
fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID),
|
||||
nil,
|
||||
&mergePreview,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
state := "open"
|
||||
merged := false
|
||||
var closeTime *time.Time
|
||||
var mergedTime *time.Time
|
||||
if pr.CloseInfo != nil {
|
||||
state = "closed"
|
||||
closeTime = pr.CloseInfo.Date
|
||||
if pr.CloseInfo.Status == "MERGED" { // "DISCARDED"
|
||||
merged = true
|
||||
mergedTime = pr.CloseInfo.Date
|
||||
}
|
||||
}
|
||||
poster := d.tryGetUser(pr.SubmitterID)
|
||||
|
||||
number := pr.Number + d.maxIssueIndex
|
||||
pullRequests = append(pullRequests, &base.PullRequest{
|
||||
Title: pr.Title,
|
||||
Number: number,
|
||||
PosterName: poster.Name,
|
||||
PosterID: poster.ID,
|
||||
Content: pr.Description,
|
||||
State: state,
|
||||
Created: pr.SubmitDate,
|
||||
Updated: pr.SubmitDate,
|
||||
Closed: closeTime,
|
||||
Merged: merged,
|
||||
MergedTime: mergedTime,
|
||||
Head: base.PullRequestBranch{
|
||||
Ref: pr.SourceBranch,
|
||||
SHA: mergePreview.HeadCommitHash,
|
||||
RepoName: d.repoName,
|
||||
},
|
||||
Base: base.PullRequestBranch{
|
||||
Ref: pr.TargetBranch,
|
||||
SHA: mergePreview.TargetHeadCommitHash,
|
||||
RepoName: d.repoName,
|
||||
},
|
||||
Context: onedevIssueContext{
|
||||
foreignID: pr.ID,
|
||||
localID: number,
|
||||
IsPullRequest: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return pullRequests, len(pullRequests) == 0, nil
|
||||
}
|
||||
|
||||
// GetReviews returns pull requests reviews
|
||||
func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
|
||||
rawReviews := make([]struct {
|
||||
ID int64 `json:"id"`
|
||||
UserID int64 `json:"userId"`
|
||||
Result *struct {
|
||||
Commit string `json:"commit"`
|
||||
Approved bool `json:"approved"`
|
||||
Comment string `json:"comment"`
|
||||
}
|
||||
}, 0, 100)
|
||||
|
||||
err := d.callAPI(
|
||||
fmt.Sprintf("/api/pull-requests/%d/reviews", context.ForeignID()),
|
||||
nil,
|
||||
&rawReviews,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var reviews = make([]*base.Review, 0, len(rawReviews))
|
||||
for _, review := range rawReviews {
|
||||
state := base.ReviewStatePending
|
||||
content := ""
|
||||
if review.Result != nil {
|
||||
if len(review.Result.Comment) > 0 {
|
||||
state = base.ReviewStateCommented
|
||||
content = review.Result.Comment
|
||||
}
|
||||
if review.Result.Approved {
|
||||
state = base.ReviewStateApproved
|
||||
}
|
||||
}
|
||||
|
||||
poster := d.tryGetUser(review.UserID)
|
||||
reviews = append(reviews, &base.Review{
|
||||
IssueIndex: context.LocalID(),
|
||||
ReviewerID: poster.ID,
|
||||
ReviewerName: poster.Name,
|
||||
Content: content,
|
||||
State: state,
|
||||
})
|
||||
}
|
||||
|
||||
return reviews, nil
|
||||
}
|
||||
|
||||
// GetTopics return repository topics
|
||||
func (d *OneDevDownloader) GetTopics() ([]string, error) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser {
|
||||
user, ok := d.userMap[userID]
|
||||
if !ok {
|
||||
err := d.callAPI(
|
||||
fmt.Sprintf("/api/users/%d", userID),
|
||||
nil,
|
||||
&user,
|
||||
)
|
||||
if err != nil {
|
||||
user = &onedevUser{
|
||||
Name: fmt.Sprintf("User %d", userID),
|
||||
}
|
||||
}
|
||||
d.userMap[userID] = user
|
||||
}
|
||||
|
||||
return user
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue