From dd08853b10781177253b581fde482fe67ab14edf Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Dec 2020 11:34:19 +0800 Subject: [PATCH] Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244) * Dump github/gitlab repository data to a local directory * Fix lint * Adjust directory structure * Allow migration special units * Allow migration ignore release assets * Fix lint * Add restore repository * stage the changes * Merge * Fix lint * Update the interface * Add some restore methods * Finish restore * Add comments * Fix restore * Add a token flag * Fix bug * Fix test * Fix test * Fix bug * Fix bug * Fix lint * Fix restore * refactor downloader * fmt * Fix bug isEnd detection on getIssues * Refactor maxPerPage * Remove unused codes * Remove unused codes * Fix bug * Fix restore * Fix dump * Uploader should not depend downloader * use release attachment name but not id * Fix restore bug * Fix lint * Fix restore bug * Add a method of DownloadFunc for base.Release to make uploader not depend on downloader * fix Release yml marshal * Fix trace information * Fix bug when dump & restore * Save relative path on yml file * Fix bug * Use relative path * Update docs * Use git service string but not int * Recognize clone addr to service type --- cmd/dump_repo.go | 162 +++++ cmd/restore_repo.go | 119 ++++ docs/content/doc/usage/command-line.en-us.md | 25 + main.go | 2 + models/admin.go | 13 + models/task.go | 4 - modules/migrations/base/comment.go | 8 +- modules/migrations/base/downloader.go | 7 - modules/migrations/base/issue.go | 8 +- modules/migrations/base/options.go | 1 + modules/migrations/base/pullrequest.go | 22 +- modules/migrations/base/reaction.go | 4 +- modules/migrations/base/release.go | 25 +- modules/migrations/base/repo.go | 8 +- modules/migrations/base/review.go | 26 +- modules/migrations/base/uploader.go | 3 +- modules/migrations/dump.go | 591 +++++++++++++++++++ modules/migrations/error.go | 3 + modules/migrations/git.go | 6 - modules/migrations/gitea_downloader.go | 31 +- modules/migrations/gitea_uploader.go | 60 +- modules/migrations/gitea_uploader_test.go | 1 + modules/migrations/github.go | 25 +- modules/migrations/gitlab.go | 44 +- modules/migrations/migrate.go | 182 +++--- modules/migrations/restore.go | 276 +++++++++ modules/uri/uri.go | 40 ++ modules/uri/uri_test.go | 20 + routers/api/v1/repo/migrate.go | 7 +- 29 files changed, 1491 insertions(+), 232 deletions(-) create mode 100644 cmd/dump_repo.go create mode 100644 cmd/restore_repo.go create mode 100644 modules/migrations/dump.go create mode 100644 modules/migrations/restore.go create mode 100644 modules/uri/uri.go create mode 100644 modules/uri/uri_test.go diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go new file mode 100644 index 0000000000..cea640b534 --- /dev/null +++ b/cmd/dump_repo.go @@ -0,0 +1,162 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "context" + "errors" + "strings" + + "code.gitea.io/gitea/modules/convert" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" + + "github.com/urfave/cli" +) + +// CmdDumpRepository represents the available dump repository sub-command. +var CmdDumpRepository = cli.Command{ + Name: "dump-repo", + Usage: "Dump the repository from git/github/gitea/gitlab", + Description: "This is a command for dumping the repository data.", + Action: runDumpRepository, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "git_service", + Value: "", + Usage: "Git service, git, github, gitea, gitlab. If clone_addr could be recognized, this could be ignored.", + }, + cli.StringFlag{ + Name: "repo_dir, r", + Value: "./data", + Usage: "Repository dir path to store the data", + }, + cli.StringFlag{ + Name: "clone_addr", + Value: "", + Usage: "The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL", + }, + cli.StringFlag{ + Name: "auth_username", + Value: "", + Usage: "The username to visit the clone_addr", + }, + cli.StringFlag{ + Name: "auth_password", + Value: "", + Usage: "The password to visit the clone_addr", + }, + cli.StringFlag{ + Name: "auth_token", + Value: "", + Usage: "The personal token to visit the clone_addr", + }, + cli.StringFlag{ + Name: "owner_name", + Value: "", + Usage: "The data will be stored on a directory with owner name if not empty", + }, + cli.StringFlag{ + Name: "repo_name", + Value: "", + Usage: "The data will be stored on a directory with repository name if not empty", + }, + cli.StringFlag{ + Name: "units", + Value: "", + Usage: `Which items will be migrated, one or more units should be separated as comma. +wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`, + }, + }, +} + +func runDumpRepository(ctx *cli.Context) error { + if err := initDB(); err != nil { + return err + } + + log.Trace("AppPath: %s", setting.AppPath) + log.Trace("AppWorkPath: %s", setting.AppWorkPath) + log.Trace("Custom path: %s", setting.CustomPath) + log.Trace("Log path: %s", setting.LogRootPath) + setting.InitDBConfig() + + var ( + serviceType structs.GitServiceType + cloneAddr = ctx.String("clone_addr") + serviceStr = ctx.String("git_service") + ) + + if strings.HasPrefix(strings.ToLower(cloneAddr), "https://github.com/") { + serviceStr = "github" + } else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitlab.com/") { + serviceStr = "gitlab" + } else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitea.com/") { + serviceStr = "gitea" + } + if serviceStr == "" { + return errors.New("git_service missed or clone_addr cannot be recognized") + } + serviceType = convert.ToGitServiceType(serviceStr) + + var opts = base.MigrateOptions{ + GitServiceType: serviceType, + CloneAddr: cloneAddr, + AuthUsername: ctx.String("auth_username"), + AuthPassword: ctx.String("auth_password"), + AuthToken: ctx.String("auth_token"), + RepoName: ctx.String("repo_name"), + } + + if len(ctx.String("units")) == 0 { + opts.Wiki = true + opts.Issues = true + opts.Milestones = true + opts.Labels = true + opts.Releases = true + opts.Comments = true + opts.PullRequests = true + opts.ReleaseAssets = true + } else { + units := strings.Split(ctx.String("units"), ",") + for _, unit := range units { + switch strings.ToLower(unit) { + case "wiki": + opts.Wiki = true + case "issues": + opts.Issues = true + case "milestones": + opts.Milestones = true + case "labels": + opts.Labels = true + case "releases": + opts.Releases = true + case "release_assets": + opts.ReleaseAssets = true + case "comments": + opts.Comments = true + case "pull_requests": + opts.PullRequests = true + } + } + } + + if err := migrations.DumpRepository( + context.Background(), + ctx.String("repo_dir"), + ctx.String("owner_name"), + opts, + ); err != nil { + log.Fatal("Failed to dump repository: %v", err) + return err + } + + log.Trace("Dump finished!!!") + + return nil +} diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go new file mode 100644 index 0000000000..541995879b --- /dev/null +++ b/cmd/restore_repo.go @@ -0,0 +1,119 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "context" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + pull_service "code.gitea.io/gitea/services/pull" + + "github.com/urfave/cli" +) + +// CmdRestoreRepository represents the available restore a repository sub-command. +var CmdRestoreRepository = cli.Command{ + Name: "restore-repo", + Usage: "Restore the repository from disk", + Description: "This is a command for restoring the repository data.", + Action: runRestoreRepository, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "repo_dir, r", + Value: "./data", + Usage: "Repository dir path to restore from", + }, + cli.StringFlag{ + Name: "owner_name", + Value: "", + Usage: "Restore destination owner name", + }, + cli.StringFlag{ + Name: "repo_name", + Value: "", + Usage: "Restore destination repository name", + }, + cli.StringFlag{ + Name: "units", + Value: "", + Usage: `Which items will be restored, one or more units should be separated as comma. +wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`, + }, + }, +} + +func runRestoreRepository(ctx *cli.Context) error { + if err := initDB(); err != nil { + return err + } + + log.Trace("AppPath: %s", setting.AppPath) + log.Trace("AppWorkPath: %s", setting.AppWorkPath) + log.Trace("Custom path: %s", setting.CustomPath) + log.Trace("Log path: %s", setting.LogRootPath) + setting.InitDBConfig() + + if err := storage.Init(); err != nil { + return err + } + + if err := pull_service.Init(); err != nil { + return err + } + + var opts = base.MigrateOptions{ + RepoName: ctx.String("repo_name"), + } + + if len(ctx.String("units")) == 0 { + opts.Wiki = true + opts.Issues = true + opts.Milestones = true + opts.Labels = true + opts.Releases = true + opts.Comments = true + opts.PullRequests = true + opts.ReleaseAssets = true + } else { + units := strings.Split(ctx.String("units"), ",") + for _, unit := range units { + switch strings.ToLower(unit) { + case "wiki": + opts.Wiki = true + case "issues": + opts.Issues = true + case "milestones": + opts.Milestones = true + case "labels": + opts.Labels = true + case "releases": + opts.Releases = true + case "release_assets": + opts.ReleaseAssets = true + case "comments": + opts.Comments = true + case "pull_requests": + opts.PullRequests = true + } + } + } + + if err := migrations.RestoreRepository( + context.Background(), + ctx.String("repo_dir"), + ctx.String("owner_name"), + ctx.String("repo_name"), + ); err != nil { + log.Fatal("Failed to restore repository: %v", err) + return err + } + + return nil +} diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index a09d5dde73..98d047fb48 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -441,3 +441,28 @@ Manage running server operations: - `--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25) - `--send-to value`, `-s value`: Email address(es) to send to - `--subject value`, `-S value`: Subject header of sent emails + +### dump-repo + +Dump-repo dumps repository data from git/github/gitea/gitlab: + +- Options: + - `--git_service service` : Git service, it could be `git`, `github`, `gitea`, `gitlab`, If clone_addr could be recognized, this could be ignored. + - `--repo_dir dir`, `-r dir`: Repository dir path to store the data + - `--clone_addr addr`: The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL. i.e. https://github.com/lunny/tango.git + - `--auth_username lunny`: The username to visit the clone_addr + - `--auth_password `: The password to visit the clone_addr + - `--auth_token `: The personal token to visit the clone_addr + - `--owner_name lunny`: The data will be stored on a directory with owner name if not empty + - `--repo_name tango`: The data will be stored on a directory with repository name if not empty + - `--units `: Which items will be migrated, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units. + +### restore-repo + +Restore-repo restore repository data from disk dir: + +- Options: + - `--repo_dir dir`, `-r dir`: Repository dir path to restore from + - `--owner_name lunny`: Restore destination owner name + - `--repo_name tango`: Restore destination repository name + - `--units `: Which items will be restored, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units. \ No newline at end of file diff --git a/main.go b/main.go index 8ee6ffa92c..6cbdc24401 100644 --- a/main.go +++ b/main.go @@ -72,6 +72,8 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.Cmdembedded, cmd.CmdMigrateStorage, cmd.CmdDocs, + cmd.CmdDumpRepository, + cmd.CmdRestoreRepository, } // Now adjust these commands to add our global configuration options diff --git a/models/admin.go b/models/admin.go index 420adbcda9..4635676d0c 100644 --- a/models/admin.go +++ b/models/admin.go @@ -132,3 +132,16 @@ func DeleteNoticesByIDs(ids []int64) error { Delete(new(Notice)) return err } + +// GetAdminUser returns the first administrator +func GetAdminUser() (*User, error) { + var admin User + has, err := x.Where("is_admin=?", true).Get(&admin) + if err != nil { + return nil, err + } else if !has { + return nil, ErrUserNotExist{} + } + + return &admin, nil +} diff --git a/models/task.go b/models/task.go index b86314b449..b729bb8632 100644 --- a/models/task.go +++ b/models/task.go @@ -211,10 +211,6 @@ func FinishMigrateTask(task *Task) error { if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil { return err } - task.Repo.Status = RepositoryReady - if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil { - return err - } return sess.Commit() } diff --git a/modules/migrations/base/comment.go b/modules/migrations/base/comment.go index 4a653e474b..3c32e63b82 100644 --- a/modules/migrations/base/comment.go +++ b/modules/migrations/base/comment.go @@ -9,10 +9,10 @@ import "time" // Comment is a standard comment information type Comment struct { - IssueIndex int64 - PosterID int64 - PosterName string - PosterEmail string + IssueIndex int64 `yaml:"issue_index"` + PosterID int64 `yaml:"poster_id"` + PosterName string `yaml:"poster_name"` + PosterEmail string `yaml:"poster_email"` Created time.Time Updated time.Time Content string diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 5c47ed5305..afa99105c9 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -7,20 +7,13 @@ package base import ( "context" - "io" "time" "code.gitea.io/gitea/modules/structs" ) -// AssetDownloader downloads an asset (attachment) for a release -type AssetDownloader interface { - GetAsset(relTag string, relID, id int64) (io.ReadCloser, error) -} - // Downloader downloads the site repo informations type Downloader interface { - AssetDownloader SetContext(context.Context) GetRepoInfo() (*Repository, error) GetTopics() ([]string, error) diff --git a/modules/migrations/base/issue.go b/modules/migrations/base/issue.go index f9dc8b93fe..8b1b461244 100644 --- a/modules/migrations/base/issue.go +++ b/modules/migrations/base/issue.go @@ -10,15 +10,15 @@ import "time" // Issue is a standard issue information type Issue struct { Number int64 - PosterID int64 - PosterName string - PosterEmail string + PosterID int64 `yaml:"poster_id"` + PosterName string `yaml:"poster_name"` + PosterEmail string `yaml:"poster_email"` Title string Content string Ref string Milestone string State string // closed, open - IsLocked bool + IsLocked bool `yaml:"is_locked"` Created time.Time Updated time.Time Closed *time.Time diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go index dbc40b138a..3c9b2c22fc 100644 --- a/modules/migrations/base/options.go +++ b/modules/migrations/base/options.go @@ -31,5 +31,6 @@ type MigrateOptions struct { Releases bool Comments bool PullRequests bool + ReleaseAssets bool MigrateToRepoID int64 } diff --git a/modules/migrations/base/pullrequest.go b/modules/migrations/base/pullrequest.go index ee612fbb8e..6411137d0a 100644 --- a/modules/migrations/base/pullrequest.go +++ b/modules/migrations/base/pullrequest.go @@ -13,11 +13,11 @@ import ( // PullRequest defines a standard pull request information type PullRequest struct { Number int64 - OriginalNumber int64 + OriginalNumber int64 `yaml:"original_number"` Title string - PosterName string - PosterID int64 - PosterEmail string + PosterName string `yaml:"poster_name"` + PosterID int64 `yaml:"poster_id"` + PosterEmail string `yaml:"poster_email"` Content string Milestone string State string @@ -25,14 +25,14 @@ type PullRequest struct { Updated time.Time Closed *time.Time Labels []*Label - PatchURL string + PatchURL string `yaml:"patch_url"` Merged bool - MergedTime *time.Time - MergeCommitSHA string + MergedTime *time.Time `yaml:"merged_time"` + MergeCommitSHA string `yaml:"merge_commit_sha"` Head PullRequestBranch Base PullRequestBranch Assignees []string - IsLocked bool + IsLocked bool `yaml:"is_locked"` Reactions []*Reaction } @@ -43,11 +43,11 @@ func (p *PullRequest) IsForkPullRequest() bool { // PullRequestBranch represents a pull request branch type PullRequestBranch struct { - CloneURL string + CloneURL string `yaml:"clone_url"` Ref string SHA string - RepoName string - OwnerName string + RepoName string `yaml:"repo_name"` + OwnerName string `yaml:"owner_name"` } // RepoPath returns pull request repo path diff --git a/modules/migrations/base/reaction.go b/modules/migrations/base/reaction.go index b79223d4cd..1519499134 100644 --- a/modules/migrations/base/reaction.go +++ b/modules/migrations/base/reaction.go @@ -6,7 +6,7 @@ package base // Reaction represents a reaction to an issue/pr/comment. type Reaction struct { - UserID int64 - UserName string + UserID int64 `yaml:"user_id"` + UserName string `yaml:"user_name"` Content string } diff --git a/modules/migrations/base/release.go b/modules/migrations/base/release.go index c9b26ab1da..8b4339928b 100644 --- a/modules/migrations/base/release.go +++ b/modules/migrations/base/release.go @@ -4,32 +4,37 @@ package base -import "time" +import ( + "io" + "time" +) // ReleaseAsset represents a release asset type ReleaseAsset struct { ID int64 Name string - ContentType *string + ContentType *string `yaml:"content_type"` Size *int - DownloadCount *int + DownloadCount *int `yaml:"download_count"` Created time.Time Updated time.Time - DownloadURL *string + DownloadURL *string `yaml:"download_url"` + // if DownloadURL is nil, the function should be invoked + DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` } // Release represents a release type Release struct { - TagName string - TargetCommitish string + TagName string `yaml:"tag_name"` + TargetCommitish string `yaml:"target_commitish"` Name string Body string Draft bool Prerelease bool - PublisherID int64 - PublisherName string - PublisherEmail string - Assets []ReleaseAsset + PublisherID int64 `yaml:"publisher_id"` + PublisherName string `yaml:"publisher_name"` + PublisherEmail string `yaml:"publisher_email"` + Assets []*ReleaseAsset Created time.Time Published time.Time } diff --git a/modules/migrations/base/repo.go b/modules/migrations/base/repo.go index d26a911854..693a96314d 100644 --- a/modules/migrations/base/repo.go +++ b/modules/migrations/base/repo.go @@ -9,10 +9,10 @@ package base type Repository struct { Name string Owner string - IsPrivate bool - IsMirror bool + IsPrivate bool `yaml:"is_private"` + IsMirror bool `yaml:"is_mirror"` Description string - CloneURL string - OriginalURL string + CloneURL string `yaml:"clone_url"` + OriginalURL string `yaml:"original_url"` DefaultBranch string } diff --git a/modules/migrations/base/review.go b/modules/migrations/base/review.go index 0a9d03dae9..6344f0384d 100644 --- a/modules/migrations/base/review.go +++ b/modules/migrations/base/review.go @@ -17,29 +17,29 @@ const ( // Review is a standard review information type Review struct { ID int64 - IssueIndex int64 - ReviewerID int64 - ReviewerName string + IssueIndex int64 `yaml:"issue_index"` + ReviewerID int64 `yaml:"reviewer_id"` + ReviewerName string `yaml:"reviewer_name"` Official bool - CommitID string + CommitID string `yaml:"commit_id"` Content string - CreatedAt time.Time - State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT + CreatedAt time.Time `yaml:"created_at"` + State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT Comments []*ReviewComment } // ReviewComment represents a review comment type ReviewComment struct { ID int64 - InReplyTo int64 + InReplyTo int64 `yaml:"in_reply_to"` Content string - TreePath string - DiffHunk string + TreePath string `yaml:"tree_path"` + DiffHunk string `yaml:"diff_hunk"` Position int Line int - CommitID string - PosterID int64 + CommitID string `yaml:"commit_id"` + PosterID int64 `yaml:"poster_id"` Reactions []*Reaction - CreatedAt time.Time - UpdatedAt time.Time + CreatedAt time.Time `yaml:"created_at"` + UpdatedAt time.Time `yaml:"updated_at"` } diff --git a/modules/migrations/base/uploader.go b/modules/migrations/base/uploader.go index 07c2bb0d42..dfcf81d052 100644 --- a/modules/migrations/base/uploader.go +++ b/modules/migrations/base/uploader.go @@ -11,7 +11,7 @@ type Uploader interface { CreateRepo(repo *Repository, opts MigrateOptions) error CreateTopics(topic ...string) error CreateMilestones(milestones ...*Milestone) error - CreateReleases(downloader Downloader, releases ...*Release) error + CreateReleases(releases ...*Release) error SyncTags() error CreateLabels(labels ...*Label) error CreateIssues(issues ...*Issue) error @@ -19,5 +19,6 @@ type Uploader interface { CreatePullRequests(prs ...*PullRequest) error CreateReviews(reviews ...*Review) error Rollback() error + Finish() error Close() } diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go new file mode 100644 index 0000000000..3c3b9a1753 --- /dev/null +++ b/modules/migrations/dump.go @@ -0,0 +1,591 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/repository" + + "gopkg.in/yaml.v2" +) + +var ( + _ base.Uploader = &RepositoryDumper{} +) + +// RepositoryDumper implements an Uploader to the local directory +type RepositoryDumper struct { + ctx context.Context + baseDir string + repoOwner string + repoName string + opts base.MigrateOptions + milestoneFile *os.File + labelFile *os.File + releaseFile *os.File + issueFile *os.File + commentFiles map[int64]*os.File + pullrequestFile *os.File + reviewFiles map[int64]*os.File + + gitRepo *git.Repository + prHeadCache map[string]struct{} +} + +// NewRepositoryDumper creates an gitea Uploader +func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) (*RepositoryDumper, error) { + baseDir = filepath.Join(baseDir, repoOwner, repoName) + if err := os.MkdirAll(baseDir, os.ModePerm); err != nil { + return nil, err + } + return &RepositoryDumper{ + ctx: ctx, + opts: opts, + baseDir: baseDir, + repoOwner: repoOwner, + repoName: repoName, + prHeadCache: make(map[string]struct{}), + commentFiles: make(map[int64]*os.File), + reviewFiles: make(map[int64]*os.File), + }, nil +} + +// MaxBatchInsertSize returns the table's max batch insert size +func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int { + return 1000 +} + +func (g *RepositoryDumper) gitPath() string { + return filepath.Join(g.baseDir, "git") +} + +func (g *RepositoryDumper) wikiPath() string { + return filepath.Join(g.baseDir, "wiki") +} + +func (g *RepositoryDumper) commentDir() string { + return filepath.Join(g.baseDir, "comments") +} + +func (g *RepositoryDumper) reviewDir() string { + return filepath.Join(g.baseDir, "reviews") +} + +func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) { + if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 { + u, err := url.Parse(remoteAddr) + if err != nil { + return "", err + } + u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword) + if len(g.opts.AuthToken) > 0 { + u.User = url.UserPassword("oauth2", g.opts.AuthToken) + } + remoteAddr = u.String() + } + + return remoteAddr, nil +} + +// CreateRepo creates a repository +func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { + f, err := os.Create(filepath.Join(g.baseDir, "repo.yml")) + if err != nil { + return err + } + defer f.Close() + + bs, err := yaml.Marshal(map[string]interface{}{ + "name": repo.Name, + "owner": repo.Owner, + "description": repo.Description, + "clone_addr": opts.CloneAddr, + "original_url": repo.OriginalURL, + "is_private": opts.Private, + "service_type": opts.GitServiceType, + "wiki": opts.Wiki, + "issues": opts.Issues, + "milestones": opts.Milestones, + "labels": opts.Labels, + "releases": opts.Releases, + "comments": opts.Comments, + "pulls": opts.PullRequests, + "assets": opts.ReleaseAssets, + }) + if err != nil { + return err + } + + if _, err := f.Write(bs); err != nil { + return err + } + + repoPath := g.gitPath() + if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { + return err + } + + migrateTimeout := 2 * time.Hour + + remoteAddr, err := g.setURLToken(repo.CloneURL) + if err != nil { + return err + } + + err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + }) + if err != nil { + return fmt.Errorf("Clone: %v", err) + } + + if opts.Wiki { + wikiPath := g.wikiPath() + wikiRemotePath := repository.WikiRemoteURL(remoteAddr) + if len(wikiRemotePath) > 0 { + if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil { + return fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + } + + if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + Branch: "master", + }); err != nil { + log.Warn("Clone wiki: %v", err) + if err := os.RemoveAll(wikiPath); err != nil { + return fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + } + } + } + } + + g.gitRepo, err = git.OpenRepository(g.gitPath()) + return err +} + +// Close closes this uploader +func (g *RepositoryDumper) Close() { + if g.gitRepo != nil { + g.gitRepo.Close() + } + if g.milestoneFile != nil { + g.milestoneFile.Close() + } + if g.labelFile != nil { + g.labelFile.Close() + } + if g.releaseFile != nil { + g.releaseFile.Close() + } + if g.issueFile != nil { + g.issueFile.Close() + } + for _, f := range g.commentFiles { + f.Close() + } + if g.pullrequestFile != nil { + g.pullrequestFile.Close() + } + for _, f := range g.reviewFiles { + f.Close() + } +} + +// CreateTopics creates topics +func (g *RepositoryDumper) CreateTopics(topics ...string) error { + f, err := os.Create(filepath.Join(g.baseDir, "topic.yml")) + if err != nil { + return err + } + defer f.Close() + + bs, err := yaml.Marshal(map[string]interface{}{ + "topics": topics, + }) + if err != nil { + return err + } + + if _, err := f.Write(bs); err != nil { + return err + } + + return nil +} + +// CreateMilestones creates milestones +func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error { + var err error + if g.milestoneFile == nil { + g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(milestones) + if err != nil { + return err + } + + if _, err := g.milestoneFile.Write(bs); err != nil { + return err + } + + return nil +} + +// CreateLabels creates labels +func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { + var err error + if g.labelFile == nil { + g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(labels) + if err != nil { + return err + } + + if _, err := g.labelFile.Write(bs); err != nil { + return err + } + + return nil +} + +// CreateReleases creates releases +func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { + if g.opts.ReleaseAssets { + for _, release := range releases { + attachDir := filepath.Join("release_assets", release.TagName) + if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil { + return err + } + for _, asset := range release.Assets { + attachLocalPath := filepath.Join(attachDir, asset.Name) + // download attachment + + err := func(attachPath string) error { + var rc io.ReadCloser + var err error + if asset.DownloadURL == nil { + rc, err = asset.DownloadFunc() + if err != nil { + return err + } + } else { + resp, err := http.Get(*asset.DownloadURL) + if err != nil { + return err + } + rc = resp.Body + } + defer rc.Close() + + fw, err := os.Create(attachPath) + if err != nil { + return fmt.Errorf("Create: %v", err) + } + defer fw.Close() + + _, err = io.Copy(fw, rc) + return err + }(filepath.Join(g.baseDir, attachLocalPath)) + if err != nil { + return err + } + asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source + } + } + } + + var err error + if g.releaseFile == nil { + g.releaseFile, err = os.Create(filepath.Join(g.baseDir, "release.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(releases) + if err != nil { + return err + } + + if _, err := g.releaseFile.Write(bs); err != nil { + return err + } + + return nil +} + +// SyncTags syncs releases with tags in the database +func (g *RepositoryDumper) SyncTags() error { + return nil +} + +// CreateIssues creates issues +func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { + var err error + if g.issueFile == nil { + g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(issues) + if err != nil { + return err + } + + if _, err := g.issueFile.Write(bs); err != nil { + return err + } + + return nil +} + +func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, itemsMap map[int64][]interface{}) error { + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + + for number, items := range itemsMap { + var err error + itemFile := itemFiles[number] + if itemFile == nil { + itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number))) + if err != nil { + return err + } + itemFiles[number] = itemFile + } + + bs, err := yaml.Marshal(items) + if err != nil { + return err + } + + if _, err := itemFile.Write(bs); err != nil { + return err + } + } + + return nil +} + +// CreateComments creates comments of issues +func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { + var commentsMap = make(map[int64][]interface{}, len(comments)) + for _, comment := range comments { + commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment) + } + + return g.createItems(g.commentDir(), g.commentFiles, commentsMap) +} + +// CreatePullRequests creates pull requests +func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { + for _, pr := range prs { + // download patch file + err := func() error { + u, err := g.setURLToken(pr.PatchURL) + if err != nil { + return err + } + resp, err := http.Get(u) + if err != nil { + return err + } + defer resp.Body.Close() + pullDir := filepath.Join(g.gitPath(), "pulls") + if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { + return err + } + fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)) + f, err := os.Create(fPath) + if err != nil { + return err + } + defer f.Close() + if _, err = io.Copy(f, resp.Body); err != nil { + return err + } + pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number) + + return nil + }() + if err != nil { + return err + } + + // set head information + pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) + if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { + return err + } + p, err := os.Create(filepath.Join(pullHead, "head")) + if err != nil { + return err + } + _, err = p.WriteString(pr.Head.SHA) + p.Close() + if err != nil { + return err + } + + if pr.IsForkPullRequest() && pr.State != "closed" { + if pr.Head.OwnerName != "" { + remote := pr.Head.OwnerName + _, ok := g.prHeadCache[remote] + if !ok { + // git remote add + // TODO: how to handle private CloneURL? + err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) + if err != nil { + log.Error("AddRemote failed: %s", err) + } else { + g.prHeadCache[remote] = struct{}{} + ok = true + } + } + + if ok { + _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.gitPath()) + if err != nil { + log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) + } else { + headBranch := filepath.Join(g.gitPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) + if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { + return err + } + b, err := os.Create(headBranch) + if err != nil { + return err + } + _, err = b.WriteString(pr.Head.SHA) + b.Close() + if err != nil { + return err + } + } + } + } + } + } + + var err error + if g.pullrequestFile == nil { + if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { + return err + } + g.pullrequestFile, err = os.Create(filepath.Join(g.baseDir, "pull_request.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(prs) + if err != nil { + return err + } + + if _, err := g.pullrequestFile.Write(bs); err != nil { + return err + } + + return nil +} + +// CreateReviews create pull request reviews +func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { + var reviewsMap = make(map[int64][]interface{}, len(reviews)) + for _, review := range reviews { + reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review) + } + + return g.createItems(g.reviewDir(), g.reviewFiles, reviewsMap) +} + +// Rollback when migrating failed, this will rollback all the changes. +func (g *RepositoryDumper) Rollback() error { + g.Close() + return os.RemoveAll(g.baseDir) +} + +// Finish when migrating succeed, this will update something. +func (g *RepositoryDumper) Finish() error { + return nil +} + +// DumpRepository dump repository according MigrateOptions to a local directory +func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error { + downloader, err := newDownloader(ctx, ownerName, opts) + if err != nil { + return err + } + uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts) + if err != nil { + return err + } + + if err := migrateRepository(downloader, uploader, opts); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } + return err + } + return nil +} + +// RestoreRepository restore a repository from the disk directory +func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error { + doer, err := models.GetAdminUser() + if err != nil { + return err + } + var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) + downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) + if err != nil { + return err + } + if err = migrateRepository(downloader, uploader, base.MigrateOptions{ + Wiki: true, + Issues: true, + Milestones: true, + Labels: true, + Releases: true, + Comments: true, + PullRequests: true, + ReleaseAssets: true, + }); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } + return err + } + return nil +} diff --git a/modules/migrations/error.go b/modules/migrations/error.go index b2e2315fc8..462ba29026 100644 --- a/modules/migrations/error.go +++ b/modules/migrations/error.go @@ -14,6 +14,9 @@ import ( var ( // ErrNotSupported returns the error not supported ErrNotSupported = errors.New("not supported") + + // ErrRepoNotCreated returns the error that repository not created + ErrRepoNotCreated = errors.New("repository is not created yet") ) // IsRateLimitError returns true if the err is github.RateLimitError diff --git a/modules/migrations/git.go b/modules/migrations/git.go index 0aad8dbef5..88222086e4 100644 --- a/modules/migrations/git.go +++ b/modules/migrations/git.go @@ -6,7 +6,6 @@ package migrations import ( "context" - "io" "code.gitea.io/gitea/modules/migrations/base" ) @@ -65,11 +64,6 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) { return nil, ErrNotSupported } -// GetAsset returns an asset -func (g *PlainGitDownloader) GetAsset(_ string, _, _ int64) (io.ReadCloser, error) { - return nil, ErrNotSupported -} - // GetIssues returns issues according page and perPage func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { return nil, false, ErrNotSupported diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index 0509c708bf..0c690464fa 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -268,13 +268,27 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele for _, asset := range rel.Attachments { size := int(asset.Size) dlCount := int(asset.DownloadCount) - r.Assets = append(r.Assets, base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.ReleaseAsset{ ID: asset.ID, Name: asset.Name, Size: &size, DownloadCount: &dlCount, Created: asset.Created, DownloadURL: &asset.DownloadURL, + DownloadFunc: func() (io.ReadCloser, error) { + asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, rel.ID, asset.ID) + if err != nil { + return nil, err + } + // FIXME: for a private download? + resp, err := http.Get(asset.DownloadURL) + if err != nil { + return nil, err + } + + // resp.Body is closed by the uploader + return resp.Body, nil + }, }) } return r @@ -310,21 +324,6 @@ func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) { return releases, nil } -// GetAsset returns an asset -func (g *GiteaDownloader) GetAsset(_ string, relID, id int64) (io.ReadCloser, error) { - asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, relID, id) - if err != nil { - return nil, err - } - resp, err := http.Get(asset.DownloadURL) - if err != nil { - return nil, err - } - - // resp.Body is closed by the uploader - return resp.Body, nil -} - func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) { var reactions []*base.Reaction if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil { diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 91ddda9c39..6118b3b5c1 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -10,7 +10,6 @@ import ( "context" "fmt" "io" - "net/http" "net/url" "os" "path/filepath" @@ -28,6 +27,7 @@ import ( "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/uri" "code.gitea.io/gitea/services/pull" gouuid "github.com/google/uuid" @@ -86,6 +86,22 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int { return 10 } +func fullURL(opts base.MigrateOptions, remoteAddr string) (string, error) { + var fullRemoteAddr = remoteAddr + if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { + u, err := url.Parse(remoteAddr) + if err != nil { + return "", err + } + u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) + if len(opts.AuthToken) > 0 { + u.User = url.UserPassword("oauth2", opts.AuthToken) + } + fullRemoteAddr = u.String() + } + return fullRemoteAddr, nil +} + // CreateRepo creates a repository func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { owner, err := models.GetUserByName(g.repoOwner) @@ -93,19 +109,10 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate return err } - var remoteAddr = repo.CloneURL - if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { - u, err := url.Parse(repo.CloneURL) - if err != nil { - return err - } - u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) - if len(opts.AuthToken) > 0 { - u.User = url.UserPassword("oauth2", opts.AuthToken) - } - remoteAddr = u.String() + remoteAddr, err := fullURL(opts, repo.CloneURL) + if err != nil { + return err } - var r *models.Repository if opts.MigrateToRepoID <= 0 { r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{ @@ -224,7 +231,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { } // CreateReleases creates releases -func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error { +func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { var rels = make([]*models.Release, 0, len(releases)) for _, release := range releases { var rel = models.Release{ @@ -283,25 +290,27 @@ func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases // download attachment err = func() error { + // asset.DownloadURL maybe a local file var rc io.ReadCloser if asset.DownloadURL == nil { - rc, err = downloader.GetAsset(rel.TagName, rel.ID, asset.ID) + rc, err = asset.DownloadFunc() if err != nil { return err } } else { - resp, err := http.Get(*asset.DownloadURL) + rc, err = uri.Open(*asset.DownloadURL) if err != nil { return err } - rc = resp.Body } + defer rc.Close() _, err = storage.Attachments.Save(attach.RelativePath(), rc) return err }() if err != nil { return err } + rel.Attachments = append(rel.Attachments, &attach) } @@ -559,11 +568,12 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR // download patch file err := func() error { - resp, err := http.Get(pr.PatchURL) + // pr.PatchURL maybe a local file + ret, err := uri.Open(pr.PatchURL) if err != nil { return err } - defer resp.Body.Close() + defer ret.Close() pullDir := filepath.Join(g.repo.RepoPath(), "pulls") if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { return err @@ -573,7 +583,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR return err } defer f.Close() - _, err = io.Copy(f, resp.Body) + _, err = io.Copy(f, ret) return err }() if err != nil { @@ -859,3 +869,13 @@ func (g *GiteaLocalUploader) Rollback() error { } return nil } + +// Finish when migrating success, this will do some status update things. +func (g *GiteaLocalUploader) Finish() error { + if g.repo == nil || g.repo.ID <= 0 { + return ErrRepoNotCreated + } + + g.repo.Status = models.RepositoryReady + return models.UpdateRepositoryCols(g.repo, "status") +} diff --git a/modules/migrations/gitea_uploader_test.go b/modules/migrations/gitea_uploader_test.go index 8432a1eecd..3c7def4675 100644 --- a/modules/migrations/gitea_uploader_test.go +++ b/modules/migrations/gitea_uploader_test.go @@ -52,6 +52,7 @@ func TestGiteaUploadRepo(t *testing.T) { repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository) assert.True(t, repo.HasWiki()) + assert.EqualValues(t, models.RepositoryReady, repo.Status) milestones, err := models.GetMilestones(models.GetMilestonesOption{ RepoID: repo.ID, diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 7aa1e57274..178517ba42 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -291,7 +291,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) } for _, asset := range rel.Assets { - r.Assets = append(r.Assets, base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.ReleaseAsset{ ID: *asset.ID, Name: *asset.Name, ContentType: asset.ContentType, @@ -299,6 +299,16 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) DownloadCount: asset.DownloadCount, Created: asset.CreatedAt.Time, Updated: asset.UpdatedAt.Time, + DownloadFunc: func() (io.ReadCloser, error) { + asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, *asset.ID, http.DefaultClient) + if err != nil { + return nil, err + } + if asset == nil { + return ioutil.NopCloser(bytes.NewBufferString(redir)), nil + } + return asset, nil + }, }) } return r @@ -330,18 +340,6 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { return releases, nil } -// GetAsset returns an asset -func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, error) { - asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, id, http.DefaultClient) - if err != nil { - return nil, err - } - if asset == nil { - return ioutil.NopCloser(bytes.NewBufferString(redir)), nil - } - return asset, nil -} - // GetIssues returns issues according start and limit func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { @@ -363,6 +361,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } + log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues)) g.rate = &resp.Rate for _, issue := range issues { if issue.IsPullRequest() { diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index b1027c4f64..e3fa956758 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -295,12 +295,32 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea } for k, asset := range rel.Assets.Links { - r.Assets = append(r.Assets, base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.ReleaseAsset{ ID: int64(asset.ID), Name: asset.Name, ContentType: &rel.Assets.Sources[k].Format, Size: &zero, DownloadCount: &zero, + DownloadFunc: func() (io.ReadCloser, error) { + link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, asset.ID, gitlab.WithContext(g.ctx)) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", link.URL, nil) + if err != nil { + return nil, err + } + req = req.WithContext(g.ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + // resp.Body is closed by the uploader + return resp.Body, nil + }, }) } return r @@ -329,28 +349,6 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { return releases, nil } -// GetAsset returns an asset -func (g *GitlabDownloader) GetAsset(tag string, _, id int64) (io.ReadCloser, error) { - link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, tag, int(id), gitlab.WithContext(g.ctx)) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("GET", link.URL, nil) - if err != nil { - return nil, err - } - req = req.WithContext(g.ctx) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - - // resp.Body is closed by the uploader - return resp.Body, nil -} - // GetIssues returns issues according start and limit // Note: issue label description and colors are not supported by the go-gitlab library at this time func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index b3ecb8114a..4c15626e57 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -73,10 +73,30 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, if err != nil { return nil, err } + downloader, err := newDownloader(ctx, ownerName, opts) + if err != nil { + return nil, err + } + var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) + uploader.gitServiceType = opts.GitServiceType + + if err := migrateRepository(downloader, uploader, opts); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } + if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { + log.Error("create respotiry notice failed: ", err2) + } + return nil, err + } + return uploader.repo, nil +} + +func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) { var ( downloader base.Downloader - uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) + err error ) for _, factory := range factories { @@ -101,24 +121,10 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, log.Trace("Will migrate from git: %s", opts.OriginalURL) } - uploader.gitServiceType = opts.GitServiceType - if setting.Migrations.MaxAttempts > 1 { downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) } - - if err := migrateRepository(downloader, uploader, opts); err != nil { - if err1 := uploader.Rollback(); err1 != nil { - log.Error("rollback failed: %v", err1) - } - - if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { - log.Error("create repository notice failed: ", err2) - } - return nil, err - } - - return uploader.repo, nil + return downloader, nil } // migrateRepository will download information and then upload it to Uploader, this is a simple @@ -204,7 +210,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts relBatchSize = len(releases) } - if err := uploader.CreateReleases(downloader, releases[:relBatchSize]...); err != nil { + if err := uploader.CreateReleases(releases[:relBatchSize]...); err != nil { return err } releases = releases[relBatchSize:] @@ -235,31 +241,30 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return err } - if !opts.Comments { - continue - } - - var allComments = make([]*base.Comment, 0, commentBatchSize) - for _, issue := range issues { - comments, err := downloader.GetComments(issue.Number) - if err != nil { - return err - } - - allComments = append(allComments, comments...) - - if len(allComments) >= commentBatchSize { - if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + if opts.Comments { + var allComments = make([]*base.Comment, 0, commentBatchSize) + for _, issue := range issues { + log.Trace("migrating issue %d's comments", issue.Number) + comments, err := downloader.GetComments(issue.Number) + if err != nil { return err } - allComments = allComments[commentBatchSize:] - } - } + allComments = append(allComments, comments...) - if len(allComments) > 0 { - if err := uploader.CreateComments(allComments...); err != nil { - return err + if len(allComments) >= commentBatchSize { + if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + return err + } + + allComments = allComments[commentBatchSize:] + } + } + + if len(allComments) > 0 { + if err := uploader.CreateComments(allComments...); err != nil { + return err + } } } @@ -282,65 +287,64 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return err } - if !opts.Comments { - continue - } - - // plain comments - var allComments = make([]*base.Comment, 0, commentBatchSize) - for _, pr := range prs { - comments, err := downloader.GetComments(pr.Number) - if err != nil { - return err - } - - allComments = append(allComments, comments...) - - if len(allComments) >= commentBatchSize { - if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + if opts.Comments { + // plain comments + var allComments = make([]*base.Comment, 0, commentBatchSize) + for _, pr := range prs { + log.Trace("migrating pull request %d's comments", pr.Number) + comments, err := downloader.GetComments(pr.Number) + if err != nil { return err } - allComments = allComments[commentBatchSize:] - } - } - if len(allComments) > 0 { - if err := uploader.CreateComments(allComments...); err != nil { - return err - } - } - // migrate reviews - var allReviews = make([]*base.Review, 0, reviewBatchSize) - for _, pr := range prs { - number := pr.Number + allComments = append(allComments, comments...) - // on gitlab migrations pull number change - if pr.OriginalNumber > 0 { - number = pr.OriginalNumber - } - - reviews, err := downloader.GetReviews(number) - if pr.OriginalNumber > 0 { - for i := range reviews { - reviews[i].IssueIndex = pr.Number + if len(allComments) >= commentBatchSize { + if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + return err + } + allComments = allComments[commentBatchSize:] } } - if err != nil { - return err - } - - allReviews = append(allReviews, reviews...) - - if len(allReviews) >= reviewBatchSize { - if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { + if len(allComments) > 0 { + if err := uploader.CreateComments(allComments...); err != nil { return err } - allReviews = allReviews[reviewBatchSize:] } - } - if len(allReviews) > 0 { - if err := uploader.CreateReviews(allReviews...); err != nil { - return err + + // migrate reviews + var allReviews = make([]*base.Review, 0, reviewBatchSize) + for _, pr := range prs { + number := pr.Number + + // on gitlab migrations pull number change + if pr.OriginalNumber > 0 { + number = pr.OriginalNumber + } + + reviews, err := downloader.GetReviews(number) + if pr.OriginalNumber > 0 { + for i := range reviews { + reviews[i].IssueIndex = pr.Number + } + } + if err != nil { + return err + } + + allReviews = append(allReviews, reviews...) + + if len(allReviews) >= reviewBatchSize { + if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { + return err + } + allReviews = allReviews[reviewBatchSize:] + } + } + if len(allReviews) > 0 { + if err := uploader.CreateReviews(allReviews...); err != nil { + return err + } } } @@ -350,7 +354,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } } - return nil + return uploader.Finish() } // Init migrations service diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go new file mode 100644 index 0000000000..5550aaeb03 --- /dev/null +++ b/modules/migrations/restore.go @@ -0,0 +1,276 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + + "code.gitea.io/gitea/modules/migrations/base" + + "gopkg.in/yaml.v2" +) + +// RepositoryRestorer implements an Downloader from the local directory +type RepositoryRestorer struct { + ctx context.Context + baseDir string + repoOwner string + repoName string +} + +// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder +func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) { + baseDir, err := filepath.Abs(baseDir) + if err != nil { + return nil, err + } + return &RepositoryRestorer{ + ctx: ctx, + baseDir: baseDir, + repoOwner: owner, + repoName: repoName, + }, nil +} + +func (r *RepositoryRestorer) commentDir() string { + return filepath.Join(r.baseDir, "comments") +} + +func (r *RepositoryRestorer) reviewDir() string { + return filepath.Join(r.baseDir, "reviews") +} + +// SetContext set context +func (r *RepositoryRestorer) SetContext(ctx context.Context) { + r.ctx = ctx +} + +// GetRepoInfo returns a repository information +func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { + p := filepath.Join(r.baseDir, "repo.yml") + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + var opts = make(map[string]string) + err = yaml.Unmarshal(bs, &opts) + if err != nil { + return nil, err + } + + isPrivate, _ := strconv.ParseBool(opts["is_private"]) + + return &base.Repository{ + Owner: r.repoOwner, + Name: r.repoName, + IsPrivate: isPrivate, + Description: opts["description"], + OriginalURL: opts["original_url"], + CloneURL: opts["clone_addr"], + DefaultBranch: opts["default_branch"], + }, nil +} + +// GetTopics return github topics +func (r *RepositoryRestorer) GetTopics() ([]string, error) { + p := filepath.Join(r.baseDir, "topic.yml") + + var topics = struct { + Topics []string `yaml:"topics"` + }{} + + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(bs, &topics) + if err != nil { + return nil, err + } + return topics.Topics, nil +} + +// GetMilestones returns milestones +func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { + var milestones = make([]*base.Milestone, 0, 10) + p := filepath.Join(r.baseDir, "milestone.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(bs, &milestones) + if err != nil { + return nil, err + } + return milestones, nil +} + +// GetReleases returns releases +func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { + var releases = make([]*base.Release, 0, 10) + p := filepath.Join(r.baseDir, "release.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(bs, &releases) + if err != nil { + return nil, err + } + for _, rel := range releases { + for _, asset := range rel.Assets { + *asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL) + } + } + return releases, nil +} + +// GetLabels returns labels +func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { + var labels = make([]*base.Label, 0, 10) + p := filepath.Join(r.baseDir, "label.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(bs, &labels) + if err != nil { + return nil, err + } + return labels, nil +} + +// GetIssues returns issues according start and limit +func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + var issues = make([]*base.Issue, 0, 10) + p := filepath.Join(r.baseDir, "issue.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, true, nil + } + return nil, false, err + } + + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, false, err + } + + err = yaml.Unmarshal(bs, &issues) + if err != nil { + return nil, false, err + } + return issues, true, nil +} + +// GetComments returns comments according issueNumber +func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) { + var comments = make([]*base.Comment, 0, 10) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber)) + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(bs, &comments) + if err != nil { + return nil, err + } + return comments, nil +} + +// GetPullRequests returns pull requests according page and perPage +func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + var pulls = make([]*base.PullRequest, 0, 10) + p := filepath.Join(r.baseDir, "pull_request.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, true, nil + } + return nil, false, err + } + + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, false, err + } + + err = yaml.Unmarshal(bs, &pulls) + if err != nil { + return nil, false, err + } + for _, pr := range pulls { + pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL) + } + return pulls, true, nil +} + +// GetReviews returns pull requests review +func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { + var reviews = make([]*base.Review, 0, 10) + p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber)) + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(bs, &reviews) + if err != nil { + return nil, err + } + return reviews, nil +} diff --git a/modules/uri/uri.go b/modules/uri/uri.go new file mode 100644 index 0000000000..0967a0802f --- /dev/null +++ b/modules/uri/uri.go @@ -0,0 +1,40 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uri + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" +) + +// ErrURISchemeNotSupported represents a scheme error +type ErrURISchemeNotSupported struct { + Scheme string +} + +func (e ErrURISchemeNotSupported) Error() string { + return fmt.Sprintf("Unsupported scheme: %v", e.Scheme) +} + +// Open open a local file or a remote file +func Open(uriStr string) (io.ReadCloser, error) { + u, err := url.Parse(uriStr) + if err != nil { + return nil, err + } + switch strings.ToLower(u.Scheme) { + case "http", "https": + f, err := http.Get(uriStr) + return f.Body, err + case "file": + return os.Open(u.Path) + default: + return nil, ErrURISchemeNotSupported{Scheme: u.Scheme} + } +} diff --git a/modules/uri/uri_test.go b/modules/uri/uri_test.go new file mode 100644 index 0000000000..8cadd6b918 --- /dev/null +++ b/modules/uri/uri_test.go @@ -0,0 +1,20 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uri + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadURI(t *testing.T) { + p, err := filepath.Abs("./uri.go") + assert.NoError(t, err) + f, err := Open("file://" + p) + assert.NoError(t, err) + defer f.Close() +} diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index ab480c29aa..3fd9300904 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -176,11 +176,8 @@ func Migrate(ctx *context.APIContext, form api.MigrateRepoOptions) { } if err == nil { - repo.Status = models.RepositoryReady - if err := models.UpdateRepositoryCols(repo, "status"); err == nil { - notification.NotifyMigrateRepository(ctx.User, repoOwner, repo) - return - } + notification.NotifyMigrateRepository(ctx.User, repoOwner, repo) + return } if repo != nil {