forked from NYANDEV/forgejo
Add LFS Migration and Mirror (#14726)
* Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
parent
f544414a23
commit
c03e488e14
75 changed files with 2159 additions and 711 deletions
|
@ -14,7 +14,6 @@ import (
|
|||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/lfs"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/storage"
|
||||
)
|
||||
|
||||
// UploadRepoFileOptions contains the uploaded repository file options
|
||||
|
@ -137,7 +136,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep
|
|||
|
||||
// OK now we can insert the data into the store - there's no way to clean up the store
|
||||
// once it's in there, it's in there.
|
||||
contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
|
||||
contentStore := lfs.NewContentStore()
|
||||
for _, info := range infos {
|
||||
if err := uploadToLFSContentStore(info, contentStore); err != nil {
|
||||
return cleanUpAfterFailure(&infos, t, err)
|
||||
|
@ -163,18 +162,14 @@ func copyUploadedLFSFileIntoRepository(info *uploadInfo, filename2attribute2info
|
|||
if setting.LFS.StartServer && filename2attribute2info[info.upload.Name] != nil && filename2attribute2info[info.upload.Name]["filter"] == "lfs" {
|
||||
// Handle LFS
|
||||
// FIXME: Inefficient! this should probably happen in models.Upload
|
||||
oid, err := models.GenerateLFSOid(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileInfo, err := file.Stat()
|
||||
pointer, err := lfs.GeneratePointer(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID}
|
||||
info.lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID}
|
||||
|
||||
if objectHash, err = t.HashObject(strings.NewReader(info.lfsMetaObject.Pointer())); err != nil {
|
||||
if objectHash, err = t.HashObject(strings.NewReader(pointer.StringContent())); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if objectHash, err = t.HashObject(file); err != nil {
|
||||
|
@ -189,7 +184,7 @@ func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) er
|
|||
if info.lfsMetaObject == nil {
|
||||
return nil
|
||||
}
|
||||
exist, err := contentStore.Exists(info.lfsMetaObject)
|
||||
exist, err := contentStore.Exists(info.lfsMetaObject.Pointer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -202,7 +197,7 @@ func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) er
|
|||
defer file.Close()
|
||||
// FIXME: Put regenerates the hash and copies the file over.
|
||||
// I guess this strictly ensures the soundness of the store but this is inefficient.
|
||||
if err := contentStore.Put(info.lfsMetaObject, file); err != nil {
|
||||
if err := contentStore.Put(info.lfsMetaObject.Pointer, file); err != nil {
|
||||
// OK Now we need to cleanup
|
||||
// Can't clean up the store, once uploaded there they're there.
|
||||
return err
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue