forgejo/models/unittest/testdb.go

254 lines
8 KiB
Go
Raw Normal View History

// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package unittest
import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"testing"
"code.gitea.io/gitea/models/db"
system_model "code.gitea.io/gitea/models/system"
"code.gitea.io/gitea/modules/auth/password/hash"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util"
"github.com/stretchr/testify/assert"
"xorm.io/xorm"
"xorm.io/xorm/names"
)
// giteaRoot a path to the gitea root
var (
giteaRoot string
fixturesDir string
)
// FixturesDir returns the fixture directory
func FixturesDir() string {
return fixturesDir
}
func fatalTestError(fmtStr string, args ...interface{}) {
_, _ = fmt.Fprintf(os.Stderr, fmtStr, args...)
os.Exit(1)
}
// InitSettings initializes config provider and load common setttings for tests
func InitSettings(extraConfigs ...string) {
setting.Init(&setting.Options{
AllowEmpty: true,
ExtraConfig: strings.Join(extraConfigs, "\n"),
})
if err := setting.PrepareAppDataPath(); err != nil {
log.Fatalf("Can not prepare APP_DATA_PATH: %v", err)
}
// register the dummy hash algorithm function used in the test fixtures
_ = hash.Register("dummy", hash.NewDummyHasher)
setting.PasswordHashAlgo, _ = hash.SetDefaultPasswordHashAlgorithm("dummy")
}
// TestOptions represents test options
type TestOptions struct {
GiteaRootPath string
FixtureFiles []string
SetUp func() error // SetUp will be executed before all tests in this package
TearDown func() error // TearDown will be executed after all tests in this package
}
// MainTest a reusable TestMain(..) function for unit tests that need to use a
// test database. Creates the test database, and sets necessary settings.
func MainTest(m *testing.M, testOpts *TestOptions) {
setting.SetCustomPathAndConf("", "", "")
InitSettings()
var err error
giteaRoot = testOpts.GiteaRootPath
fixturesDir = filepath.Join(testOpts.GiteaRootPath, "models", "fixtures")
var opts FixturesOptions
if len(testOpts.FixtureFiles) == 0 {
opts.Dir = fixturesDir
} else {
for _, f := range testOpts.FixtureFiles {
if len(f) != 0 {
opts.Files = append(opts.Files, filepath.Join(fixturesDir, f))
}
}
}
if err = CreateTestEngine(opts); err != nil {
fatalTestError("Error creating test engine: %v\n", err)
}
setting.AppURL = "https://try.gitea.io/"
setting.RunUser = "runuser"
setting.SSH.User = "sshuser"
setting.SSH.BuiltinServerUser = "builtinuser"
setting.SSH.Port = 3000
setting.SSH.Domain = "try.gitea.io"
setting.Database.Type = "sqlite3"
setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master"
repoRootPath, err := os.MkdirTemp(os.TempDir(), "repos")
if err != nil {
fatalTestError("TempDir: %v\n", err)
}
setting.RepoRootPath = repoRootPath
appDataPath, err := os.MkdirTemp(os.TempDir(), "appdata")
if err != nil {
fatalTestError("TempDir: %v\n", err)
}
setting.AppDataPath = appDataPath
setting.AppWorkPath = testOpts.GiteaRootPath
setting.StaticRootPath = testOpts.GiteaRootPath
setting.GravatarSource = "https://secure.gravatar.com/avatar/"
setting.Attachment.Storage.Path = filepath.Join(setting.AppDataPath, "attachments")
setting.LFS.Storage.Path = filepath.Join(setting.AppDataPath, "lfs")
setting.Avatar.Storage.Path = filepath.Join(setting.AppDataPath, "avatars")
setting.RepoAvatar.Storage.Path = filepath.Join(setting.AppDataPath, "repo-avatars")
setting.RepoArchive.Storage.Path = filepath.Join(setting.AppDataPath, "repo-archive")
Add Package Registry (#16510) * Added package store settings. * Added models. * Added generic package registry. * Added tests. * Added NuGet package registry. * Moved service index to api file. * Added NPM package registry. * Added Maven package registry. * Added PyPI package registry. * Summary is deprecated. * Changed npm name. * Sanitize project url. * Allow only scoped packages. * Added user interface. * Changed method name. * Added missing migration file. * Set page info. * Added documentation. * Added documentation links. * Fixed wrong error message. * Lint template files. * Fixed merge errors. * Fixed unit test storage path. * Switch to json module. * Added suggestions. * Added package webhook. * Add package api. * Fixed swagger file. * Fixed enum and comments. * Fixed NuGet pagination. * Print test names. * Added api tests. * Fixed access level. * Fix User unmarshal. * Added RubyGems package registry. * Fix lint. * Implemented io.Writer. * Added support for sha256/sha512 checksum files. * Improved maven-metadata.xml support. * Added support for symbol package uploads. * Added tests. * Added overview docs. * Added npm dependencies and keywords. * Added no-packages information. * Display file size. * Display asset count. * Fixed filter alignment. * Added package icons. * Formatted instructions. * Allow anonymous package downloads. * Fixed comments. * Fixed postgres test. * Moved file. * Moved models to models/packages. * Use correct error response format per client. * Use simpler search form. * Fixed IsProd. * Restructured data model. * Prevent empty filename. * Fix swagger. * Implemented user/org registry. * Implemented UI. * Use GetUserByIDCtx. * Use table for dependencies. * make svg * Added support for unscoped npm packages. * Add support for npm dist tags. * Added tests for npm tags. * Unlink packages if repository gets deleted. * Prevent user/org delete if a packages exist. * Use package unlink in repository service. * Added support for composer packages. * Restructured package docs. * Added missing tests. * Fixed generic content page. * Fixed docs. * Fixed swagger. * Added missing type. * Fixed ambiguous column. * Organize content store by sha256 hash. * Added admin package management. * Added support for sorting. * Add support for multiple identical versions/files. * Added missing repository unlink. * Added file properties. * make fmt * lint * Added Conan package registry. * Updated docs. * Unify package names. * Added swagger enum. * Use longer TEXT column type. * Removed version composite key. * Merged package and container registry. * Removed index. * Use dedicated package router. * Moved files to new location. * Updated docs. * Fixed JOIN order. * Fixed GROUP BY statement. * Fixed GROUP BY #2. * Added symbol server support. * Added more tests. * Set NOT NULL. * Added setting to disable package registries. * Moved auth into service. * refactor * Use ctx everywhere. * Added package cleanup task. * Changed packages path. * Added container registry. * Refactoring * Updated comparison. * Fix swagger. * Fixed table order. * Use token auth for npm routes. * Enabled ReverseProxy auth. * Added packages link for orgs. * Fixed anonymous org access. * Enable copy button for setup instructions. * Merge error * Added suggestions. * Fixed merge. * Handle "generic". * Added link for TODO. * Added suggestions. * Changed temporary buffer filename. * Added suggestions. * Apply suggestions from code review Co-authored-by: Thomas Boerger <thomas@webhippie.de> * Update docs/content/doc/packages/nuget.en-us.md Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: Thomas Boerger <thomas@webhippie.de>
2022-03-30 10:42:47 +02:00
setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages")
Implement actions (#21937) Close #13539. Co-authored by: @lunny @appleboy @fuxiaohei and others. Related projects: - https://gitea.com/gitea/actions-proto-def - https://gitea.com/gitea/actions-proto-go - https://gitea.com/gitea/act - https://gitea.com/gitea/act_runner ### Summary The target of this PR is to bring a basic implementation of "Actions", an internal CI/CD system of Gitea. That means even though it has been merged, the state of the feature is **EXPERIMENTAL**, and please note that: - It is disabled by default; - It shouldn't be used in a production environment currently; - It shouldn't be used in a public Gitea instance currently; - Breaking changes may be made before it's stable. **Please comment on #13539 if you have any different product design ideas**, all decisions reached there will be adopted here. But in this PR, we don't talk about **naming, feature-creep or alternatives**. ### ⚠️ Breaking `gitea-actions` will become a reserved user name. If a user with the name already exists in the database, it is recommended to rename it. ### Some important reviews - What is `DEFAULT_ACTIONS_URL` in `app.ini` for? - https://github.com/go-gitea/gitea/pull/21937#discussion_r1055954954 - Why the api for runners is not under the normal `/api/v1` prefix? - https://github.com/go-gitea/gitea/pull/21937#discussion_r1061173592 - Why DBFS? - https://github.com/go-gitea/gitea/pull/21937#discussion_r1061301178 - Why ignore events triggered by `gitea-actions` bot? - https://github.com/go-gitea/gitea/pull/21937#discussion_r1063254103 - Why there's no permission control for actions? - https://github.com/go-gitea/gitea/pull/21937#discussion_r1090229868 ### What it looks like <details> #### Manage runners <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205870657-c72f590e-2e08-4cd4-be7f-2e0abb299bbf.png"> #### List runs <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205872794-50fde990-2b45-48c1-a178-908e4ec5b627.png"> #### View logs <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205872501-9b7b9000-9542-4991-8f55-18ccdada77c3.png"> </details> ### How to try it <details> #### 1. Start Gitea Clone this branch and [install from source](https://docs.gitea.io/en-us/install-from-source). Add additional configurations in `app.ini` to enable Actions: ```ini [actions] ENABLED = true ``` Start it. If all is well, you'll see the management page of runners: <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205877365-8e30a780-9b10-4154-b3e8-ee6c3cb35a59.png"> #### 2. Start runner Clone the [act_runner](https://gitea.com/gitea/act_runner), and follow the [README](https://gitea.com/gitea/act_runner/src/branch/main/README.md) to start it. If all is well, you'll see a new runner has been added: <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205878000-216f5937-e696-470d-b66c-8473987d91c3.png"> #### 3. Enable actions for a repo Create a new repo or open an existing one, check the `Actions` checkbox in settings and submit. <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205879705-53e09208-73c0-4b3e-a123-2dcf9aba4b9c.png"> <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205879383-23f3d08f-1a85-41dd-a8b3-54e2ee6453e8.png"> If all is well, you'll see a new tab "Actions": <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205881648-a8072d8c-5803-4d76-b8a8-9b2fb49516c1.png"> #### 4. Upload workflow files Upload some workflow files to `.gitea/workflows/xxx.yaml`, you can follow the [quickstart](https://docs.github.com/en/actions/quickstart) of GitHub Actions. Yes, Gitea Actions is compatible with GitHub Actions in most cases, you can use the same demo: ```yaml name: GitHub Actions Demo run-name: ${{ github.actor }} is testing out GitHub Actions 🚀 on: [push] jobs: Explore-GitHub-Actions: runs-on: ubuntu-latest steps: - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - name: Check out repository code uses: actions/checkout@v3 - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - run: echo "🖥️ The workflow is now ready to test your code on the runner." - name: List files in the repository run: | ls ${{ github.workspace }} - run: echo "🍏 This job's status is ${{ job.status }}." ``` If all is well, you'll see a new run in `Actions` tab: <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205884473-79a874bc-171b-4aaf-acd5-0241a45c3b53.png"> #### 5. Check the logs of jobs Click a run and you'll see the logs: <img width="1792" alt="image" src="https://user-images.githubusercontent.com/9418365/205884800-994b0374-67f7-48ff-be9a-4c53f3141547.png"> #### 6. Go on You can try more examples in [the documents](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions) of GitHub Actions, then you might find a lot of bugs. Come on, PRs are welcome. </details> See also: [Feature Preview: Gitea Actions](https://blog.gitea.io/2022/12/feature-preview-gitea-actions/) --------- Co-authored-by: a1012112796 <1012112796@qq.com> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: ChristopherHX <christopher.homberger@web.de> Co-authored-by: John Olheiser <john.olheiser@gmail.com>
2023-01-31 02:45:19 +01:00
setting.Actions.Storage.Path = filepath.Join(setting.AppDataPath, "actions_log")
setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home")
setting.IncomingEmail.ReplyToAddress = "incoming+%{token}@localhost"
if err = storage.Init(); err != nil {
fatalTestError("storage.Init: %v\n", err)
}
if err = system_model.Init(db.DefaultContext); err != nil {
fatalTestError("models.Init: %v\n", err)
}
if err = util.RemoveAll(repoRootPath); err != nil {
fatalTestError("util.RemoveAll: %v\n", err)
}
if err = CopyDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
fatalTestError("util.CopyDir: %v\n", err)
}
if err = git.InitFull(context.Background()); err != nil {
fatalTestError("git.Init: %v\n", err)
}
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
fatalTestError("unable to read the new repo root: %v\n", err)
}
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
if err != nil {
fatalTestError("unable to read the new repo root: %v\n", err)
}
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
}
}
if testOpts.SetUp != nil {
if err := testOpts.SetUp(); err != nil {
fatalTestError("set up failed: %v\n", err)
}
}
exitStatus := m.Run()
if testOpts.TearDown != nil {
if err := testOpts.TearDown(); err != nil {
fatalTestError("tear down failed: %v\n", err)
}
}
if err = util.RemoveAll(repoRootPath); err != nil {
fatalTestError("util.RemoveAll: %v\n", err)
}
if err = util.RemoveAll(appDataPath); err != nil {
fatalTestError("util.RemoveAll: %v\n", err)
}
os.Exit(exitStatus)
}
// FixturesOptions fixtures needs to be loaded options
type FixturesOptions struct {
Dir string
Files []string
}
// CreateTestEngine creates a memory database and loads the fixture data from fixturesDir
func CreateTestEngine(opts FixturesOptions) error {
x, err := xorm.NewEngine("sqlite3", "file::memory:?cache=shared&_txlock=immediate")
if err != nil {
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 13:49:59 +02:00
if strings.Contains(err.Error(), "unknown driver") {
return fmt.Errorf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify%s%w`, "\n", err)
}
return err
}
x.SetMapper(names.GonicMapper{})
db.SetDefaultEngine(context.Background(), x)
if err = db.SyncAllTables(); err != nil {
return err
}
switch os.Getenv("GITEA_UNIT_TESTS_LOG_SQL") {
case "true", "1":
x.ShowSQL(true)
}
return InitFixtures(opts)
}
// PrepareTestDatabase load test fixtures into test database
func PrepareTestDatabase() error {
return LoadFixtures()
}
// PrepareTestEnv prepares the environment for unit tests. Can only be called
// by tests that use the above MainTest(..) function.
func PrepareTestEnv(t testing.TB) {
assert.NoError(t, PrepareTestDatabase())
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta")
assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
assert.NoError(t, err)
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
assert.NoError(t, err)
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
}
}
base.SetupGiteaRoot() // Makes sure GITEA_ROOT is set
}