forgejo/services/cron/tasks_extended.go

242 lines
7.6 KiB
Go
Raw Normal View History

// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cron
import (
"context"
"time"
activities_model "code.gitea.io/gitea/models/activities"
asymkey_model "code.gitea.io/gitea/models/asymkey"
"code.gitea.io/gitea/models/system"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/updatechecker"
repo_service "code.gitea.io/gitea/services/repository"
archiver_service "code.gitea.io/gitea/services/repository/archiver"
user_service "code.gitea.io/gitea/services/user"
)
func registerDeleteInactiveUsers() {
RegisterTaskFatal("delete_inactive_accounts", &OlderThanConfig{
BaseConfig: BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@annually",
},
OlderThan: time.Minute * time.Duration(setting.Service.ActiveCodeLives),
}, func(ctx context.Context, _ *user_model.User, config Config) error {
olderThanConfig := config.(*OlderThanConfig)
return user_service.DeleteInactiveUsers(ctx, olderThanConfig.OlderThan)
})
}
func registerDeleteRepositoryArchives() {
RegisterTaskFatal("delete_repo_archives", &BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@annually",
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
return archiver_service.DeleteRepositoryArchives(ctx)
})
}
func registerGarbageCollectRepositories() {
type RepoHealthCheckConfig struct {
BaseConfig
Timeout time.Duration
Args []string `delim:" "`
}
RegisterTaskFatal("git_gc_repos", &RepoHealthCheckConfig{
BaseConfig: BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 72h",
},
Timeout: time.Duration(setting.Git.Timeout.GC) * time.Second,
Args: setting.Git.GCArgs,
}, func(ctx context.Context, _ *user_model.User, config Config) error {
rhcConfig := config.(*RepoHealthCheckConfig)
// the git args are set by config, they can be safe to be trusted
Refactor git command package to improve security and maintainability (#22678) This PR follows #21535 (and replace #22592) ## Review without space diff https://github.com/go-gitea/gitea/pull/22678/files?diff=split&w=1 ## Purpose of this PR 1. Make git module command completely safe (risky user inputs won't be passed as argument option anymore) 2. Avoid low-level mistakes like https://github.com/go-gitea/gitea/pull/22098#discussion_r1045234918 3. Remove deprecated and dirty `CmdArgCheck` function, hide the `CmdArg` type 4. Simplify code when using git command ## The main idea of this PR * Move the `git.CmdArg` to the `internal` package, then no other package except `git` could use it. Then developers could never do `AddArguments(git.CmdArg(userInput))` any more. * Introduce `git.ToTrustedCmdArgs`, it's for user-provided and already trusted arguments. It's only used in a few cases, for example: use git arguments from config file, help unit test with some arguments. * Introduce `AddOptionValues` and `AddOptionFormat`, they make code more clear and simple: * Before: `AddArguments("-m").AddDynamicArguments(message)` * After: `AddOptionValues("-m", message)` * - * Before: `AddArguments(git.CmdArg(fmt.Sprintf("--author='%s <%s>'", sig.Name, sig.Email)))` * After: `AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email)` ## FAQ ### Why these changes were not done in #21535 ? #21535 is mainly a search&replace, it did its best to not change too much logic. Making the framework better needs a lot of changes, so this separate PR is needed as the second step. ### The naming of `AddOptionXxx` According to git's manual, the `--xxx` part is called `option`. ### How can it guarantee that `internal.CmdArg` won't be not misused? Go's specification guarantees that. Trying to access other package's internal package causes compilation error. And, `golangci-lint` also denies the git/internal package. Only the `git/command.go` can use it carefully. ### There is still a `ToTrustedCmdArgs`, will it still allow developers to make mistakes and pass untrusted arguments? Generally speaking, no. Because when using `ToTrustedCmdArgs`, the code will be very complex (see the changes for examples). Then developers and reviewers can know that something might be unreasonable. ### Why there was a `CmdArgCheck` and why it's removed? At the moment of #21535, to reduce unnecessary changes, `CmdArgCheck` was introduced as a hacky patch. Now, almost all code could be written as `cmd := NewCommand(); cmd.AddXxx(...)`, then there is no need for `CmdArgCheck` anymore. ### Why many codes for `signArg == ""` is deleted? Because in the old code, `signArg` could never be empty string, it's either `-S[key-id]` or `--no-gpg-sign`. So the `signArg == ""` is just dead code. --------- Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2023-02-04 03:30:43 +01:00
return repo_service.GitGcRepos(ctx, rhcConfig.Timeout, git.ToTrustedCmdArgs(rhcConfig.Args))
})
}
func registerRewriteAllPublicKeys() {
RegisterTaskFatal("resync_all_sshkeys", &BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 72h",
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
return asymkey_model.RewriteAllPublicKeys(ctx)
})
}
Add ssh certificate support (#12281) * Add ssh certificate support * Add ssh certificate support to builtin ssh * Write trusted-user-ca-keys.pem based on configuration * Update app.example.ini * Update templates/user/settings/keys_principal.tmpl Co-authored-by: silverwind <me@silverwind.io> * Remove unused locale string * Update options/locale/locale_en-US.ini Co-authored-by: silverwind <me@silverwind.io> * Update options/locale/locale_en-US.ini Co-authored-by: silverwind <me@silverwind.io> * Update models/ssh_key.go Co-authored-by: silverwind <me@silverwind.io> * Add missing creation of SSH.Rootpath * Update cheatsheet, example and locale strings * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go * Optimizations based on feedback * Validate CA keys for external sshd * Add filename option and change default filename Add a SSH_TRUSTED_USER_CA_KEYS_FILENAME option which default is RUN_USER/.ssh/gitea-trusted-user-ca-keys.pem Do not write a file when SSH_TRUSTED_USER_CA_KEYS is empty. Add some more documentation. * Remove unneeded principalkey functions * Add blank line * Apply suggestions from code review Co-authored-by: zeripath <art27@cantab.net> * Add SSH_AUTHORIZED_PRINCIPALS_ALLOW option This adds a SSH_AUTHORIZED_PRINCIPALS_ALLOW which is default email,username this means that users only can add the principals that match their email or username. To allow anything the admin need to set the option anything. This allows for a safe default in gitea which protects against malicious users using other user's prinicipals. (before that user could set it). This commit also has some small other fixes from the last code review. * Rewrite principal keys file on user deletion * Use correct rewrite method * Set correct AuthorizedPrincipalsBackup default setting * Rewrite principalsfile when adding principals * Add update authorized_principals option to admin dashboard * Handle non-primary emails Signed-off-by: Andrew Thornton <art27@cantab.net> * Add the command actually to the dashboard template * Update models/ssh_key.go Co-authored-by: silverwind <me@silverwind.io> * By default do not show principal options unless there are CA keys set or they are explicitly set Signed-off-by: Andrew Thornton <art27@cantab.net> * allow settings when enabled * Fix typos in TrustedUserCAKeys path * Allow every CASignatureAlgorithms algorithm As this depends on the content of TrustedUserCAKeys we should allow all signature algorithms as admins can choose the specific algorithm on their signing CA * Update models/ssh_key.go Co-authored-by: Lauris BH <lauris@nix.lv> * Fix linting issue Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
2020-10-11 02:38:09 +02:00
func registerRewriteAllPrincipalKeys() {
RegisterTaskFatal("resync_all_sshprincipals", &BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 72h",
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
return asymkey_model.RewriteAllPrincipalKeys(ctx)
Add ssh certificate support (#12281) * Add ssh certificate support * Add ssh certificate support to builtin ssh * Write trusted-user-ca-keys.pem based on configuration * Update app.example.ini * Update templates/user/settings/keys_principal.tmpl Co-authored-by: silverwind <me@silverwind.io> * Remove unused locale string * Update options/locale/locale_en-US.ini Co-authored-by: silverwind <me@silverwind.io> * Update options/locale/locale_en-US.ini Co-authored-by: silverwind <me@silverwind.io> * Update models/ssh_key.go Co-authored-by: silverwind <me@silverwind.io> * Add missing creation of SSH.Rootpath * Update cheatsheet, example and locale strings * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go * Optimizations based on feedback * Validate CA keys for external sshd * Add filename option and change default filename Add a SSH_TRUSTED_USER_CA_KEYS_FILENAME option which default is RUN_USER/.ssh/gitea-trusted-user-ca-keys.pem Do not write a file when SSH_TRUSTED_USER_CA_KEYS is empty. Add some more documentation. * Remove unneeded principalkey functions * Add blank line * Apply suggestions from code review Co-authored-by: zeripath <art27@cantab.net> * Add SSH_AUTHORIZED_PRINCIPALS_ALLOW option This adds a SSH_AUTHORIZED_PRINCIPALS_ALLOW which is default email,username this means that users only can add the principals that match their email or username. To allow anything the admin need to set the option anything. This allows for a safe default in gitea which protects against malicious users using other user's prinicipals. (before that user could set it). This commit also has some small other fixes from the last code review. * Rewrite principal keys file on user deletion * Use correct rewrite method * Set correct AuthorizedPrincipalsBackup default setting * Rewrite principalsfile when adding principals * Add update authorized_principals option to admin dashboard * Handle non-primary emails Signed-off-by: Andrew Thornton <art27@cantab.net> * Add the command actually to the dashboard template * Update models/ssh_key.go Co-authored-by: silverwind <me@silverwind.io> * By default do not show principal options unless there are CA keys set or they are explicitly set Signed-off-by: Andrew Thornton <art27@cantab.net> * allow settings when enabled * Fix typos in TrustedUserCAKeys path * Allow every CASignatureAlgorithms algorithm As this depends on the content of TrustedUserCAKeys we should allow all signature algorithms as admins can choose the specific algorithm on their signing CA * Update models/ssh_key.go Co-authored-by: Lauris BH <lauris@nix.lv> * Fix linting issue Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
2020-10-11 02:38:09 +02:00
})
}
func registerRepositoryUpdateHook() {
RegisterTaskFatal("resync_all_hooks", &BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 72h",
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
return repo_service.SyncRepositoryHooks(ctx)
})
}
func registerReinitMissingRepositories() {
RegisterTaskFatal("reinit_missing_repos", &BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 72h",
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
return repo_service.ReinitMissingRepositories(ctx)
})
}
func registerDeleteMissingRepositories() {
RegisterTaskFatal("delete_missing_repos", &BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 72h",
}, func(ctx context.Context, user *user_model.User, _ Config) error {
return repo_service.DeleteMissingRepositories(ctx, user)
})
}
func registerRemoveRandomAvatars() {
RegisterTaskFatal("delete_generated_repository_avatars", &BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 72h",
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
return repo_service.RemoveRandomAvatars(ctx)
})
}
func registerDeleteOldActions() {
RegisterTaskFatal("delete_old_actions", &OlderThanConfig{
BaseConfig: BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 168h",
},
OlderThan: 365 * 24 * time.Hour,
}, func(ctx context.Context, _ *user_model.User, config Config) error {
olderThanConfig := config.(*OlderThanConfig)
return activities_model.DeleteOldActions(ctx, olderThanConfig.OlderThan)
})
}
func registerUpdateGiteaChecker() {
type UpdateCheckerConfig struct {
BaseConfig
HTTPEndpoint string
}
RegisterTaskFatal("update_checker", &UpdateCheckerConfig{
BaseConfig: BaseConfig{
[PRIVACY] Disable `update_checker` cron tasks - This is being disabled as it will weekly connect to a domain. - This only affects existing installations, as new installations will have a explicit value being written into app.ini due to https://github.com/go-gitea/gitea/pull/21655 (cherry picked from commit cd0b8b6852563118ab8530c01a48bc612efcf58a) (cherry picked from commit 58d43867cab15a4b93fab6f4b467756e8030fb2d) (cherry picked from commit f290b91e504933be83d5b9540a195173280ab38d) (cherry picked from commit 5bd6d7555478713c91a3c00980060041ed34ba22) (cherry picked from commit e5d0abb59244809f5181dfbadfbee47531246a18) (cherry picked from commit 4b381ce77e699ce9f0f8af8ac3488bd8e3562b15) (cherry picked from commit b6a9b1390d9ee3aa70d50b1fae1346aff69cab08) (cherry picked from commit 367436287118b0fc4030ee6527d6c366db919594) (cherry picked from commit af0f606fc1f42be49dcbedd04055f0f9a0b7d3df) (cherry picked from commit 49560da6fb01cd6ee1a69840f48ed9269330c37a) (cherry picked from commit ca06e27c5b7b8b20e34935885c4ab0ca905b9157) (cherry picked from commit 163df86df5abdda11b6d337af32e0f0f3837119b) (cherry picked from commit 0391c6ae42eb364a6db1156af93304aa73c69b2d) (cherry picked from commit f282f2d42fd850c29b489d02133698e14b21f411) (cherry picked from commit 68d31cf2478626730324eaa56ad4264580b7030c) (cherry picked from commit cfd4a479e95597f315d9d75e2b7d22c8f6941ce2) (cherry picked from commit add969bdf624df9eed3047d60a1bae979e9bfd80) (cherry picked from commit c0f3f1f1e733b6df0f028a1393549adf7f8936f4) (cherry picked from commit f11dca9090a77dbd58ff980e97cff07a5b2d27b5) (cherry picked from commit 283810387ff26fbbf1ac795d56350a7bfb87b874) (cherry picked from commit 08af1f0ea882bbe3fada3e61988baa9f0d290d1a) (cherry picked from commit 1c1a6da319f9218cc05693e0b8461f3defe1d891) (cherry picked from commit c1444bae65daa29497209a0bcc0758b1af9828a0) (cherry picked from commit 6361df634bd8fda81793e35820f0a42c56e5e55a) (cherry picked from commit cc8bb3f94605858c3ff9412644d90d80150c3f23) (cherry picked from commit c7ae4411fd041dde1c73a5fdc64679317b6daef7) (cherry picked from commit f73f4eef02c41430cc6a7375fc3b73aab8b75de7) (cherry picked from commit 2122896e8529db6b6158bb9029492f8f5e12efb7) (cherry picked from commit d469c99671cf6a5a8087b1ad106a04b43f4edc85) (cherry picked from commit 41dbbd731edf52c1c78d34cba1dfd7e03029e5f7) (cherry picked from commit e8e742b4376ab2437097a94a5f541576f9da3c81) (cherry picked from commit 0fa9f87b5fbb81b9205650da918bab02c7f86916) (cherry picked from commit 870645b963faaa38400bfd179e5ca187f1160d65) (cherry picked from commit 532a5860eb0957afd37f01d3f178d8686b8c09e3) (cherry picked from commit 275c671d5f57be23798360a5d97da035f9827f5f) (cherry picked from commit 2cec78e493e0c5969de0846685788640f84a5e27) (cherry picked from commit 10ab0befd7dc0e94bf1fb4030f86fbf3d59b346f) (cherry picked from commit a0a0c0adaf0ad4dec87d01b016228fd2bbd2cce0)
2022-12-18 19:56:08 +01:00
Enabled: false,
RunAtStart: false,
Schedule: "@every 168h",
},
HTTPEndpoint: "https://dl.gitea.com/gitea/version.json",
}, func(ctx context.Context, _ *user_model.User, config Config) error {
updateCheckerConfig := config.(*UpdateCheckerConfig)
return updatechecker.GiteaUpdateChecker(updateCheckerConfig.HTTPEndpoint)
})
}
func registerDeleteOldSystemNotices() {
RegisterTaskFatal("delete_old_system_notices", &OlderThanConfig{
BaseConfig: BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 168h",
},
OlderThan: 365 * 24 * time.Hour,
}, func(ctx context.Context, _ *user_model.User, config Config) error {
olderThanConfig := config.(*OlderThanConfig)
return system.DeleteOldSystemNotices(ctx, olderThanConfig.OlderThan)
})
}
func registerGCLFS() {
if !setting.LFS.StartServer {
return
}
type GCLFSConfig struct {
OlderThanConfig
LastUpdatedMoreThanAgo time.Duration
NumberToCheckPerRepo int64
ProportionToCheckPerRepo float64
}
RegisterTaskFatal("gc_lfs", &GCLFSConfig{
OlderThanConfig: OlderThanConfig{
BaseConfig: BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@every 24h",
},
// Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload
// and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby
// an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid
// changes in new branches that might lead to lfs objects becoming temporarily unassociated with git
// objects.
//
// It is likely that a week is potentially excessive but it should definitely be enough that any
// unassociated LFS object is genuinely unassociated.
OlderThan: 24 * time.Hour * 7,
},
// Only GC things that haven't been looked at in the past 3 days
LastUpdatedMoreThanAgo: 24 * time.Hour * 3,
NumberToCheckPerRepo: 100,
ProportionToCheckPerRepo: 0.6,
}, func(ctx context.Context, _ *user_model.User, config Config) error {
gcLFSConfig := config.(*GCLFSConfig)
return repo_service.GarbageCollectLFSMetaObjects(ctx, repo_service.GarbageCollectLFSMetaObjectsOptions{
AutoFix: true,
OlderThan: time.Now().Add(-gcLFSConfig.OlderThan),
UpdatedLessRecentlyThan: time.Now().Add(-gcLFSConfig.LastUpdatedMoreThanAgo),
})
})
}
func registerRebuildIssueIndexer() {
RegisterTaskFatal("rebuild_issue_indexer", &BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@annually",
}, func(ctx context.Context, _ *user_model.User, config Config) error {
return issue_indexer.PopulateIssueIndexer(ctx)
})
}
func initExtendedTasks() {
registerDeleteInactiveUsers()
registerDeleteRepositoryArchives()
registerGarbageCollectRepositories()
registerRewriteAllPublicKeys()
Add ssh certificate support (#12281) * Add ssh certificate support * Add ssh certificate support to builtin ssh * Write trusted-user-ca-keys.pem based on configuration * Update app.example.ini * Update templates/user/settings/keys_principal.tmpl Co-authored-by: silverwind <me@silverwind.io> * Remove unused locale string * Update options/locale/locale_en-US.ini Co-authored-by: silverwind <me@silverwind.io> * Update options/locale/locale_en-US.ini Co-authored-by: silverwind <me@silverwind.io> * Update models/ssh_key.go Co-authored-by: silverwind <me@silverwind.io> * Add missing creation of SSH.Rootpath * Update cheatsheet, example and locale strings * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go Co-authored-by: zeripath <art27@cantab.net> * Update models/ssh_key.go * Optimizations based on feedback * Validate CA keys for external sshd * Add filename option and change default filename Add a SSH_TRUSTED_USER_CA_KEYS_FILENAME option which default is RUN_USER/.ssh/gitea-trusted-user-ca-keys.pem Do not write a file when SSH_TRUSTED_USER_CA_KEYS is empty. Add some more documentation. * Remove unneeded principalkey functions * Add blank line * Apply suggestions from code review Co-authored-by: zeripath <art27@cantab.net> * Add SSH_AUTHORIZED_PRINCIPALS_ALLOW option This adds a SSH_AUTHORIZED_PRINCIPALS_ALLOW which is default email,username this means that users only can add the principals that match their email or username. To allow anything the admin need to set the option anything. This allows for a safe default in gitea which protects against malicious users using other user's prinicipals. (before that user could set it). This commit also has some small other fixes from the last code review. * Rewrite principal keys file on user deletion * Use correct rewrite method * Set correct AuthorizedPrincipalsBackup default setting * Rewrite principalsfile when adding principals * Add update authorized_principals option to admin dashboard * Handle non-primary emails Signed-off-by: Andrew Thornton <art27@cantab.net> * Add the command actually to the dashboard template * Update models/ssh_key.go Co-authored-by: silverwind <me@silverwind.io> * By default do not show principal options unless there are CA keys set or they are explicitly set Signed-off-by: Andrew Thornton <art27@cantab.net> * allow settings when enabled * Fix typos in TrustedUserCAKeys path * Allow every CASignatureAlgorithms algorithm As this depends on the content of TrustedUserCAKeys we should allow all signature algorithms as admins can choose the specific algorithm on their signing CA * Update models/ssh_key.go Co-authored-by: Lauris BH <lauris@nix.lv> * Fix linting issue Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
2020-10-11 02:38:09 +02:00
registerRewriteAllPrincipalKeys()
registerRepositoryUpdateHook()
registerReinitMissingRepositories()
registerDeleteMissingRepositories()
registerRemoveRandomAvatars()
registerDeleteOldActions()
registerUpdateGiteaChecker()
registerDeleteOldSystemNotices()
registerGCLFS()
registerRebuildIssueIndexer()
}