mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-05-11 04:05:10 +02:00
feat: use XORM EngineGroup instead of single Engine connection (#7212)
Resolves #7207 Add new configuration to make XORM work with a main and replicas database instances. The follow configuration parameters were added: - `HOST_PRIMARY` - `HOST_REPLICAS` - `LOAD_BALANCE_POLICY`. Options: - `"WeightRandom"` -> `xorm.WeightRandomPolicy` - `"WeightRoundRobin` -> `WeightRoundRobinPolicy` - `"LeastCon"` -> `LeastConnPolicy` - `"RoundRobin"` -> `xorm.RoundRobinPolicy()` - default: `xorm.RandomPolicy()` - `LOAD_BALANCE_WEIGHTS` Co-authored-by: pat-s <patrick.schratz@gmail.com@> Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/7212 Reviewed-by: Gusted <gusted@noreply.codeberg.org> Co-authored-by: pat-s <patrick.schratz@gmail.com> Co-committed-by: pat-s <patrick.schratz@gmail.com>
This commit is contained in:
parent
a23d0453a3
commit
63a80bf2b9
19 changed files with 463 additions and 129 deletions
|
@ -10,8 +10,13 @@ import (
|
|||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"forgejo.org/modules/log"
|
||||
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -24,35 +29,41 @@ var (
|
|||
EnableSQLite3 bool
|
||||
|
||||
// Database holds the database settings
|
||||
Database = struct {
|
||||
Type DatabaseType
|
||||
Host string
|
||||
Name string
|
||||
User string
|
||||
Passwd string
|
||||
Schema string
|
||||
SSLMode string
|
||||
Path string
|
||||
LogSQL bool
|
||||
MysqlCharset string
|
||||
CharsetCollation string
|
||||
Timeout int // seconds
|
||||
SQLiteJournalMode string
|
||||
DBConnectRetries int
|
||||
DBConnectBackoff time.Duration
|
||||
MaxIdleConns int
|
||||
MaxOpenConns int
|
||||
ConnMaxIdleTime time.Duration
|
||||
ConnMaxLifetime time.Duration
|
||||
IterateBufferSize int
|
||||
AutoMigration bool
|
||||
SlowQueryThreshold time.Duration
|
||||
}{
|
||||
Database = DatabaseSettings{
|
||||
Timeout: 500,
|
||||
IterateBufferSize: 50,
|
||||
}
|
||||
)
|
||||
|
||||
type DatabaseSettings struct {
|
||||
Type DatabaseType
|
||||
Host string
|
||||
HostPrimary string
|
||||
HostReplica string
|
||||
LoadBalancePolicy string
|
||||
LoadBalanceWeights string
|
||||
Name string
|
||||
User string
|
||||
Passwd string
|
||||
Schema string
|
||||
SSLMode string
|
||||
Path string
|
||||
LogSQL bool
|
||||
MysqlCharset string
|
||||
CharsetCollation string
|
||||
Timeout int // seconds
|
||||
SQLiteJournalMode string
|
||||
DBConnectRetries int
|
||||
DBConnectBackoff time.Duration
|
||||
MaxIdleConns int
|
||||
MaxOpenConns int
|
||||
ConnMaxIdleTime time.Duration
|
||||
ConnMaxLifetime time.Duration
|
||||
IterateBufferSize int
|
||||
AutoMigration bool
|
||||
SlowQueryThreshold time.Duration
|
||||
}
|
||||
|
||||
// LoadDBSetting loads the database settings
|
||||
func LoadDBSetting() {
|
||||
loadDBSetting(CfgProvider)
|
||||
|
@ -63,6 +74,10 @@ func loadDBSetting(rootCfg ConfigProvider) {
|
|||
Database.Type = DatabaseType(sec.Key("DB_TYPE").String())
|
||||
|
||||
Database.Host = sec.Key("HOST").String()
|
||||
Database.HostPrimary = sec.Key("HOST_PRIMARY").String()
|
||||
Database.HostReplica = sec.Key("HOST_REPLICA").String()
|
||||
Database.LoadBalancePolicy = sec.Key("LOAD_BALANCE_POLICY").String()
|
||||
Database.LoadBalanceWeights = sec.Key("LOAD_BALANCE_WEIGHTS").String()
|
||||
Database.Name = sec.Key("NAME").String()
|
||||
Database.User = sec.Key("USER").String()
|
||||
if len(Database.Passwd) == 0 {
|
||||
|
@ -99,8 +114,93 @@ func loadDBSetting(rootCfg ConfigProvider) {
|
|||
}
|
||||
}
|
||||
|
||||
// DBConnStr returns database connection string
|
||||
func DBConnStr() (string, error) {
|
||||
// DBMasterConnStr returns the connection string for the master (primary) database.
|
||||
// If a primary host is defined in the configuration, it is used;
|
||||
// otherwise, it falls back to Database.Host.
|
||||
// Returns an error if no master host is provided but a slave is defined.
|
||||
func DBMasterConnStr() (string, error) {
|
||||
var host string
|
||||
if Database.HostPrimary != "" {
|
||||
host = Database.HostPrimary
|
||||
} else {
|
||||
host = Database.Host
|
||||
}
|
||||
if host == "" && Database.HostReplica != "" {
|
||||
return "", errors.New("master host is not defined while slave is defined; cannot proceed")
|
||||
}
|
||||
|
||||
// For SQLite, no host is needed
|
||||
if host == "" && !Database.Type.IsSQLite3() {
|
||||
return "", errors.New("no database host defined")
|
||||
}
|
||||
|
||||
return dbConnStrWithHost(host)
|
||||
}
|
||||
|
||||
// DBSlaveConnStrs returns one or more connection strings for the replica databases.
|
||||
// If a replica host is defined (possibly as a comma-separated list) then those DSNs are returned.
|
||||
// Otherwise, this function falls back to the master DSN (with a warning log).
|
||||
func DBSlaveConnStrs() ([]string, error) {
|
||||
var dsns []string
|
||||
if Database.HostReplica != "" {
|
||||
// support multiple replica hosts separated by commas
|
||||
replicas := strings.SplitSeq(Database.HostReplica, ",")
|
||||
for r := range replicas {
|
||||
trimmed := strings.TrimSpace(r)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
dsn, err := dbConnStrWithHost(trimmed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dsns = append(dsns, dsn)
|
||||
}
|
||||
}
|
||||
// Fall back to master if no slave DSN was provided.
|
||||
if len(dsns) == 0 {
|
||||
master, err := DBMasterConnStr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug("Database: No dedicated replica host defined; falling back to primary DSN for replica connections")
|
||||
dsns = append(dsns, master)
|
||||
}
|
||||
return dsns, nil
|
||||
}
|
||||
|
||||
func BuildLoadBalancePolicy(settings *DatabaseSettings, slaveEngines []*xorm.Engine) xorm.GroupPolicy {
|
||||
var policy xorm.GroupPolicy
|
||||
switch settings.LoadBalancePolicy { // Use the settings parameter directly
|
||||
case "WeightRandom":
|
||||
var weights []int
|
||||
if settings.LoadBalanceWeights != "" { // Use the settings parameter directly
|
||||
for part := range strings.SplitSeq(settings.LoadBalanceWeights, ",") {
|
||||
w, err := strconv.Atoi(strings.TrimSpace(part))
|
||||
if err != nil {
|
||||
w = 1 // use a default weight if conversion fails
|
||||
}
|
||||
weights = append(weights, w)
|
||||
}
|
||||
}
|
||||
// If no valid weights were provided, default each slave to weight 1
|
||||
if len(weights) == 0 {
|
||||
weights = make([]int, len(slaveEngines))
|
||||
for i := range weights {
|
||||
weights[i] = 1
|
||||
}
|
||||
}
|
||||
policy = xorm.WeightRandomPolicy(weights)
|
||||
case "RoundRobin":
|
||||
policy = xorm.RoundRobinPolicy()
|
||||
default:
|
||||
policy = xorm.RandomPolicy()
|
||||
}
|
||||
return policy
|
||||
}
|
||||
|
||||
// dbConnStrWithHost constructs the connection string, given a host value.
|
||||
func dbConnStrWithHost(host string) (string, error) {
|
||||
var connStr string
|
||||
paramSep := "?"
|
||||
if strings.Contains(Database.Name, paramSep) {
|
||||
|
@ -109,23 +209,25 @@ func DBConnStr() (string, error) {
|
|||
switch Database.Type {
|
||||
case "mysql":
|
||||
connType := "tcp"
|
||||
if len(Database.Host) > 0 && Database.Host[0] == '/' { // looks like a unix socket
|
||||
// if the host starts with '/' it is assumed to be a unix socket path
|
||||
if len(host) > 0 && host[0] == '/' {
|
||||
connType = "unix"
|
||||
}
|
||||
tls := Database.SSLMode
|
||||
if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
|
||||
// allow the "disable" value (borrowed from Postgres defaults) to behave as false
|
||||
if tls == "disable" {
|
||||
tls = "false"
|
||||
}
|
||||
connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%sparseTime=true&tls=%s",
|
||||
Database.User, Database.Passwd, connType, Database.Host, Database.Name, paramSep, tls)
|
||||
Database.User, Database.Passwd, connType, host, Database.Name, paramSep, tls)
|
||||
case "postgres":
|
||||
connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Database.SSLMode)
|
||||
connStr = getPostgreSQLConnectionString(host, Database.User, Database.Passwd, Database.Name, Database.SSLMode)
|
||||
case "sqlite3":
|
||||
if !EnableSQLite3 {
|
||||
return "", errors.New("this Gitea binary was not built with SQLite3 support")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(Database.Path), os.ModePerm); err != nil {
|
||||
return "", fmt.Errorf("Failed to create directories: %w", err)
|
||||
return "", fmt.Errorf("failed to create directories: %w", err)
|
||||
}
|
||||
journalMode := ""
|
||||
if Database.SQLiteJournalMode != "" {
|
||||
|
@ -136,7 +238,6 @@ func DBConnStr() (string, error) {
|
|||
default:
|
||||
return "", fmt.Errorf("unknown database type: %s", Database.Type)
|
||||
}
|
||||
|
||||
return connStr, nil
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue