[DB] Forgejo database migrations

- Implements https://codeberg.org/forgejo/discussions/issues/32#issuecomment-918737
- Allows to add Forgejo-specific migrations that don't interfere with Gitea's migration logic. Please do note that we cannot liberally add migrations for Gitea tables, as they might do their own migrations in a future version on that table, and that could undo our migrations. Luckily,  we don't have a scenario where that's needed and thus not taken into account.

Co-authored-by: Gusted <postmaster@gusted.xyz>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/795
(cherry picked from commit 8ee32978c0af1f8f71679c87f695df2b90b617c8)
(cherry picked from commit c240b34f59)
(cherry picked from commit 03936c649243a0a29701393d58e63e33064c7461)
(cherry picked from commit a20ed852f8b6d28872c05d688bffe5c6976bfa03)
(cherry picked from commit 1dfa82676f1feb745633618fde2d362bf19c4f28)
(cherry picked from commit c39ae0bf8abced8fd5dc32589e68515ac308b69b)
(cherry picked from commit cfaff08996c9f42592c95a63fe907b45b8a9317a)
(cherry picked from commit 94a458835a2b0336b26c1c9df64fdfe2de47f496)
(cherry picked from commit 61a3cf77dfe3f612ff110eb19f94dcb08051daf1)
(cherry picked from commit abb350fde879cc495761dc4616b7aa0fc5d94d54)
(cherry picked from commit 5194829d6b4ed702cf50ff875da57d04d77c8a18)
(cherry picked from commit 89239a60f23cad7dad03add744e23a4f3b10d6a4)
(cherry picked from commit 683cfd86efc5fa8cc04973ce3115351515a20917)
(cherry picked from commit f4546cfed92844e3666b80130eadabb9348b88ae)
(cherry picked from commit 86614d5826392b3fbe68355baeab9a0a761883a5)
(cherry picked from commit e4b9c32187a039a83686a82856a9a192919c6e82)
(cherry picked from commit 8c253719afa9b82f169757df007587d38560c06d)
(cherry picked from commit 857365d6c15b5471d63662b1d89d1523151c4f79)
(cherry picked from commit a488b3952f58bbf28bfa101a24e52dad7c9662eb)
(cherry picked from commit 98313c49109c941426beecc1a3e7887f28b99970)
(cherry picked from commit 430d95e8240971e266705d2e7202a5c785379cb2)
(cherry picked from commit 08bf9d918fbb67f5ac06c0cfdc24229aa14ff83f)
(cherry picked from commit f8a170e2d042fcb8f314e123de6918317ac1e909)
(cherry picked from commit d20e325378)
(cherry picked from commit 6c0aa7dd4f)
(cherry picked from commit 46c08c26c7bd3260b3ac7678f24566b467f4a2fb)
(cherry picked from commit 9ee22153c4ec62392693c9151d5395221d097f70)

[DB] Ensure forgejo migration up to date (squash)

- Hook Forgejo's `EnsureUpToDate` to Gitea's `EnsureUpToDate`, such that
the Forgejo migrations are also being checked to be up to date.
- I'm not sure how I missed this and if this has caused any problems,
but due to the lack of any open issue about it it seems to not be a big
problem.

(cherry picked from commit 6c65b6dcf6ab0d58e5c2d03a866e4e38294f72ad)
(cherry picked from commit 6d45c37d843147e69b0a27ebe35c617d7f574b76)

[DB] Add test for TestEnsureUpToDate (squash)

- Add a test for the behavior of `EnsureUpToDate`, to ensure it will
error when needed and succeed when the forgejo version is up to date.
- Add forgejo_migrations package to GO_TEST_PACKAGES, to avoid running
it with `test-unit` and instead test it with `test-*-migration`.

(cherry picked from commit b172a506914fee40a50daa51f0c8e547427fd2f8)
(cherry picked from commit d8af3088205b592340fd836135ffe97da9cec5a6)
(cherry picked from commit e69e64a32c5e38247e94ab880536e3cfeab67cc6)
(cherry picked from commit 4e8363fad4e08845960912a3ea3fe7265ee60602)
(cherry picked from commit fc9ecd6c533eca864503423cf4a21710984a6b75)
(cherry picked from commit e5c446e3dc9bc6e9549862f7b764a634f4fbaaae)
(cherry picked from commit 7066a15655a33f57ccfb68cf2cb994ea57ad3666)

Makefile

@@ -100,7 +100,7 @@ LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(G
 LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64
 
 GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
-GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
+GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) $(shell $(GO) list code.gitea.io/gitea/models/forgejo_migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
 
 FOMANTIC_WORK_DIR := web_src/fomantic
 
@@ -712,7 +712,7 @@ migrations.sqlite.test: $(GO_SOURCES) generate-ini-sqlite
 
 .PHONY: migrations.individual.mysql.test
 migrations.individual.mysql.test: $(GO_SOURCES)
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
+	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/... code.gitea.io/gitea/models/forgejo_migrations/...); do \
 		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/mysql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg; \
 	done
 
@@ -722,7 +722,7 @@ migrations.individual.sqlite.test\#%: $(GO_SOURCES) generate-ini-sqlite
 
 .PHONY: migrations.individual.pgsql.test
 migrations.individual.pgsql.test: $(GO_SOURCES)
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
+	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/... code.gitea.io/gitea/models/forgejo_migrations/...); do \
 		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/pgsql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg; \
 	done
 
@@ -733,7 +733,7 @@ migrations.individual.pgsql.test\#%: $(GO_SOURCES) generate-ini-pgsql
 
 .PHONY: migrations.individual.mssql.test
 migrations.individual.mssql.test: $(GO_SOURCES) generate-ini-mssql
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
+	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/... code.gitea.io/gitea/models/forgejo_migrations/...); do \
 		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/mssql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg -test.failfast; \
 	done
 
@@ -743,7 +743,7 @@ migrations.individual.mssql.test\#%: $(GO_SOURCES) generate-ini-mssql
 
 .PHONY: migrations.individual.sqlite.test
 migrations.individual.sqlite.test: $(GO_SOURCES) generate-ini-sqlite
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
+	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/... code.gitea.io/gitea/models/forgejo_migrations/...); do \
 		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/sqlite.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg; \
 	done
 

models/forgejo_migrations/main_test.go

@@ -0,0 +1,14 @@
+// Copyright 2023 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package forgejo_migrations //nolint:revive
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/migrations/base"
+)
+
+func TestMain(m *testing.M) {
+	base.MainTest(m)
+}

models/forgejo_migrations/migrate.go

@@ -0,0 +1,139 @@
+// Copyright 2023 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package forgejo_migrations //nolint:revive
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+	"xorm.io/xorm/names"
+)
+
+// ForgejoVersion describes the Forgejo version table. Should have only one row with id = 1.
+type ForgejoVersion struct {
+	ID      int64 `xorm:"pk autoincr"`
+	Version int64
+}
+
+type Migration struct {
+	description string
+	migrate     func(*xorm.Engine) error
+}
+
+// NewMigration creates a new migration.
+func NewMigration(desc string, fn func(*xorm.Engine) error) *Migration {
+	return &Migration{desc, fn}
+}
+
+// This is a sequence of additional Forgejo migrations.
+// Add new migrations to the bottom of the list.
+var migrations = []*Migration{}
+
+// GetCurrentDBVersion returns the current Forgejo database version.
+func GetCurrentDBVersion(x *xorm.Engine) (int64, error) {
+	if err := x.Sync(new(ForgejoVersion)); err != nil {
+		return -1, fmt.Errorf("sync: %w", err)
+	}
+
+	currentVersion := &ForgejoVersion{ID: 1}
+	has, err := x.Get(currentVersion)
+	if err != nil {
+		return -1, fmt.Errorf("get: %w", err)
+	}
+	if !has {
+		return -1, nil
+	}
+	return currentVersion.Version, nil
+}
+
+// ExpectedVersion returns the expected Forgejo database version.
+func ExpectedVersion() int64 {
+	return int64(len(migrations))
+}
+
+// EnsureUpToDate will check if the Forgejo database is at the correct version.
+func EnsureUpToDate(x *xorm.Engine) error {
+	currentDB, err := GetCurrentDBVersion(x)
+	if err != nil {
+		return err
+	}
+
+	if currentDB < 0 {
+		return fmt.Errorf("database has not been initialized")
+	}
+
+	expected := ExpectedVersion()
+
+	if currentDB != expected {
+		return fmt.Errorf(`current Forgejo database version %d is not equal to the expected version %d. Please run "forgejo [--config /path/to/app.ini] migrate" to update the database version`, currentDB, expected)
+	}
+
+	return nil
+}
+
+// Migrate Forgejo database to current version.
+func Migrate(x *xorm.Engine) error {
+	// Set a new clean the default mapper to GonicMapper as that is the default for .
+	x.SetMapper(names.GonicMapper{})
+	if err := x.Sync(new(ForgejoVersion)); err != nil {
+		return fmt.Errorf("sync: %w", err)
+	}
+
+	currentVersion := &ForgejoVersion{ID: 1}
+	has, err := x.Get(currentVersion)
+	if err != nil {
+		return fmt.Errorf("get: %w", err)
+	} else if !has {
+		// If the version record does not exist we think
+		// it is a fresh installation and we can skip all migrations.
+		currentVersion.ID = 0
+		currentVersion.Version = ExpectedVersion()
+
+		if _, err = x.InsertOne(currentVersion); err != nil {
+			return fmt.Errorf("insert: %w", err)
+		}
+	}
+
+	v := currentVersion.Version
+
+	// Downgrading Forgejo's database version not supported
+	if v > ExpectedVersion() {
+		msg := fmt.Sprintf("Your Forgejo database (migration version: %d) is for a newer version of Forgejo, you cannot use the newer database for this old Forgejo release (%d).", v, ExpectedVersion())
+		msg += "\nForgejo will exit to keep your database safe and unchanged. Please use the correct Forgejo release, do not change the migration version manually (incorrect manual operation may cause data loss)."
+		if !setting.IsProd {
+			msg += fmt.Sprintf("\nIf you are in development and really know what you're doing, you can force changing the migration version by executing: UPDATE forgejo_version SET version=%d WHERE id=1;", ExpectedVersion())
+		}
+		_, _ = fmt.Fprintln(os.Stderr, msg)
+		log.Fatal(msg)
+		return nil
+	}
+
+	// Some migration tasks depend on the git command
+	if git.DefaultContext == nil {
+		if err = git.InitSimple(context.Background()); err != nil {
+			return err
+		}
+	}
+
+	// Migrate
+	for i, m := range migrations[v:] {
+		log.Info("Migration[%d]: %s", v+int64(i), m.description)
+		// Reset the mapper between each migration - migrations are not supposed to depend on each other
+		x.SetMapper(names.GonicMapper{})
+		if err = m.migrate(x); err != nil {
+			return fmt.Errorf("migration[%d]: %s failed: %w", v+int64(i), m.description, err)
+		}
+		currentVersion.Version = v + int64(i) + 1
+		if _, err = x.ID(1).Update(currentVersion); err != nil {
+			return err
+		}
+	}
+	return nil
+}

models/forgejo_migrations/migrate_test.go

@@ -0,0 +1,39 @@
+// Copyright 2023 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package forgejo_migrations //nolint:revive
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/migrations/base"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestEnsureUpToDate tests the behavior of EnsureUpToDate.
+func TestEnsureUpToDate(t *testing.T) {
+	x, deferable := base.PrepareTestEnv(t, 0, new(ForgejoVersion))
+	defer deferable()
+	if x == nil || t.Failed() {
+		return
+	}
+
+	// Ensure error if there's no row in Forgejo Version.
+	err := EnsureUpToDate(x)
+	assert.Error(t, err)
+
+	// Insert 'good' Forgejo Version row.
+	_, err = x.InsertOne(&ForgejoVersion{ID: 1, Version: ExpectedVersion()})
+	assert.NoError(t, err)
+
+	err = EnsureUpToDate(x)
+	assert.NoError(t, err)
+
+	// Modify forgejo version to have a lower version.
+	_, err = x.Exec("UPDATE `forgejo_version` SET version = ? WHERE id = 1", ExpectedVersion()-1)
+	assert.NoError(t, err)
+
+	err = EnsureUpToDate(x)
+	assert.Error(t, err)
+}

models/migrations/migrations.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 
+	"code.gitea.io/gitea/models/forgejo_migrations"
 	"code.gitea.io/gitea/models/migrations/v1_10"
 	"code.gitea.io/gitea/models/migrations/v1_11"
 	"code.gitea.io/gitea/models/migrations/v1_12"
@@ -597,7 +598,7 @@ func EnsureUpToDate(x *xorm.Engine) error {
 		return fmt.Errorf(`Current database version %d is not equal to the expected version %d. Please run "gitea [--config /path/to/app.ini] migrate" to update the database version`, currentDB, expected)
 	}
 
-	return nil
+	return forgejo_migrations.EnsureUpToDate(x)
 }
 
 // Migrate database to current version
@@ -661,5 +662,7 @@ Please try upgrading to a lower version first (suggested v1.6.4), then upgrade t
 			return err
 		}
 	}
-	return nil
+
+	// Execute Forgejo specific migrations.
+	return forgejo_migrations.Migrate(x)
 }