From 40ffe2d226f1ea1e78ae40eb0deeccc0838b5d48 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Fri, 11 Aug 2023 19:12:44 +0200 Subject: [PATCH] [TESTS] upgrade tests for storage (cherry picked from commit 884ca63738cc2e2c7cde31c649e9fa77cd590044) (cherry picked from commit 0a45d9c37b187249d347020ca79e71c523177d61) [TESTS] upgrade tests for storage (squash) relative paths (cherry picked from commit 3bb19285f365ae0b56dedc183db04504946a2a50) (cherry picked from commit c640c09e6110a8c684dfcc8f9fc05495f30c16f1) --- .forgejo/upgrades/default-app.ini | 26 ++ .forgejo/upgrades/legagy-relative-app.ini | 28 ++ .forgejo/upgrades/merged-app.ini | 28 ++ .forgejo/upgrades/misplace-app.ini | 55 ++++ .forgejo/upgrades/relative-app.ini | 40 +++ .forgejo/upgrades/specific-app.ini | 43 +++ .forgejo/upgrades/storage-relative-app.ini | 40 +++ .forgejo/upgrades/test-upgrade.sh | 364 +++++++++++++++++++++ .forgejo/workflows/testing.yml | 30 ++ 9 files changed, 654 insertions(+) create mode 100644 .forgejo/upgrades/default-app.ini create mode 100644 .forgejo/upgrades/legagy-relative-app.ini create mode 100644 .forgejo/upgrades/merged-app.ini create mode 100644 .forgejo/upgrades/misplace-app.ini create mode 100644 .forgejo/upgrades/relative-app.ini create mode 100644 .forgejo/upgrades/specific-app.ini create mode 100644 .forgejo/upgrades/storage-relative-app.ini create mode 100755 .forgejo/upgrades/test-upgrade.sh diff --git a/.forgejo/upgrades/default-app.ini b/.forgejo/upgrades/default-app.ini new file mode 100644 index 0000000000..8ae71431ea --- /dev/null +++ b/.forgejo/upgrades/default-app.ini @@ -0,0 +1,26 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 +PATH = ${WORK_PATH}/forgejo.db + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[actions] +ENABLED = true diff --git a/.forgejo/upgrades/legagy-relative-app.ini b/.forgejo/upgrades/legagy-relative-app.ini new file mode 100644 index 0000000000..078b0da671 --- /dev/null +++ b/.forgejo/upgrades/legagy-relative-app.ini @@ -0,0 +1,28 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true +LFS_CONTENT_PATH = relative-lfs + +[database] +DB_TYPE = sqlite3 +PATH = ${WORK_PATH}/forgejo.db + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[picture] +AVATAR_UPLOAD_PATH = relative-avatars +REPOSITORY_AVATAR_UPLOAD_PATH = relative-repo-avatars diff --git a/.forgejo/upgrades/merged-app.ini b/.forgejo/upgrades/merged-app.ini new file mode 100644 index 0000000000..b812f1f042 --- /dev/null +++ b/.forgejo/upgrades/merged-app.ini @@ -0,0 +1,28 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[actions] +ENABLED = true + +[storage] +PATH = ${WORK_PATH}/merged diff --git a/.forgejo/upgrades/misplace-app.ini b/.forgejo/upgrades/misplace-app.ini new file mode 100644 index 0000000000..42ebaab0d4 --- /dev/null +++ b/.forgejo/upgrades/misplace-app.ini @@ -0,0 +1,55 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[actions] +ENABLED = true + +[attachment] + +[storage.attachments] +PATH = ${WORK_PATH}/data/attachments + +[lfs] + +[storage.lfs] +PATH = ${WORK_PATH}/data/lfs + +[avatar] + +[storage.avatars] +PATH = ${WORK_PATH}/data/avatars + +[repo-avatar] + +[storage.repo-avatars] +PATH = ${WORK_PATH}/data/repo-avatars + +[repo-archive] + +[storage.repo-archive] +PATH = ${WORK_PATH}/data/repo-archive + +[packages] + +[storage.packages] +PATH = ${WORK_PATH}/data/packages diff --git a/.forgejo/upgrades/relative-app.ini b/.forgejo/upgrades/relative-app.ini new file mode 100644 index 0000000000..b587500dcf --- /dev/null +++ b/.forgejo/upgrades/relative-app.ini @@ -0,0 +1,40 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[attachment] +PATH = relative-attachments + +[lfs] +PATH = relative-lfs + +[avatar] +PATH = relative-avatars + +[repo-avatar] +PATH = relative-repo-avatars + +[repo-archive] +PATH = relative-repo-archive + +[packages] +PATH = relative-packages diff --git a/.forgejo/upgrades/specific-app.ini b/.forgejo/upgrades/specific-app.ini new file mode 100644 index 0000000000..c4e36e1e99 --- /dev/null +++ b/.forgejo/upgrades/specific-app.ini @@ -0,0 +1,43 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[actions] +ENABLED = true + +[attachment] +PATH = ${WORK_PATH}/data/attachments + +[lfs] +PATH = ${WORK_PATH}/data/lfs + +[avatar] +PATH = ${WORK_PATH}/data/avatars + +[repo-avatar] +PATH = ${WORK_PATH}/data/repo-avatars + +[repo-archive] +PATH = ${WORK_PATH}/data/repo-archive + +[packages] +PATH = ${WORK_PATH}/data/packages diff --git a/.forgejo/upgrades/storage-relative-app.ini b/.forgejo/upgrades/storage-relative-app.ini new file mode 100644 index 0000000000..c413811424 --- /dev/null +++ b/.forgejo/upgrades/storage-relative-app.ini @@ -0,0 +1,40 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[storage.attachments] +PATH = relative-attachments + +[storage.lfs] +PATH = relative-lfs + +[storage.avatars] +PATH = relative-avatars + +[storage.repo-avatars] +PATH = relative-repo-avatars + +[storage.repo-archive] +PATH = relative-repo-archive + +[storage.packages] +PATH = relative-packages diff --git a/.forgejo/upgrades/test-upgrade.sh b/.forgejo/upgrades/test-upgrade.sh new file mode 100755 index 0000000000..3630907b85 --- /dev/null +++ b/.forgejo/upgrades/test-upgrade.sh @@ -0,0 +1,364 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +# +# Debug loop from the source tree: +# +# ./.forgejo/upgrades/test-upgrade.sh dependencies +# ./.forgejo/upgrades/test-upgrade.sh build_all +# ./.forgejo/upgrades/test-upgrade.sh test_downgrade_1.20.2_fails +# +# Everything happens in /tmp/forgejo-upgrades +# + +set -ex + +HOST_PORT=0.0.0.0:3000 +STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" +DIR=/tmp/forgejo-upgrades +SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' + +function maybe_sudo() { + if test $(id -u) != 0 ; then + SUDO=sudo + fi +} + +function dependencies() { + if ! which curl daemon > /dev/null ; then + maybe_sudo + $SUDO apt-get install -y -qq curl daemon + fi +} + +function build() { + local version=$1 + local semver=$2 + + if ! test -f $DIR/forgejo-$version ; then + mkdir -p $DIR + make VERSION=v$version GITEA_VERSION=v$version FORGEJO_VERSION=$semver TAGS='bindata sqlite sqlite_unlock_notify' generate gitea + mv gitea $DIR/forgejo-$version + fi +} + +function build_all() { + test -f Makefile + build 1.21.0-0 6.0.0+0-gitea-1.21.0 +} + +function wait_for() { + rm -f $DIR/setup-forgejo.out + success=false + for delay in 1 1 5 5 15 ; do + if "$@" >> $DIR/setup-forgejo.out 2>&1 ; then + success=true + break + fi + cat $DIR/setup-forgejo.out + echo waiting $delay + sleep $delay + done + if test $success = false ; then + cat $DIR/setup-forgejo.out + return 1 + fi +} + +function download() { + local version=$1 + + if ! test -f $DIR/forgejo-$version ; then + mkdir -p $DIR + wget -O $DIR/forgejo-$version --quiet https://codeberg.org/forgejo/forgejo/releases/download/v$version/forgejo-$version-linux-amd64 + chmod +x $DIR/forgejo-$version + fi +} + +function cleanup_logs() { + local work_path=$DIR/forgejo-work-path + + rm -f $DIR/*.log + rm -f $work_path/log/*.log +} + +function start() { + local version=$1 + + download $version + local work_path=$DIR/forgejo-work-path + daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path + if ! wait_for grep 'Starting server on' $work_path/log/forgejo.log ; then + cat $DIR/*.log + cat $work_path/log/*.log + return 1 + fi + create_user $version + $work_path/forgejo-api http://${HOST_PORT}/api/v1/version +} + +function create_user() { + local version=$1 + + local work_path=$DIR/forgejo-work-path + + if test -f $work_path/forgejo-token; then + return + fi + + local user=root + local password=admin1234 + local cli="$DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path" + $cli admin user create --admin --username "$user" --password "$password" --email "$user@example.com" + local scopes="--scopes all" + if echo $version | grep --quiet 1.18. ; then + scopes="" + fi + $cli admin user generate-access-token -u $user --raw $scopes > $work_path/forgejo-token + ( echo -n 'Authorization: token ' ; cat $work_path/forgejo-token ) > $work_path/forgejo-header + ( echo "#!/bin/sh" ; echo 'curl -sS -H "Content-Type: application/json" -H @'$work_path/forgejo-header' "$@"' ) > $work_path/forgejo-api && chmod +x $work_path/forgejo-api +} + +function stop() { + if test -f $DIR/forgejo-pid ; then + local pid=$(cat $DIR/forgejo-pid) + kill -TERM $pid + pidwait $pid || true + for delay in 1 1 2 2 5 5 ; do + if ! test -f $DIR/forgejo-pid ; then + break + fi + sleep $delay + done + ! test -f $DIR/forgejo-pid + fi + cleanup_logs +} + +function reset() { + local config=$1 + local work_path=$DIR/forgejo-work-path + rm -fr $work_path + mkdir -p $work_path + WORK_PATH=$work_path envsubst < $SELF_DIR/$config-app.ini > $work_path/app.ini +} + +function verify_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/$path + done +} + +function cleanup_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + rm -fr $work_path/data/$path + done +} + +function test_downgrade_1.20.2_fails() { + local work_path=$DIR/forgejo-work-path + + echo "================ See also https://codeberg.org/forgejo/forgejo/pulls/1225" + + echo "================ downgrading from 1.20.3-0 to 1.20.2-0 fails" + stop + reset default + start 1.20.3-0 + stop + download 1.20.2-0 + timeout 60 $DIR/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true + if ! grep --fixed-strings --quiet 'use the newer database' $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi +} + +function test_bug_storage_merged() { + local work_path=$DIR/forgejo-work-path + + echo "================ See also https://codeberg.org/forgejo/forgejo/pulls/1225" + + echo "================ using < 1.20.3-0 and [storage].PATH merge all storage" + for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do + stop + reset merged + start $version + for path in ${STORAGE_PATHS} ; do + ! test -d $work_path/data/$path + done + for path in ${STORAGE_PATHS} ; do + ! test -d $work_path/merged/$path + done + test -d $work_path/merged + done + stop + + echo "================ upgrading from 1.20.2-0 with [storage].PATH fails" + download 1.20.3-0 + timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + if ! grep --fixed-strings --quiet '[storage].PATH is set and may create storage issues' $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi +} + +function test_bug_storage_relative_path() { + local work_path=$DIR/forgejo-work-path + + echo "================ using < 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to WORK_PATH" + for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do + stop + reset legagy-relative + start $version + test -d $work_path/relative-lfs + test -d $work_path/relative-avatars + test -d $work_path/relative-repo-avatars + done + + echo "================ using >= 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to APP_DATA_PATH" + for version in 1.20.3-0 1.21.0-0 ; do + stop + reset legagy-relative + start $version + test -d $work_path/data/relative-lfs + test -d $work_path/data/relative-avatars + test -d $work_path/data/relative-repo-avatars + done + + echo "================ using >= 1.20.3-0 relative [storage.XXXX].PATHS are relative to APP_DATA_PATH" + for version in 1.20.3-0 1.21.0-0 ; do + stop + reset storage-relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/relative-$path + done + done + + echo "================ using 1.20.[12]-0 relative [storage.XXXX].PATHS are inconsistent" + for version in 1.20.2-0 ; do + stop + reset storage-relative + start $version + test -d $work_path/data/packages + test -d $work_path/relative-repo-archive + test -d $work_path/relative-attachments + test -d $work_path/relative-lfs + test -d $work_path/data/avatars + test -d $work_path/data/repo-avatars + done + + echo "================ using < 1.20 relative [storage.XXXX].PATHS are inconsistent" + for version in 1.18.5-0 1.19.4-0 ; do + stop + reset storage-relative + start $version + test -d $work_path/relative-packages + test -d $work_path/relative-repo-archive + test -d $work_path/relative-attachments + test -d $work_path/data/lfs + test -d $work_path/data/avatars + test -d $work_path/data/repo-avatars + done + + echo "================ using < 1.20.3-0 relative [XXXX].PATHS are relative to WORK_PATH" + for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do + stop + reset relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/relative-$path + done + done + + echo "================ using >= 1.20.3-0 relative [XXXX].PATHS are relative to APP_DATA_PATH" + for version in 1.20.3-0 1.21.0-0 ; do + stop + reset relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/relative-$path + done + done + + stop +} + +function test_bug_storage_misplace() { + local work_path=$DIR/forgejo-work-path + + echo "================ See also https://codeberg.org/forgejo/forgejo/pulls/1225" + + echo "================ using < 1.20 and conflicting sections misplace storage" + for version in 1.18.5-0 1.19.4-0 ; do + stop + reset misplace + start $version + # + # some storage are where they should be + # + test -d $work_path/data/packages + test -d $work_path/data/repo-archive + test -d $work_path/data/attachments + # + # others are under APP_DATA_PATH + # + test -d $work_path/elsewhere/lfs + test -d $work_path/elsewhere/avatars + test -d $work_path/elsewhere/repo-avatars + done + + echo "================ using < 1.20.[12]-0 and conflicting sections ignores [storage.*]" + for version in 1.20.2-0 ; do + stop + reset misplace + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/elsewhere/$path + done + done + + stop + + echo "================ upgrading from 1.20.2-0 with conflicting sections fails" + download 1.20.3-0 + timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + for path in ${STORAGE_PATHS} ; do + if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi + done +} + +function test_successful_upgrades() { + for config in default specific ; do + echo "================ using $config app.ini" + reset $config + + for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 1.21.0-0 ; do + echo "================ run $version" + cleanup_storage + start $version + verify_storage + stop + done + done +} + +function test_upgrades() { + stop + dependencies + build_all + test_successful_upgrades + test_bug_storage_misplace + test_bug_storage_merged + test_downgrade_1.20.2_fails +} + +"$@" diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 06d1b4fa21..f56d589d53 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -165,3 +165,33 @@ jobs: RACE_ENABLED: true TEST_TAGS: gogit sqlite sqlite_unlock_notify USE_REPO_TEST_DIR: 1 + upgrade: + needs: [test-sqlite] + runs-on: docker + container: + image: codeberg.org/forgejo/test_env:main + steps: + - uses: https://code.forgejo.org/actions/checkout@v3 + - uses: https://code.forgejo.org/actions/setup-go@v4 + with: + go-version: "1.20" + - run: | + git config --add safe.directory '*' + chown -R gitea:gitea . /go + - run: | + su gitea -c 'make deps-backend' + - run: | + su gitea -c 'make backend' + env: + TAGS: bindata sqlite sqlite_unlock_notify + - run: | + su gitea -c 'make gitea' + cp -a gitea /tmp/forgejo-development + timeout-minutes: 50 + env: + TAGS: bindata sqlite sqlite_unlock_notify + - run: | + script=$(pwd)/.forgejo/upgrades/test-upgrade.sh + $script dependencies + su gitea -c "$script test_upgrades" +