From b44ee44ce85f0c55df96fbd179b005506db7787e Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Fri, 11 Aug 2023 19:12:44 +0200 Subject: [PATCH] [TESTS] upgrade tests for storage (cherry picked from commit 884ca63738cc2e2c7cde31c649e9fa77cd590044) (cherry picked from commit 0a45d9c37b187249d347020ca79e71c523177d61) [TESTS] upgrade tests for storage (squash) relative paths (cherry picked from commit 3bb19285f365ae0b56dedc183db04504946a2a50) (cherry picked from commit c640c09e6110a8c684dfcc8f9fc05495f30c16f1) (cherry picked from commit 40ffe2d226f1ea1e78ae40eb0deeccc0838b5d48) [UPGRADE] S3 storage and fixtures (cherry picked from commit c466c9c6571b18b1426e07439e7aa6445f5d86dd) (cherry picked from commit e80abbe2cd7968c5c5601bf5e39de9b265e40ca4) (cherry picked from commit 50a47df1d17df60cda6504165d5b02ac42e63c86) [UPGRADE] add sanity checks for [storage*] (squash) speedup upgrade tests (cherry picked from commit f578279cfe85d4e23b83d3759ec782790fed869d) (cherry picked from commit 6dd677f151d3ba3b6f016d788585372958608cd6) (cherry picked from commit ee7e7eca27a2e123188cdf05fb493f51c8c8a3e4) (cherry picked from commit 879ffca697e6cd04d364ad46a6d4a2d957ba3eb3) --- .forgejo/upgrades/default-app.ini | 30 ++ .forgejo/upgrades/fixtures.sh | 187 +++++++ .forgejo/upgrades/legagy-relative-app.ini | 32 ++ .forgejo/upgrades/merged-app.ini | 32 ++ .forgejo/upgrades/misplace-app.ini | 59 +++ .forgejo/upgrades/misplace-s3-app.ini | 89 ++++ .forgejo/upgrades/relative-app.ini | 44 ++ .forgejo/upgrades/specific-app.ini | 47 ++ .forgejo/upgrades/stable-s3-app.ini | 39 ++ .forgejo/upgrades/storage-relative-app.ini | 44 ++ .forgejo/upgrades/test-upgrade.sh | 590 +++++++++++++++++++++ .forgejo/workflows/upgrade.yml | 45 ++ 12 files changed, 1238 insertions(+) create mode 100644 .forgejo/upgrades/default-app.ini create mode 100644 .forgejo/upgrades/fixtures.sh create mode 100644 .forgejo/upgrades/legagy-relative-app.ini create mode 100644 .forgejo/upgrades/merged-app.ini create mode 100644 .forgejo/upgrades/misplace-app.ini create mode 100644 .forgejo/upgrades/misplace-s3-app.ini create mode 100644 .forgejo/upgrades/relative-app.ini create mode 100644 .forgejo/upgrades/specific-app.ini create mode 100644 .forgejo/upgrades/stable-s3-app.ini create mode 100644 .forgejo/upgrades/storage-relative-app.ini create mode 100755 .forgejo/upgrades/test-upgrade.sh create mode 100644 .forgejo/workflows/upgrade.yml diff --git a/.forgejo/upgrades/default-app.ini b/.forgejo/upgrades/default-app.ini new file mode 100644 index 0000000000..a51290a8b2 --- /dev/null +++ b/.forgejo/upgrades/default-app.ini @@ -0,0 +1,30 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 +PATH = ${WORK_PATH}/forgejo.db + +[log] +MODE = file +LEVEL = trace +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true diff --git a/.forgejo/upgrades/fixtures.sh b/.forgejo/upgrades/fixtures.sh new file mode 100644 index 0000000000..1b8cb44cac --- /dev/null +++ b/.forgejo/upgrades/fixtures.sh @@ -0,0 +1,187 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +#ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +# +# one pixel scaled to 290x290 because that's what versions lower or equal to v1.19.4-0 want +# by default and any other size will be transformed which make it difficult to compare. +# +ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAASIAAAEiCAYAAABdvt+2AAADrElEQVR4nOzUMRHAMADEsL9eeQd6AsOLhMCT/7udAYS+OgDAiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDuBQAA//+4jAPFe1H1tgAAAABJRU5ErkJggg==" + +function fixture_get_paths_s3() { + local path=$1 + + ( + echo -n $path/ + mc ls --quiet --recursive testS3/$path | sed -e 's/.* //' + ) > $DIR/path +} + +function fixture_get_paths_local() { + local path=$1 + local work_path=$DIR/forgejo-work-path + + ( cd $work_path ; find $path -type f) > $DIR/path +} + +function fixture_get_one_path() { + local storage=$1 + local path=$2 + + fixture_get_paths_$storage $path + + if test $(wc -l < $DIR/path) != 1 ; then + echo expected one path but got + cat $DIR/path + return 1 + fi + cat $DIR/path +} + +function fixture_repo_archive_create() { + retry curl -f -sS http://${HOST_PORT}/root/fixture/archive/main.zip -o /dev/null +} + +function fixture_repo_archive_assert_s3() { + mc ls --recursive testS3/forgejo/repo-archive | grep --quiet '.zip$' +} + +function fixture_repo_archive_assert_local() { + local path=$1 + local work_path=$DIR/forgejo-work-path + + find $work_path/$path | grep --quiet '.zip$' +} + +function fixture_lfs_create() { + ( + cd $DIR/fixture + git lfs track "*.txt" + echo CONTENT > file.txt + git add . + git commit -m 'lfs files' + git push + ) +} + +function fixture_lfs_assert_s3() { + local content=$(mc cat testS3/forgejo/lfs/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7) + test "$content" = CONTENT +} + +function fixture_lfs_assert_local() { + local path=$1 + local work_path=$DIR/forgejo-work-path + + local content=$(mc cat $work_path/$path/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7) + test "$content" = CONTENT +} + +function fixture_packages_create() { + echo PACKAGE_CONTENT > $DIR/fixture/package + $work_path/forgejo-api -X DELETE http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt || true + $work_path/forgejo-api --upload-file $DIR/fixture/package http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt +} + +function fixture_packages_assert_s3() { + local content=$(mc cat testS3/forgejo/packages/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17) + test "$content" = PACKAGE_CONTENT +} + +function fixture_packages_assert_local() { + local path=$1 + + local content=$(cat $work_path/$path/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17) + test "$content" = PACKAGE_CONTENT +} + +function fixture_avatars_create() { + echo -n $ONEPIXEL | base64 --decode > $DIR/avatar.png + $work_path/forgejo-client --form avatar=@$DIR/avatar.png http://${HOST_PORT}/user/settings/avatar +} + +function fixture_avatars_assert_s3() { + local filename=$(fixture_get_one_path s3 forgejo/avatars) + local content=$(mc cat testS3/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_avatars_assert_local() { + local path=$1 + + local filename=$(fixture_get_one_path local $path) + local content=$(cat $work_path/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_repo_avatars_create() { + echo -n $ONEPIXEL | base64 --decode > $DIR/repo-avatar.png + $work_path/forgejo-client --form avatar=@$DIR/repo-avatar.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/settings/avatar + # v1.21 only + #$work_path/forgejo-api -X POST --data-raw '{"body":"'$avatar'"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/avatar +} + +function fixture_repo_avatars_assert_s3() { + local filename=$(fixture_get_one_path s3 forgejo/repo-avatars) + local content=$(mc cat testS3/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_repo_avatars_assert_local() { + local path=$1 + + local filename=$(fixture_get_one_path local $path) + local content=$(cat $work_path/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_attachments_create_1_18() { + echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png + $work_path/forgejo-client --trace-ascii - --form file=@$DIR/attachment.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/issues/attachments +} + +function fixture_attachments_create() { + if $work_path/forgejo-api http://${HOST_PORT}/api/v1/version | grep --quiet --fixed-strings 1.18. ; then + fixture_attachments_create_1_18 + return + fi + id=$($work_path/forgejo-api --data-raw '{"title":"TITLE"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues | jq .id) + echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png + $work_path/forgejo-client -H @$DIR/forgejo-work-path/forgejo-header --form name=attachment.png --form attachment=@$DIR/attachment.png http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues/$id/assets +} + +function fixture_attachments_assert_s3() { + local filename=$(fixture_get_one_path s3 forgejo/attachments) + local content=$(mc cat testS3/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_attachments_assert_local() { + local path=$1 + + local filename=$(fixture_get_one_path local $path) + local content=$(cat $work_path/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_create() { + local work_path=$DIR/forgejo-work-path + + rm -fr $DIR/fixture + mkdir -p $DIR/fixture + ( + cd $DIR/fixture + git init + git checkout -b main + git remote add origin http://${FORGEJO_USER}:${FORGEJO_PASSWORD}@${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO} + git config user.email root@example.com + git config user.name username + echo SOMETHING > README + git add README + git commit -m 'initial commit' + git push --set-upstream --force origin main + ) + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_create + done +} diff --git a/.forgejo/upgrades/legagy-relative-app.ini b/.forgejo/upgrades/legagy-relative-app.ini new file mode 100644 index 0000000000..130294a4aa --- /dev/null +++ b/.forgejo/upgrades/legagy-relative-app.ini @@ -0,0 +1,32 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true +LFS_CONTENT_PATH = relative-lfs + +[database] +DB_TYPE = sqlite3 +PATH = ${WORK_PATH}/forgejo.db + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[picture] +AVATAR_UPLOAD_PATH = relative-avatars +REPOSITORY_AVATAR_UPLOAD_PATH = relative-repo-avatars diff --git a/.forgejo/upgrades/merged-app.ini b/.forgejo/upgrades/merged-app.ini new file mode 100644 index 0000000000..0b7150c077 --- /dev/null +++ b/.forgejo/upgrades/merged-app.ini @@ -0,0 +1,32 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[storage] +PATH = ${WORK_PATH}/merged diff --git a/.forgejo/upgrades/misplace-app.ini b/.forgejo/upgrades/misplace-app.ini new file mode 100644 index 0000000000..0aeff45319 --- /dev/null +++ b/.forgejo/upgrades/misplace-app.ini @@ -0,0 +1,59 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[attachment] + +[storage.attachments] +PATH = ${WORK_PATH}/data/attachments + +[lfs] + +[storage.lfs] +PATH = ${WORK_PATH}/data/lfs + +[avatar] + +[storage.avatars] +PATH = ${WORK_PATH}/data/avatars + +[repo-avatar] + +[storage.repo-avatars] +PATH = ${WORK_PATH}/data/repo-avatars + +[repo-archive] + +[storage.repo-archive] +PATH = ${WORK_PATH}/data/repo-archive + +[packages] + +[storage.packages] +PATH = ${WORK_PATH}/data/packages diff --git a/.forgejo/upgrades/misplace-s3-app.ini b/.forgejo/upgrades/misplace-s3-app.ini new file mode 100644 index 0000000000..d9243dd250 --- /dev/null +++ b/.forgejo/upgrades/misplace-s3-app.ini @@ -0,0 +1,89 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[attachment] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[lfs] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[repo-avatar] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[avatar] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[repo-archive] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[packages] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false diff --git a/.forgejo/upgrades/relative-app.ini b/.forgejo/upgrades/relative-app.ini new file mode 100644 index 0000000000..d53c291780 --- /dev/null +++ b/.forgejo/upgrades/relative-app.ini @@ -0,0 +1,44 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[attachment] +PATH = relative-attachments + +[lfs] +PATH = relative-lfs + +[avatar] +PATH = relative-avatars + +[repo-avatar] +PATH = relative-repo-avatars + +[repo-archive] +PATH = relative-repo-archive + +[packages] +PATH = relative-packages diff --git a/.forgejo/upgrades/specific-app.ini b/.forgejo/upgrades/specific-app.ini new file mode 100644 index 0000000000..d7a0badc13 --- /dev/null +++ b/.forgejo/upgrades/specific-app.ini @@ -0,0 +1,47 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[attachment] +PATH = ${WORK_PATH}/data/attachments + +[lfs] +PATH = ${WORK_PATH}/data/lfs + +[avatar] +PATH = ${WORK_PATH}/data/avatars + +[repo-avatar] +PATH = ${WORK_PATH}/data/repo-avatars + +[repo-archive] +PATH = ${WORK_PATH}/data/repo-archive + +[packages] +PATH = ${WORK_PATH}/data/packages diff --git a/.forgejo/upgrades/stable-s3-app.ini b/.forgejo/upgrades/stable-s3-app.ini new file mode 100644 index 0000000000..e8c48ae2e1 --- /dev/null +++ b/.forgejo/upgrades/stable-s3-app.ini @@ -0,0 +1,39 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[storage] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false diff --git a/.forgejo/upgrades/storage-relative-app.ini b/.forgejo/upgrades/storage-relative-app.ini new file mode 100644 index 0000000000..eba5232dd0 --- /dev/null +++ b/.forgejo/upgrades/storage-relative-app.ini @@ -0,0 +1,44 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[storage.attachments] +PATH = relative-attachments + +[storage.lfs] +PATH = relative-lfs + +[storage.avatars] +PATH = relative-avatars + +[storage.repo-avatars] +PATH = relative-repo-avatars + +[storage.repo-archive] +PATH = relative-repo-archive + +[storage.packages] +PATH = relative-packages diff --git a/.forgejo/upgrades/test-upgrade.sh b/.forgejo/upgrades/test-upgrade.sh new file mode 100755 index 0000000000..287a92a591 --- /dev/null +++ b/.forgejo/upgrades/test-upgrade.sh @@ -0,0 +1,590 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +# +# Debug loop from the source tree: +# +# ./.forgejo/upgrades/test-upgrade.sh dependencies +# ./.forgejo/upgrades/test-upgrade.sh build_all +# VERBOSE=true ./.forgejo/upgrades/test-upgrade.sh test_downgrade_1.20.2_fails +# +# Everything happens in /tmp/forgejo-upgrades +# + +PREFIX=============== +HOST_PORT=0.0.0.0:3000 +STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" +STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" +DIR=/tmp/forgejo-upgrades +if ${VERBOSE:-false} ; then + set -ex + PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' +else + set -e +fi +SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +: ${FORGEJO_USER:=root} +: ${FORGEJO_REPO:=fixture} +: ${FORGEJO_PASSWORD:=admin1234} + +source $SELF_DIR/fixtures.sh + +function maybe_sudo() { + if test $(id -u) != 0 ; then + SUDO=sudo + fi +} + +function log_info() { + echo "$PREFIX $@" +} + +function dependencies() { + maybe_sudo + if ! which curl daemon jq git-lfs > /dev/null ; then + $SUDO apt-get install -y -qq curl daemon git-lfs jq + fi + + if ! test -f /usr/local/bin/mc || ! test -f /usr/local/bin/minio > /dev/null ; then + $SUDO curl --fail -sS https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + $SUDO curl --fail -sS https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2023-08-23T10-07-06Z -o /usr/local/bin/minio + fi + if ! test -x /usr/local/bin/mc || ! test -x /usr/local/bin/minio > /dev/null ; then + $SUDO chmod +x /usr/local/bin/mc + $SUDO chmod +x /usr/local/bin/minio + fi + + if ! test -f /usr/local/bin/garage > /dev/null ; then + $SUDO curl --fail -sS https://garagehq.deuxfleurs.fr/_releases/v0.8.2/x86_64-unknown-linux-musl/garage -o /usr/local/bin/garage + fi + if ! test -x /usr/local/bin/garage > /dev/null ; then + $SUDO chmod +x /usr/local/bin/garage + fi +} + +function build() { + local version=$1 + local semver=$2 + + if ! test -f $DIR/forgejo-$version ; then + mkdir -p $DIR + make VERSION=v$version GITEA_VERSION=v$version FORGEJO_VERSION=$semver TAGS='bindata sqlite sqlite_unlock_notify' generate gitea + mv gitea $DIR/forgejo-$version + fi +} + +function build_all() { + test -f Makefile + build 1.21.0-0 6.0.0+0-gitea-1.21.0 +} + +function retry() { + rm -f $DIR/wait-for.out + success=false + for delay in 1 1 5 5 15 ; do + if "$@" >> $DIR/wait-for.out 2>&1 ; then + success=true + break + fi + cat $DIR/wait-for.out + echo waiting $delay + sleep $delay + done + if test $success = false ; then + cat $DIR/wait-for.out + return 1 + fi +} + +function download() { + local version=$1 + + if ! test -f $DIR/forgejo-$version ; then + mkdir -p $DIR + wget -O $DIR/forgejo-$version --quiet https://codeberg.org/forgejo/forgejo/releases/download/v$version/forgejo-$version-linux-amd64 + chmod +x $DIR/forgejo-$version + fi +} + +function cleanup_logs() { + local work_path=$DIR/forgejo-work-path + + rm -f $DIR/*.log + rm -f $work_path/log/*.log +} + +function clobber() { + rm -fr /tmp/forgejo-upgrades +} + +function start_forgejo() { + local version=$1 + + download $version + local work_path=$DIR/forgejo-work-path + daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path + if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then + cat $DIR/*.log + cat $work_path/log/*.log + return 1 + fi + create_user $version +} + +function start_minio() { + mkdir -p $DIR/minio + daemon --chdir=$DIR --unsafe \ + --env="PATH=$PATH" \ + --env=MINIO_ROOT_USER=123456 \ + --env=MINIO_ROOT_PASSWORD=12345678 \ + --env=MINIO_VOLUMES=$DIR/minio \ + --pidfile=$DIR/minio-pid --errlog=$DIR/minio-err.log --output=$DIR/minio-out.log -- /usr/local/bin/minio server + retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678 +} + +function start_garage() { + mkdir -p $DIR/garage/{data,meta} + cat > $DIR/garage/garage.toml < $work_path/forgejo-token + ( echo -n 'Authorization: token ' ; cat $work_path/forgejo-token ) > $work_path/forgejo-header + ( echo "#!/bin/sh" ; echo 'curl -f -sS -H "Content-Type: application/json" -H @'$work_path/forgejo-header' "$@"' ) > $work_path/forgejo-api && chmod +x $work_path/forgejo-api + $work_path/forgejo-api http://${HOST_PORT}/api/v1/version + + # + # forgejo-client is to use with web endpoints + # + # + # login and obtain a CSRF, all stored in the cookie file + # + ( echo "#!/bin/sh" ; echo 'curl --cookie-jar '$DIR/cookies' --cookie '$DIR/cookies' -f -sS "$@"' ) > $work_path/forgejo-client-update-cookies && chmod +x $work_path/forgejo-client-update-cookies + $work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null + $work_path/forgejo-client-update-cookies --verbose -X POST --data user_name=${FORGEJO_USER} --data password=${FORGEJO_PASSWORD} http://${HOST_PORT}/user/login >& $DIR/login.html + $work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null + local csrf=$(sed -n -e '/csrf/s/.*csrf\t//p' $DIR/cookies) + # + # use the cookie file but do not modify it + # + ( echo "#!/bin/sh" ; echo 'curl --cookie '$DIR/cookies' -H "X-Csrf-Token: '$csrf'" -f -sS "$@"' ) > $work_path/forgejo-client && chmod +x $work_path/forgejo-client +} + +function stop_daemon() { + local daemon=$1 + + if test -f $DIR/$daemon-pid ; then + local pid=$(cat $DIR/$daemon-pid) + kill -TERM $pid + pidwait $pid || true + for delay in 1 1 2 2 5 5 ; do + if ! test -f $DIR/$daemon-pid ; then + break + fi + sleep $delay + done + ! test -f $DIR/$daemon-pid + fi +} + +function stop() { + stop_daemon forgejo + stop_daemon minio + stop_daemon garage + + cleanup_logs +} + +function reset_forgejo() { + local config=$1 + local work_path=$DIR/forgejo-work-path + rm -fr $work_path + mkdir -p $work_path + WORK_PATH=$work_path envsubst < $SELF_DIR/$config-app.ini > $work_path/app.ini +} + +function reset_minio() { + rm -fr $DIR/minio +} + +function reset_garage() { + rm -fr $DIR/garage +} + +function reset() { + local config=$1 + reset_forgejo $config + reset_minio + reset_garage +} + +function verify_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/$path + done +} + +function cleanup_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + rm -fr $work_path/data/$path + done +} + +function test_downgrade_1.20.2_fails() { + local work_path=$DIR/forgejo-work-path + + log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225" + + log_info "downgrading from 1.20.3-0 to 1.20.2-0 fails" + stop + reset default + start 1.20.3-0 + stop + download 1.20.2-0 + timeout 60 $DIR/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true + if ! grep --fixed-strings --quiet 'use the newer database' $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi +} + +function test_bug_storage_merged() { + local work_path=$DIR/forgejo-work-path + + log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225" + + log_info "using < 1.20.3-0 and [storage].PATH merge all storage" + for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do + stop + reset merged + start $version + for path in ${STORAGE_PATHS} ; do + ! test -d $work_path/data/$path + done + for path in ${STORAGE_PATHS} ; do + ! test -d $work_path/merged/$path + done + test -d $work_path/merged + done + stop + + log_info "upgrading from 1.20.2-0 with [storage].PATH fails" + download 1.20.3-0 + timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + if ! grep --fixed-strings --quiet '[storage].PATH is set and may create storage issues' $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi +} + +function test_bug_storage_relative_path() { + local work_path=$DIR/forgejo-work-path + + log_info "using < 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to WORK_PATH" + for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do + stop + reset legagy-relative + start $version + test -d $work_path/relative-lfs + test -d $work_path/relative-avatars + test -d $work_path/relative-repo-avatars + done + + log_info "using >= 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to APP_DATA_PATH" + for version in 1.20.3-0 1.21.0-0 ; do + stop + reset legagy-relative + start $version + test -d $work_path/data/relative-lfs + test -d $work_path/data/relative-avatars + test -d $work_path/data/relative-repo-avatars + done + + log_info "using >= 1.20.3-0 relative [storage.XXXX].PATHS are relative to APP_DATA_PATH" + for version in 1.20.3-0 1.21.0-0 ; do + stop + reset storage-relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/relative-$path + done + done + + log_info "using 1.20.[12]-0 relative [storage.XXXX].PATHS are inconsistent" + for version in 1.20.2-0 ; do + stop + reset storage-relative + start $version + test -d $work_path/data/packages + test -d $work_path/relative-repo-archive + test -d $work_path/relative-attachments + test -d $work_path/relative-lfs + test -d $work_path/data/avatars + test -d $work_path/data/repo-avatars + done + + log_info "using < 1.20 relative [storage.XXXX].PATHS are inconsistent" + for version in 1.18.5-0 1.19.4-0 ; do + stop + reset storage-relative + start $version + test -d $work_path/relative-packages + test -d $work_path/relative-repo-archive + test -d $work_path/relative-attachments + test -d $work_path/data/lfs + test -d $work_path/data/avatars + test -d $work_path/data/repo-avatars + done + + log_info "using < 1.20.3-0 relative [XXXX].PATHS are relative to WORK_PATH" + for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do + stop + reset relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/relative-$path + done + done + + log_info "using >= 1.20.3-0 relative [XXXX].PATHS are relative to APP_DATA_PATH" + for version in 1.20.3-0 1.21.0-0 ; do + stop + reset relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/relative-$path + done + done + + stop +} + +function test_bug_storage_s3_misplace() { + local work_path=$DIR/forgejo-work-path + local s3_backend=${2:-minio} + + log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338" + + for version in 1.20.2-0 1.20.3-0 ; do + log_info "Forgejo $version & $s3_backend" + stop + reset misplace-s3 + start $version $s3_backend + fixture_create + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_assert_s3 + done + done + + for version in 1.18.5-0 1.19.4-0 ; do + log_info "Forgejo $version & $s3_backend" + stop + reset misplace-s3 + start $version $s3_backend + fixture_create + # + # some storage are in S3 + # + fixture_attachments_assert_s3 + fixture_lfs_assert_s3 + # + # others are in local + # + fixture_repo_archive_assert_local elsewhere/repo-archive + fixture_avatars_assert_local elsewhere/avatars + fixture_packages_assert_local elsewhere/packages + fixture_repo_avatars_assert_local elsewhere/repo-avatars + done +} + +function test_storage_stable_s3() { + local work_path=$DIR/forgejo-work-path + local s3_backend=${1:-minio} + + log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338" + + for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 ; do + log_info "Forgejo $version & $s3_backend" + stop + reset stable-s3 + start $version $s3_backend + fixture_create + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_assert_s3 + done + done +} + +function test_bug_storage_misplace() { + local work_path=$DIR/forgejo-work-path + + log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225" + + log_info "using < 1.20 and conflicting sections misplace storage" + for version in 1.18.5-0 1.19.4-0 ; do + stop + reset misplace + start $version + # + # some storage are where they should be + # + test -d $work_path/data/packages + test -d $work_path/data/repo-archive + test -d $work_path/data/attachments + # + # others are under APP_DATA_PATH + # + test -d $work_path/elsewhere/lfs + test -d $work_path/elsewhere/avatars + test -d $work_path/elsewhere/repo-avatars + done + + log_info "using < 1.20.[12]-0 and conflicting sections ignores [storage.*]" + for version in 1.20.2-0 ; do + stop + reset misplace + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/elsewhere/$path + done + done + + stop + + log_info "upgrading from 1.20.2-0 with conflicting sections fails" + download 1.20.3-0 + timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + for path in ${STORAGE_PATHS} ; do + if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi + done +} + +function test_successful_upgrades() { + for config in default specific ; do + log_info "using $config app.ini" + reset $config + + for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 1.21.0-0 ; do + log_info "run $version" + cleanup_storage + start $version + verify_storage + stop + done + done +} + +function run() { + local fun=$1 + shift + + echo Start running $fun + mkdir -p $DIR + > $DIR/$fun.out + tail --follow $DIR/$fun.out | sed --unbuffered -n -e "/^$PREFIX/s/^$PREFIX //p" & + pid=$! + if ! VERBOSE=true ${BASH_SOURCE[0]} $fun "$@" >& $DIR/$fun.out ; then + kill $pid + cat $DIR/$fun.out + echo Failure running $fun + return 1 + fi + kill $pid + echo Success running $fun +} + +function test_upgrades() { + run stop + run dependencies + run build_all + run test_successful_upgrades + run test_bug_storage_misplace + run test_bug_storage_merged + run test_downgrade_1.20.2_fails + run test_bug_storage_s3_misplace + run test_storage_stable_s3 minio + run test_storage_stable_s3 garage +} + +"$@" diff --git a/.forgejo/workflows/upgrade.yml b/.forgejo/workflows/upgrade.yml new file mode 100644 index 0000000000..e07a349944 --- /dev/null +++ b/.forgejo/workflows/upgrade.yml @@ -0,0 +1,45 @@ +name: upgrade + +on: + pull_request_review: + push: + branches: + - 'forgejo*' + - 'v*/forgejo*' + +jobs: + upgrade: + runs-on: docker + container: + image: codeberg.org/forgejo/test_env:main + steps: + - run: apt-get install -y -qq zstd + + - name: cache S3 binaries + id: S3 + uses: https://code.forgejo.org/actions/cache@v3 + with: + path: | + /usr/local/bin/minio + /usr/local/bin/mc + /usr/local/bin/garage + key: S3 + + - name: skip if S3 cache hit + if: steps.S3.outputs.cache-hit != 'true' + run: echo no hit + + - uses: https://code.forgejo.org/actions/checkout@v3 + - uses: https://code.forgejo.org/actions/setup-go@v4 + with: + go-version: "1.21" + - run: | + git config --add safe.directory '*' + chown -R gitea:gitea . /go + - run: | + su gitea -c 'make deps-backend' + - run: | + script=$(pwd)/.forgejo/upgrades/test-upgrade.sh + $script run dependencies + $script clobber + su gitea -c "$script test_upgrades"