From d6dd6d641b593c54fe1a1041c153111ce81dbc20 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 14 Jun 2023 11:42:38 +0800 Subject: [PATCH] Fix all possible setting error related storages and added some tests (#23911) Follow up #22405 Fix #20703 This PR rewrites storage configuration read sequences with some breaks and tests. It becomes more strict than before and also fixed some inherit problems. - Move storage's MinioConfig struct into setting, so after the configuration loading, the values will be stored into the struct but not still on some section. - All storages configurations should be stored on one section, configuration items cannot be overrided by multiple sections. The prioioty of configuration is `[attachment]` > `[storage.attachments]` | `[storage.customized]` > `[storage]` > `default` - For extra override configuration items, currently are `SERVE_DIRECT`, `MINIO_BASE_PATH`, `MINIO_BUCKET`, which could be configured in another section. The prioioty of the override configuration is `[attachment]` > `[storage.attachments]` > `default`. - Add more tests for storages configurations. - Update the storage documentations. --------- Co-authored-by: wxiaoguang --- cmd/dump.go | 6 +- cmd/migrate_storage.go | 28 +-- cmd/migrate_storage_test.go | 3 +- custom/conf/app.example.ini | 21 ++ .../config-cheat-sheet.en-us.md | 61 ++++- .../config-cheat-sheet.zh-cn.md | 57 ++++- models/migrations/v1_10/v96.go | 2 +- models/migrations/v1_11/v112.go | 2 +- models/migrations/v1_11/v115.go | 8 +- modules/setting/actions.go | 27 ++- modules/setting/actions_test.go | 97 ++++++++ modules/setting/attachment.go | 22 +- modules/setting/attachment_test.go | 133 +++++++++++ modules/setting/config_provider.go | 19 ++ modules/setting/lfs.go | 52 +++-- modules/setting/lfs_test.go | 77 +++++++ modules/setting/packages.go | 26 ++- modules/setting/packages_test.go | 167 ++++++++++++++ modules/setting/picture.go | 30 ++- modules/setting/repository.go | 8 +- modules/setting/repository_archive.go | 25 ++ modules/setting/repository_archive_test.go | 111 +++++++++ modules/setting/setting.go | 34 ++- modules/setting/storage.go | 215 +++++++++++++----- modules/setting/storage_test.go | 190 ++++------------ modules/storage/helper.go | 56 ----- modules/storage/local.go | 20 +- modules/storage/local_test.go | 4 +- modules/storage/minio.go | 30 +-- modules/storage/minio_test.go | 16 +- modules/storage/storage.go | 29 ++- modules/storage/storage_test.go | 4 +- routers/api/v1/repo/file.go | 4 +- routers/install/install.go | 2 +- routers/web/base.go | 4 +- routers/web/repo/attachment.go | 2 +- routers/web/repo/download.go | 2 +- routers/web/repo/repo.go | 2 +- services/lfs/server.go | 2 +- templates/admin/config.tmpl | 2 +- tests/test_utils.go | 4 +- 41 files changed, 1152 insertions(+), 452 deletions(-) create mode 100644 modules/setting/actions_test.go create mode 100644 modules/setting/attachment_test.go create mode 100644 modules/setting/lfs_test.go create mode 100644 modules/setting/repository_archive.go create mode 100644 modules/setting/repository_archive_test.go diff --git a/cmd/dump.go b/cmd/dump.go index 40524f48d4..7dda7fd2b3 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -353,9 +353,9 @@ func runDump(ctx *cli.Context) error { } excludes = append(excludes, setting.RepoRootPath) - excludes = append(excludes, setting.LFS.Path) - excludes = append(excludes, setting.Attachment.Path) - excludes = append(excludes, setting.Packages.Path) + excludes = append(excludes, setting.LFS.Storage.Path) + excludes = append(excludes, setting.Attachment.Storage.Path) + excludes = append(excludes, setting.Packages.Storage.Path) excludes = append(excludes, setting.Log.RootPath) excludes = append(excludes, absFileName) if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil { diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 291e7695b5..511db6cbf7 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -179,7 +179,7 @@ func runMigrateStorage(ctx *cli.Context) error { switch strings.ToLower(ctx.String("storage")) { case "": fallthrough - case string(storage.LocalStorageType): + case string(setting.LocalStorageType): p := ctx.String("path") if p == "" { log.Fatal("Path must be given when storage is loal") @@ -187,22 +187,24 @@ func runMigrateStorage(ctx *cli.Context) error { } dstStorage, err = storage.NewLocalStorage( stdCtx, - storage.LocalStorageConfig{ + &setting.Storage{ Path: p, }) - case string(storage.MinioStorageType): + case string(setting.MinioStorageType): dstStorage, err = storage.NewMinioStorage( stdCtx, - storage.MinioStorageConfig{ - Endpoint: ctx.String("minio-endpoint"), - AccessKeyID: ctx.String("minio-access-key-id"), - SecretAccessKey: ctx.String("minio-secret-access-key"), - Bucket: ctx.String("minio-bucket"), - Location: ctx.String("minio-location"), - BasePath: ctx.String("minio-base-path"), - UseSSL: ctx.Bool("minio-use-ssl"), - InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"), - ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"), + &setting.Storage{ + MinioConfig: setting.MinioStorageConfig{ + Endpoint: ctx.String("minio-endpoint"), + AccessKeyID: ctx.String("minio-access-key-id"), + SecretAccessKey: ctx.String("minio-secret-access-key"), + Bucket: ctx.String("minio-bucket"), + Location: ctx.String("minio-location"), + BasePath: ctx.String("minio-base-path"), + UseSSL: ctx.Bool("minio-use-ssl"), + InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"), + ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"), + }, }) default: return fmt.Errorf("unsupported storage type: %s", ctx.String("storage")) diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go index 7f3de894ba..644e0dc18b 100644 --- a/cmd/migrate_storage_test.go +++ b/cmd/migrate_storage_test.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" packages_service "code.gitea.io/gitea/services/packages" @@ -57,7 +58,7 @@ func TestMigratePackages(t *testing.T) { dstStorage, err := storage.NewLocalStorage( ctx, - storage.LocalStorageConfig{ + &setting.Storage{ Path: p, }) assert.NoError(t, err) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 23dfefa013..f53d9ee089 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2392,6 +2392,10 @@ LEVEL = Info ;; Enable/Disable package registry capabilities ;ENABLED = true ;; +;STORAGE_TYPE = local +;; override the minio base path if storage type is minio +;MINIO_BASE_PATH = packages/ +;; ;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload` ;CHUNKED_UPLOAD_PATH = tmp/package-upload ;; @@ -2452,6 +2456,19 @@ LEVEL = Info ;; storage type ;STORAGE_TYPE = local +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; repo-archive storage will override storage +;; +;[repo-archive] +;STORAGE_TYPE = local +;; +;; Where your lfs files reside, default is data/lfs. +;PATH = data/repo-archive +;; +;; override the minio base path if storage type is minio +;MINIO_BASE_PATH = repo-archive/ + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; settings for repository archives, will override storage setting @@ -2471,6 +2488,9 @@ LEVEL = Info ;; ;; Where your lfs files reside, default is data/lfs. ;PATH = data/lfs +;; +;; override the minio base path if storage type is minio +;MINIO_BASE_PATH = lfs/ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2520,6 +2540,7 @@ LEVEL = Info ; [actions] ;; Enable/Disable actions capabilities ;ENABLED = false +;; ;; Default address to get action plugins, e.g. the default value means downloading from "https://gitea.com/actions/checkout" for "uses: actions/checkout@v3" ;DEFAULT_ACTIONS_URL = https://gitea.com diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index e3612f3759..7b94c7a488 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -1254,8 +1254,9 @@ is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`. ## Storage (`storage`) -Default storage configuration for attachments, lfs, avatars and etc. +Default storage configuration for attachments, lfs, avatars, repo-avatars, repo-archive, packages, actions_log, actions_artifact. +- `STORAGE_TYPE`: **local**: Storage type, `local` for local disk or `minio` for s3 compatible object storage service. - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing. - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio` - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio` @@ -1265,9 +1266,56 @@ Default storage configuration for attachments, lfs, avatars and etc. - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` -And you can also define a customize storage like below: +The recommanded storage configuration for minio like below: ```ini +[storage] +STORAGE_TYPE = minio +; Minio endpoint to connect only available when STORAGE_TYPE is `minio` +MINIO_ENDPOINT = localhost:9000 +; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio` +MINIO_ACCESS_KEY_ID = +; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio` +MINIO_SECRET_ACCESS_KEY = +; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio` +MINIO_BUCKET = gitea +; Minio location to create bucket only available when STORAGE_TYPE is `minio` +MINIO_LOCATION = us-east-1 +; Minio enabled ssl only available when STORAGE_TYPE is `minio` +MINIO_USE_SSL = false +; Minio skip SSL verification available when STORAGE_TYPE is `minio` +MINIO_INSECURE_SKIP_VERIFY = false +SERVE_DIRECT = true +``` + +Defaultly every storage has their default base path like below + +| storage | default base path | +| ----------------- | ------------------ | +| attachments | attachments/ | +| lfs | lfs/ | +| avatars | avatars/ | +| repo-avatars | repo-avatars/ | +| repo-archive | repo-archive/ | +| packages | packages/ | +| actions_log | actions_log/ | +| actions_artifacts | actions_artifacts/ | + +And bucket, basepath or `SERVE_DIRECT` could be special or overrided, if you want to use a different you can: + +```ini +[storage.actions_log] +MINIO_BUCKET = gitea_actions_log +SERVE_DIRECT = true +MINIO_BASE_PATH = my_actions_log/ ; default is actions_log/ if blank +``` + +If you want to customerize a different storage for `lfs` if above default storage defined + +```ini +[lfs] +STORAGE_TYPE = my_minio + [storage.my_minio] STORAGE_TYPE = minio ; Minio endpoint to connect only available when STORAGE_TYPE is `minio` @@ -1286,8 +1334,6 @@ MINIO_USE_SSL = false MINIO_INSECURE_SKIP_VERIFY = false ``` -And used by `[attachment]`, `[lfs]` and etc. as `STORAGE_TYPE`. - ## Repository Archive Storage (`storage.repo-archive`) Configuration for repository archive storage. It will inherit from default `[storage]` or @@ -1306,6 +1352,11 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`. - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` +## Repository Archives (`repo-archive`) + +- `STORAGE_TYPE`: **local**: Storage type for actions logs, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]` +- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio` + ## Proxy (`proxy`) - `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy @@ -1324,6 +1375,8 @@ PROXY_HOSTS = *.github.com - `ENABLED`: **false**: Enable/Disable actions capabilities - `DEFAULT_ACTIONS_URL`: **https://gitea.com**: Default address to get action plugins, e.g. the default value means downloading from "" for "uses: actions/checkout@v3" +- `STORAGE_TYPE`: **local**: Storage type for actions logs, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]` +- `MINIO_BASE_PATH`: **actions_log/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio` `DEFAULT_ACTIONS_URL` indicates where should we find the relative path action plugin. i.e. when use an action in a workflow file like diff --git a/docs/content/doc/administration/config-cheat-sheet.zh-cn.md b/docs/content/doc/administration/config-cheat-sheet.zh-cn.md index 233d252a05..d0af323dc0 100644 --- a/docs/content/doc/administration/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/administration/config-cheat-sheet.zh-cn.md @@ -414,7 +414,7 @@ LFS 的存储配置。 如果 `STORAGE_TYPE` 为空，则此配置将从 `[stora ## Storage (`storage`) -Attachments, lfs, avatars and etc 的默认存储配置。 +Attachments, lfs, avatars, repo-avatars, repo-archive, packages, actions_log, actions_artifact 的默认存储配置。 - `STORAGE_TYPE`: **local**: 附件存储类型，`local` 将存储到本地文件夹， `minio` 将存储到 s3 兼容的对象存储服务中。 - `SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前，仅 Minio/S3 是支持的。 @@ -425,9 +425,59 @@ Attachments, lfs, avatars and etc 的默认存储配置。 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket，仅当 `STORAGE_TYPE` 是 `minio` 时有效。 - `MINIO_USE_SSL`: **false**: Minio enabled ssl，仅当 `STORAGE_TYPE` 是 `minio` 时有效。 -你也可以自定义一个存储的名字如下： +以下为推荐的 recommanded storage configuration for minio like below: ```ini +[storage] +STORAGE_TYPE = minio +; uncomment when STORAGE_TYPE = local +; PATH = storage root path +; Minio endpoint to connect only available when STORAGE_TYPE is `minio` +MINIO_ENDPOINT = localhost:9000 +; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio` +MINIO_ACCESS_KEY_ID = +; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio` +MINIO_SECRET_ACCESS_KEY = +; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio` +MINIO_BUCKET = gitea +; Minio location to create bucket only available when STORAGE_TYPE is `minio` +MINIO_LOCATION = us-east-1 +; Minio enabled ssl only available when STORAGE_TYPE is `minio` +MINIO_USE_SSL = false +; Minio skip SSL verification available when STORAGE_TYPE is `minio` +MINIO_INSECURE_SKIP_VERIFY = false +SERVE_DIRECT = true +``` + +默认的，每一个存储都会有各自默认的 BasePath 在同一个minio中，默认值如下： + +| storage | default base path | +| ----------------- | ------------------ | +| attachments | attachments/ | +| lfs | lfs/ | +| avatars | avatars/ | +| repo-avatars | repo-avatars/ | +| repo-archive | repo-archive/ | +| packages | packages/ | +| actions_log | actions_log/ | +| actions_artifacts | actions_artifacts/ | + +同时 bucket, basepath or `SERVE_DIRECT` 是可以被覆写的，像如下所示： + +```ini +[storage.actions_log] +MINIO_BUCKET = gitea_actions_log +SERVE_DIRECT = true +MINIO_BASE_PATH = my_actions_log/ ; default is actions_log/ if blank +``` + +当然你也可以完全自定义，像如下 + +```ini +[lfs] +STORAGE_TYPE = my_minio +MINIO_BASE_PATH = my_lfs_basepath + [storage.my_minio] STORAGE_TYPE = minio ; Minio endpoint to connect only available when STORAGE_TYPE is `minio` @@ -444,10 +494,9 @@ MINIO_LOCATION = us-east-1 MINIO_USE_SSL = false ; Minio skip SSL verification available when STORAGE_TYPE is `minio` MINIO_INSECURE_SKIP_VERIFY = false +SERVE_DIRECT = true ``` -然后你在 `[attachment]`, `[lfs]` 等中可以把这个名字用作 `STORAGE_TYPE` 的值。 - ## Repository Archive Storage (`storage.repo-archive`) Repository archive 的存储配置。 如果 `STORAGE_TYPE` 为空，则此配置将从 `[storage]` 继承。如果不为 `local` 或者 `minio` 而为 `xxx`， 则从 `[storage.xxx]` 继承。当继承时， `PATH` 默认为 `data/repo-archive`，`MINIO_BASE_PATH` 默认为 `repo-archive/`。 diff --git a/models/migrations/v1_10/v96.go b/models/migrations/v1_10/v96.go index 422defe838..34c8240031 100644 --- a/models/migrations/v1_10/v96.go +++ b/models/migrations/v1_10/v96.go @@ -53,7 +53,7 @@ func DeleteOrphanedAttachments(x *xorm.Engine) error { for _, attachment := range attachments { uuid := attachment.UUID - if err := util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil { + if err := util.RemoveAll(filepath.Join(setting.Attachment.Storage.Path, uuid[0:1], uuid[1:2], uuid)); err != nil { return err } } diff --git a/models/migrations/v1_11/v112.go b/models/migrations/v1_11/v112.go index ff1f972204..0857663119 100644 --- a/models/migrations/v1_11/v112.go +++ b/models/migrations/v1_11/v112.go @@ -30,7 +30,7 @@ func RemoveAttachmentMissedRepo(x *xorm.Engine) error { for i := 0; i < len(attachments); i++ { uuid := attachments[i].UUID - if err = util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil { + if err = util.RemoveAll(filepath.Join(setting.Attachment.Storage.Path, uuid[0:1], uuid[1:2], uuid)); err != nil { fmt.Printf("Error: %v", err) //nolint:forbidigo } } diff --git a/models/migrations/v1_11/v115.go b/models/migrations/v1_11/v115.go index da935f6514..8c631cfd0b 100644 --- a/models/migrations/v1_11/v115.go +++ b/models/migrations/v1_11/v115.go @@ -61,7 +61,7 @@ func RenameExistingUserAvatarName(x *xorm.Engine) error { for _, user := range users { oldAvatar := user.Avatar - if stat, err := os.Stat(filepath.Join(setting.Avatar.Path, oldAvatar)); err != nil || !stat.Mode().IsRegular() { + if stat, err := os.Stat(filepath.Join(setting.Avatar.Storage.Path, oldAvatar)); err != nil || !stat.Mode().IsRegular() { if err == nil { err = fmt.Errorf("Error: \"%s\" is not a regular file", oldAvatar) } @@ -86,7 +86,7 @@ func RenameExistingUserAvatarName(x *xorm.Engine) error { return fmt.Errorf("[user: %s] user table update: %w", user.LowerName, err) } - deleteList.Add(filepath.Join(setting.Avatar.Path, oldAvatar)) + deleteList.Add(filepath.Join(setting.Avatar.Storage.Path, oldAvatar)) migrated++ select { case <-ticker.C: @@ -135,7 +135,7 @@ func RenameExistingUserAvatarName(x *xorm.Engine) error { // copyOldAvatarToNewLocation copies oldAvatar to newAvatarLocation // and returns newAvatar location func copyOldAvatarToNewLocation(userID int64, oldAvatar string) (string, error) { - fr, err := os.Open(filepath.Join(setting.Avatar.Path, oldAvatar)) + fr, err := os.Open(filepath.Join(setting.Avatar.Storage.Path, oldAvatar)) if err != nil { return "", fmt.Errorf("os.Open: %w", err) } @@ -151,7 +151,7 @@ func copyOldAvatarToNewLocation(userID int64, oldAvatar string) (string, error) return newAvatar, nil } - if err := os.WriteFile(filepath.Join(setting.Avatar.Path, newAvatar), data, 0o666); err != nil { + if err := os.WriteFile(filepath.Join(setting.Avatar.Storage.Path, newAvatar), data, 0o666); err != nil { return "", fmt.Errorf("os.WriteFile: %w", err) } diff --git a/modules/setting/actions.go b/modules/setting/actions.go index eb1b637a1c..1c8075cd6c 100644 --- a/modules/setting/actions.go +++ b/modules/setting/actions.go @@ -4,14 +4,14 @@ package setting import ( - "code.gitea.io/gitea/modules/log" + "fmt" ) // Actions settings var ( Actions = struct { - LogStorage Storage // how the created logs should be stored - ArtifactStorage Storage // how the created artifacts should be stored + LogStorage *Storage // how the created logs should be stored + ArtifactStorage *Storage // how the created artifacts should be stored Enabled bool DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"` }{ @@ -20,15 +20,22 @@ var ( } ) -func loadActionsFrom(rootCfg ConfigProvider) { +func loadActionsFrom(rootCfg ConfigProvider) error { sec := rootCfg.Section("actions") - if err := sec.MapTo(&Actions); err != nil { - log.Fatal("Failed to map Actions settings: %v", err) + err := sec.MapTo(&Actions) + if err != nil { + return fmt.Errorf("failed to map Actions settings: %v", err) } - actionsSec := rootCfg.Section("actions.artifacts") - storageType := actionsSec.Key("STORAGE_TYPE").MustString("") + // don't support to read configuration from [actions] + Actions.LogStorage, err = getStorage(rootCfg, "actions_log", "", nil) + if err != nil { + return err + } - Actions.LogStorage = getStorage(rootCfg, "actions_log", "", nil) - Actions.ArtifactStorage = getStorage(rootCfg, "actions_artifacts", storageType, actionsSec) + actionsSec, _ := rootCfg.GetSection("actions.artifacts") + + Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec) + + return err } diff --git a/modules/setting/actions_test.go b/modules/setting/actions_test.go new file mode 100644 index 0000000000..a1cc8fe333 --- /dev/null +++ b/modules/setting/actions_test.go @@ -0,0 +1,97 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getStorageInheritNameSectionTypeForActions(t *testing.T) { + iniStr := ` + [storage] + STORAGE_TYPE = minio + ` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "minio", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + + iniStr = ` +[storage.actions_log] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "minio", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + + iniStr = ` +[storage.actions_log] +STORAGE_TYPE = my_storage + +[storage.my_storage] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "minio", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + + iniStr = ` +[storage.actions_artifacts] +STORAGE_TYPE = my_storage + +[storage.my_storage] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "local", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + + iniStr = ` +[storage.actions_artifacts] +STORAGE_TYPE = my_storage + +[storage.my_storage] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "local", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + + iniStr = `` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "local", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) +} diff --git a/modules/setting/attachment.go b/modules/setting/attachment.go index 4d4b8e3b02..491564c9dc 100644 --- a/modules/setting/attachment.go +++ b/modules/setting/attachment.go @@ -5,29 +5,31 @@ package setting // Attachment settings var Attachment = struct { - Storage + Storage *Storage AllowedTypes string MaxSize int64 MaxFiles int Enabled bool }{ - Storage: Storage{ - ServeDirect: false, - }, - AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip", + Storage: &Storage{}, + AllowedTypes: ".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip", MaxSize: 4, MaxFiles: 5, Enabled: true, } -func loadAttachmentFrom(rootCfg ConfigProvider) { - sec := rootCfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - - Attachment.Storage = getStorage(rootCfg, "attachments", storageType, sec) +func loadAttachmentFrom(rootCfg ConfigProvider) (err error) { + sec, _ := rootCfg.GetSection("attachment") + if sec == nil { + Attachment.Storage, err = getStorage(rootCfg, "attachments", "", nil) + return err + } Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip") Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4) Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5) Attachment.Enabled = sec.Key("ENABLED").MustBool(true) + + Attachment.Storage, err = getStorage(rootCfg, "attachments", "", sec) + return err } diff --git a/modules/setting/attachment_test.go b/modules/setting/attachment_test.go new file mode 100644 index 0000000000..3e8d2da4d9 --- /dev/null +++ b/modules/setting/attachment_test.go @@ -0,0 +1,133 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getStorageCustomType(t *testing.T) { + iniStr := ` +[attachment] +STORAGE_TYPE = my_minio +MINIO_BUCKET = gitea-attachment + +[storage.my_minio] +STORAGE_TYPE = minio +MINIO_ENDPOINT = my_minio:9000 +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + assert.EqualValues(t, "minio", Attachment.Storage.Type) + assert.EqualValues(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint) + assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) +} + +func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) { + iniStr := ` +[attachment] +STORAGE_TYPE = minio + +[storage.minio] +MINIO_BUCKET = gitea-minio + +[storage] +MINIO_BUCKET = gitea +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + assert.EqualValues(t, "minio", Attachment.Storage.Type) + assert.EqualValues(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) +} + +func Test_getStorageSpecificOverridesStorage(t *testing.T) { + iniStr := ` +[attachment] +STORAGE_TYPE = minio +MINIO_BUCKET = gitea-attachment + +[storage.attachments] +MINIO_BUCKET = gitea + +[storage] +STORAGE_TYPE = local +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + assert.EqualValues(t, "minio", Attachment.Storage.Type) + assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) +} + +func Test_getStorageGetDefaults(t *testing.T) { + cfg, err := NewConfigProviderFromData("") + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + // default storage is local, so bucket is empty + assert.EqualValues(t, "", Attachment.Storage.MinioConfig.Bucket) +} + +func Test_getStorageInheritNameSectionType(t *testing.T) { + iniStr := ` +[storage.attachments] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + assert.EqualValues(t, "minio", Attachment.Storage.Type) +} + +func Test_AttachmentStorage(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + storage := Attachment.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) +} + +func Test_AttachmentStorage1(t *testing.T) { + iniStr := ` +[storage] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + assert.EqualValues(t, "minio", Attachment.Storage.Type) + assert.EqualValues(t, "gitea", Attachment.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) +} diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index 8b317d94e3..526d69bbdc 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "time" @@ -95,6 +96,18 @@ func ConfigSectionKeyString(sec ConfigSection, key string, def ...string) string return "" } +func ConfigSectionKeyBool(sec ConfigSection, key string, def ...bool) bool { + k := ConfigSectionKey(sec, key) + if k != nil && k.String() != "" { + b, _ := strconv.ParseBool(k.String()) + return b + } + if len(def) > 0 { + return def[0] + } + return false +} + // ConfigInheritedKey works like ini.Section.Key(), but it always returns a new key instance, it is O(n) because NewKey is O(n) // and the returned key is safe to be used with "MustXxx", it doesn't change the parent's values. // Otherwise, ini.Section.Key().MustXxx would pollute the parent section's keys. @@ -287,6 +300,12 @@ func deprecatedSetting(rootCfg ConfigProvider, oldSection, oldKey, newSection, n } } +func deprecatedSettingFatal(rootCfg ConfigProvider, oldSection, oldKey, newSection, newKey, version string) { + if rootCfg.Section(oldSection).HasKey(oldKey) { + log.Fatal("Deprecated fallback `[%s]` `%s` present. Use `[%s]` `%s` instead. This fallback will be/has been removed in %s", oldSection, oldKey, newSection, newKey, version) + } +} + // deprecatedSettingDB add a hint that the configuration has been moved to database but still kept in app.ini func deprecatedSettingDB(rootCfg ConfigProvider, oldSection, oldKey string) { if rootCfg.Section(oldSection).HasKey(oldKey) { diff --git a/modules/setting/lfs.go b/modules/setting/lfs.go index a5c3631681..d68349be86 100644 --- a/modules/setting/lfs.go +++ b/modules/setting/lfs.go @@ -5,10 +5,10 @@ package setting import ( "encoding/base64" + "fmt" "time" "code.gitea.io/gitea/modules/generate" - "code.gitea.io/gitea/modules/log" ) // LFS represents the configuration for Git LFS @@ -20,25 +20,27 @@ var LFS = struct { MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"` LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"` - Storage + Storage *Storage }{} -func loadLFSFrom(rootCfg ConfigProvider) { +func loadLFSFrom(rootCfg ConfigProvider) error { sec := rootCfg.Section("server") if err := sec.MapTo(&LFS); err != nil { - log.Fatal("Failed to map LFS settings: %v", err) + return fmt.Errorf("failed to map LFS settings: %v", err) } - lfsSec := rootCfg.Section("lfs") - storageType := lfsSec.Key("STORAGE_TYPE").MustString("") + lfsSec, _ := rootCfg.GetSection("lfs") // Specifically default PATH to LFS_CONTENT_PATH // DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version // if these are removed, the warning will not be shown - deprecatedSetting(rootCfg, "server", "LFS_CONTENT_PATH", "lfs", "PATH", "v1.19.0") - lfsSec.Key("PATH").MustString(sec.Key("LFS_CONTENT_PATH").String()) + deprecatedSettingFatal(rootCfg, "server", "LFS_CONTENT_PATH", "lfs", "PATH", "v1.19.0") - LFS.Storage = getStorage(rootCfg, "lfs", storageType, lfsSec) + var err error + LFS.Storage, err = getStorage(rootCfg, "lfs", "", lfsSec) + if err != nil { + return err + } // Rest of LFS service settings if LFS.LocksPagingNum == 0 { @@ -47,23 +49,25 @@ func loadLFSFrom(rootCfg ConfigProvider) { LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(24 * time.Hour) - if LFS.StartServer { - LFS.JWTSecretBytes = make([]byte, 32) - n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) + if !LFS.StartServer { + return nil + } - if err != nil || n != 32 { - LFS.JWTSecretBase64, err = generate.NewJwtSecretBase64() - if err != nil { - log.Fatal("Error generating JWT Secret for custom config: %v", err) - return - } + LFS.JWTSecretBytes = make([]byte, 32) + n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) - // Save secret - sec.Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) - if err := rootCfg.Save(); err != nil { - log.Fatal("Error saving JWT Secret for custom config: %v", err) - return - } + if err != nil || n != 32 { + LFS.JWTSecretBase64, err = generate.NewJwtSecretBase64() + if err != nil { + return fmt.Errorf("Error generating JWT Secret for custom config: %v", err) + } + + // Save secret + sec.Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) + if err := rootCfg.Save(); err != nil { + return fmt.Errorf("Error saving JWT Secret for custom config: %v", err) } } + + return nil } diff --git a/modules/setting/lfs_test.go b/modules/setting/lfs_test.go new file mode 100644 index 0000000000..3313cae0eb --- /dev/null +++ b/modules/setting/lfs_test.go @@ -0,0 +1,77 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getStorageInheritNameSectionTypeForLFS(t *testing.T) { + iniStr := ` + [storage] + STORAGE_TYPE = minio + ` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + + iniStr = ` +[storage.lfs] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + + iniStr = ` +[lfs] +STORAGE_TYPE = my_minio + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + + iniStr = ` +[lfs] +STORAGE_TYPE = my_minio +MINIO_BASE_PATH = my_lfs/ + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath) +} + +func Test_LFSStorage1(t *testing.T) { + iniStr := ` +[storage] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadLFSFrom(cfg)) + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "gitea", LFS.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 5e64d7fe9f..dc8d98d29f 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -4,20 +4,19 @@ package setting import ( + "fmt" "math" "net/url" "os" "path/filepath" - "code.gitea.io/gitea/modules/log" - "github.com/dustin/go-humanize" ) // Package registry settings var ( Packages = struct { - Storage + Storage *Storage Enabled bool ChunkedUploadPath string RegistryHost string @@ -51,13 +50,21 @@ var ( } ) -func loadPackagesFrom(rootCfg ConfigProvider) { - sec := rootCfg.Section("packages") - if err := sec.MapTo(&Packages); err != nil { - log.Fatal("Failed to map Packages settings: %v", err) +func loadPackagesFrom(rootCfg ConfigProvider) (err error) { + sec, _ := rootCfg.GetSection("packages") + if sec == nil { + Packages.Storage, err = getStorage(rootCfg, "packages", "", nil) + return err } - Packages.Storage = getStorage(rootCfg, "packages", "", nil) + if err = sec.MapTo(&Packages); err != nil { + return fmt.Errorf("failed to map Packages settings: %v", err) + } + + Packages.Storage, err = getStorage(rootCfg, "packages", "", sec) + if err != nil { + return err + } appURL, _ := url.Parse(AppURL) Packages.RegistryHost = appURL.Host @@ -68,7 +75,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) { } if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { - log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) + return fmt.Errorf("unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) } Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") @@ -93,6 +100,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) { Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS") Packages.LimitSizeSwift = mustBytes(sec, "LIMIT_SIZE_SWIFT") Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT") + return nil } func mustBytes(section ConfigSection, key string) int64 { diff --git a/modules/setting/packages_test.go b/modules/setting/packages_test.go index d9f6e10528..87de276041 100644 --- a/modules/setting/packages_test.go +++ b/modules/setting/packages_test.go @@ -29,3 +29,170 @@ func TestMustBytes(t *testing.T) { assert.EqualValues(t, 1782579, test("1.7mib")) assert.EqualValues(t, -1, test("1 yib")) // too large } + +func Test_getStorageInheritNameSectionTypeForPackages(t *testing.T) { + // packages storage inherits from storage if nothing configured + iniStr := ` +[storage] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadPackagesFrom(cfg)) + + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + + // we can also configure packages storage directly + iniStr = ` +[storage.packages] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadPackagesFrom(cfg)) + + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + + // or we can indicate the storage type in the packages section + iniStr = ` +[packages] +STORAGE_TYPE = my_minio + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadPackagesFrom(cfg)) + + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + + // or we can indicate the storage type and minio base path in the packages section + iniStr = ` +[packages] +STORAGE_TYPE = my_minio +MINIO_BASE_PATH = my_packages/ + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadPackagesFrom(cfg)) + + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "my_packages/", Packages.Storage.MinioConfig.BasePath) +} + +func Test_PackageStorage1(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[packages] +MINIO_BASE_PATH = packages/ +SERVE_DIRECT = true +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadPackagesFrom(cfg)) + storage := Packages.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.True(t, storage.MinioConfig.ServeDirect) +} + +func Test_PackageStorage2(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage.packages] +MINIO_BASE_PATH = packages/ +SERVE_DIRECT = true +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadPackagesFrom(cfg)) + storage := Packages.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.True(t, storage.MinioConfig.ServeDirect) +} + +func Test_PackageStorage3(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[packages] +STORAGE_TYPE = my_cfg +MINIO_BASE_PATH = my_packages/ +SERVE_DIRECT = true +[storage.my_cfg] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadPackagesFrom(cfg)) + storage := Packages.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.True(t, storage.MinioConfig.ServeDirect) +} + +func Test_PackageStorage4(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage.packages] +STORAGE_TYPE = my_cfg +MINIO_BASE_PATH = my_packages/ +SERVE_DIRECT = true +[storage.my_cfg] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadPackagesFrom(cfg)) + storage := Packages.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.True(t, storage.MinioConfig.ServeDirect) +} diff --git a/modules/setting/picture.go b/modules/setting/picture.go index 64d9a608e6..fafae45bab 100644 --- a/modules/setting/picture.go +++ b/modules/setting/picture.go @@ -7,7 +7,7 @@ package setting var ( Avatar = struct { - Storage + Storage *Storage MaxWidth int MaxHeight int @@ -27,23 +27,26 @@ var ( EnableFederatedAvatar bool // Depreciated: migrated to database RepoAvatar = struct { - Storage + Storage *Storage Fallback string FallbackImage string }{} ) -func loadPictureFrom(rootCfg ConfigProvider) { +func loadAvatarsFrom(rootCfg ConfigProvider) error { sec := rootCfg.Section("picture") avatarSec := rootCfg.Section("avatar") storageType := sec.Key("AVATAR_STORAGE_TYPE").MustString("") // Specifically default PATH to AVATAR_UPLOAD_PATH - avatarSec.Key("PATH").MustString( - sec.Key("AVATAR_UPLOAD_PATH").String()) + avatarSec.Key("PATH").MustString(sec.Key("AVATAR_UPLOAD_PATH").String()) - Avatar.Storage = getStorage(rootCfg, "avatars", storageType, avatarSec) + var err error + Avatar.Storage, err = getStorage(rootCfg, "avatars", storageType, avatarSec) + if err != nil { + return err + } Avatar.MaxWidth = sec.Key("AVATAR_MAX_WIDTH").MustInt(4096) Avatar.MaxHeight = sec.Key("AVATAR_MAX_HEIGHT").MustInt(4096) @@ -67,7 +70,7 @@ func loadPictureFrom(rootCfg ConfigProvider) { EnableFederatedAvatar = sec.Key("ENABLE_FEDERATED_AVATAR").MustBool(GetDefaultEnableFederatedAvatar(DisableGravatar)) deprecatedSettingDB(rootCfg, "", "ENABLE_FEDERATED_AVATAR") - loadRepoAvatarFrom(rootCfg) + return nil } func GetDefaultDisableGravatar() bool { @@ -85,17 +88,22 @@ func GetDefaultEnableFederatedAvatar(disableGravatar bool) bool { return v } -func loadRepoAvatarFrom(rootCfg ConfigProvider) { +func loadRepoAvatarFrom(rootCfg ConfigProvider) error { sec := rootCfg.Section("picture") repoAvatarSec := rootCfg.Section("repo-avatar") storageType := sec.Key("REPOSITORY_AVATAR_STORAGE_TYPE").MustString("") // Specifically default PATH to AVATAR_UPLOAD_PATH - repoAvatarSec.Key("PATH").MustString( - sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String()) + repoAvatarSec.Key("PATH").MustString(sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String()) - RepoAvatar.Storage = getStorage(rootCfg, "repo-avatars", storageType, repoAvatarSec) + var err error + RepoAvatar.Storage, err = getStorage(rootCfg, "repo-avatars", storageType, repoAvatarSec) + if err != nil { + return err + } RepoAvatar.Fallback = sec.Key("REPOSITORY_AVATAR_FALLBACK").MustString("none") RepoAvatar.FallbackImage = sec.Key("REPOSITORY_AVATAR_FALLBACK_IMAGE").MustString(AppSubURL + "/assets/img/repo_default.png") + + return nil } diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 406068b59d..42ffb99138 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -265,10 +265,6 @@ var ( } RepoRootPath string ScriptType = "bash" - - RepoArchive = struct { - Storage - }{} ) func loadRepositoryFrom(rootCfg ConfigProvider) { @@ -359,5 +355,7 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath) } - RepoArchive.Storage = getStorage(rootCfg, "repo-archive", "", nil) + if err := loadRepoArchiveFrom(rootCfg); err != nil { + log.Fatal("loadRepoArchiveFrom: %v", err) + } } diff --git a/modules/setting/repository_archive.go b/modules/setting/repository_archive.go new file mode 100644 index 0000000000..9d24afa9e7 --- /dev/null +++ b/modules/setting/repository_archive.go @@ -0,0 +1,25 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import "fmt" + +var RepoArchive = struct { + Storage *Storage +}{} + +func loadRepoArchiveFrom(rootCfg ConfigProvider) (err error) { + sec, _ := rootCfg.GetSection("repo-archive") + if sec == nil { + RepoArchive.Storage, err = getStorage(rootCfg, "repo-archive", "", nil) + return err + } + + if err := sec.MapTo(&RepoArchive); err != nil { + return fmt.Errorf("mapto repoarchive failed: %v", err) + } + + RepoArchive.Storage, err = getStorage(rootCfg, "repo-archive", "", sec) + return err +} diff --git a/modules/setting/repository_archive_test.go b/modules/setting/repository_archive_test.go new file mode 100644 index 0000000000..a0f91f0da1 --- /dev/null +++ b/modules/setting/repository_archive_test.go @@ -0,0 +1,111 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getStorageInheritNameSectionTypeForRepoArchive(t *testing.T) { + // packages storage inherits from storage if nothing configured + iniStr := ` +[storage] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + + // we can also configure packages storage directly + iniStr = ` +[storage.repo-archive] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + + // or we can indicate the storage type in the packages section + iniStr = ` +[repo-archive] +STORAGE_TYPE = my_minio + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + + // or we can indicate the storage type and minio base path in the packages section + iniStr = ` +[repo-archive] +STORAGE_TYPE = my_minio +MINIO_BASE_PATH = my_archive/ + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath) +} + +func Test_RepoArchiveStorage(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadRepoArchiveFrom(cfg)) + storage := RepoArchive.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + + iniStr = ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage.repo-archive] +STORAGE_TYPE = s3 +[storage.s3] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadRepoArchiveFrom(cfg)) + storage = RepoArchive.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 1967d9e79a..293333a95b 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -203,16 +203,18 @@ func Init(opts *Options) { var err error CfgProvider, err = NewConfigProviderFromFile(opts) if err != nil { - log.Fatal("Init[%v]: %v", opts, err) + log.Fatal("newConfigProviderFromFile[%v]: %v", opts, err) } if !opts.DisableLoadCommonSettings { - loadCommonSettingsFrom(CfgProvider) + if err := loadCommonSettingsFrom(CfgProvider); err != nil { + log.Fatal("loadCommonSettingsFrom[%v]: %v", opts, err) + } } } // loadCommonSettingsFrom loads common configurations from a configuration provider. -func loadCommonSettingsFrom(cfg ConfigProvider) { - // WARNING: don't change the sequence except you know what you are doing. +func loadCommonSettingsFrom(cfg ConfigProvider) error { + // WARNNING: don't change the sequence except you know what you are doing. loadRunModeFrom(cfg) loadLogGlobalFrom(cfg) loadServerFrom(cfg) @@ -222,13 +224,26 @@ func loadCommonSettingsFrom(cfg ConfigProvider) { loadOAuth2From(cfg) loadSecurityFrom(cfg) - loadAttachmentFrom(cfg) - loadLFSFrom(cfg) + if err := loadAttachmentFrom(cfg); err != nil { + return err + } + if err := loadLFSFrom(cfg); err != nil { + return err + } loadTimeFrom(cfg) loadRepositoryFrom(cfg) - loadPictureFrom(cfg) - loadPackagesFrom(cfg) - loadActionsFrom(cfg) + if err := loadAvatarsFrom(cfg); err != nil { + return err + } + if err := loadRepoAvatarFrom(cfg); err != nil { + return err + } + if err := loadPackagesFrom(cfg); err != nil { + return err + } + if err := loadActionsFrom(cfg); err != nil { + return err + } loadUIFrom(cfg) loadAdminFrom(cfg) loadAPIFrom(cfg) @@ -239,6 +254,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) { loadMirrorFrom(cfg) loadMarkupFrom(cfg) loadOtherFrom(cfg) + return nil } func loadRunModeFrom(rootCfg ConfigProvider) { diff --git a/modules/setting/storage.go b/modules/setting/storage.go index 6da52807ec..ed804a910a 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -4,87 +4,182 @@ package setting import ( + "errors" + "fmt" "path/filepath" - "reflect" ) +// StorageType is a type of Storage +type StorageType string + +const ( + // LocalStorageType is the type descriptor for local storage + LocalStorageType StorageType = "local" + // MinioStorageType is the type descriptor for minio storage + MinioStorageType StorageType = "minio" +) + +var storageTypes = []StorageType{ + LocalStorageType, + MinioStorageType, +} + +// IsValidStorageType returns true if the given storage type is valid +func IsValidStorageType(storageType StorageType) bool { + for _, t := range storageTypes { + if t == storageType { + return true + } + } + return false +} + +// MinioStorageConfig represents the configuration for a minio storage +type MinioStorageConfig struct { + Endpoint string `ini:"MINIO_ENDPOINT" json:",omitempty"` + AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID" json:",omitempty"` + SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY" json:",omitempty"` + Bucket string `ini:"MINIO_BUCKET" json:",omitempty"` + Location string `ini:"MINIO_LOCATION" json:",omitempty"` + BasePath string `ini:"MINIO_BASE_PATH" json:",omitempty"` + UseSSL bool `ini:"MINIO_USE_SSL"` + InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"` + ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM" json:",omitempty"` + ServeDirect bool `ini:"SERVE_DIRECT"` +} + // Storage represents configuration of storages type Storage struct { - Type string - Path string - Section ConfigSection - ServeDirect bool + Type StorageType // local or minio + Path string `json:",omitempty"` // for local type + TemporaryPath string `json:",omitempty"` + MinioConfig MinioStorageConfig // for minio type } -// MapTo implements the Mappable interface -func (s *Storage) MapTo(v interface{}) error { - pathValue := reflect.ValueOf(v).Elem().FieldByName("Path") - if pathValue.IsValid() && pathValue.Kind() == reflect.String { - pathValue.SetString(s.Path) +func (storage *Storage) ToShadowCopy() Storage { + shadowStorage := *storage + if shadowStorage.MinioConfig.AccessKeyID != "" { + shadowStorage.MinioConfig.AccessKeyID = "******" } - if s.Section != nil { - return s.Section.MapTo(v) + if shadowStorage.MinioConfig.SecretAccessKey != "" { + shadowStorage.MinioConfig.SecretAccessKey = "******" } - return nil + return shadowStorage } -func getStorage(rootCfg ConfigProvider, name, typ string, targetSec ConfigSection) Storage { - const sectionName = "storage" - sec := rootCfg.Section(sectionName) +const storageSectionName = "storage" +func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection { + storageSec := rootCfg.Section(storageSectionName) // Global Defaults - sec.Key("MINIO_ENDPOINT").MustString("localhost:9000") - sec.Key("MINIO_ACCESS_KEY_ID").MustString("") - sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") - sec.Key("MINIO_BUCKET").MustString("gitea") - sec.Key("MINIO_LOCATION").MustString("us-east-1") - sec.Key("MINIO_USE_SSL").MustBool(false) - sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false) - sec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default") + storageSec.Key("STORAGE_TYPE").MustString("local") + storageSec.Key("MINIO_ENDPOINT").MustString("localhost:9000") + storageSec.Key("MINIO_ACCESS_KEY_ID").MustString("") + storageSec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") + storageSec.Key("MINIO_BUCKET").MustString("gitea") + storageSec.Key("MINIO_LOCATION").MustString("us-east-1") + storageSec.Key("MINIO_USE_SSL").MustBool(false) + storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false) + storageSec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default") + return storageSec +} + +func getStorage(rootCfg ConfigProvider, name, typ string, sec ConfigSection) (*Storage, error) { + if name == "" { + return nil, errors.New("no name for storage") + } + + var targetSec ConfigSection + if typ != "" { + var err error + targetSec, err = rootCfg.GetSection(storageSectionName + "." + typ) + if err != nil { + if !IsValidStorageType(StorageType(typ)) { + return nil, fmt.Errorf("get section via storage type %q failed: %v", typ, err) + } + } + if targetSec != nil { + targetType := targetSec.Key("STORAGE_TYPE").String() + if targetType == "" { + if !IsValidStorageType(StorageType(typ)) { + return nil, fmt.Errorf("unknow storage type %q", typ) + } + targetSec.Key("STORAGE_TYPE").SetValue(typ) + } else if !IsValidStorageType(StorageType(targetType)) { + return nil, fmt.Errorf("unknow storage type %q for section storage.%v", targetType, typ) + } + } + } + + storageNameSec, _ := rootCfg.GetSection(storageSectionName + "." + name) if targetSec == nil { - targetSec, _ = rootCfg.NewSection(name) + targetSec = sec + } + if targetSec == nil { + targetSec = storageNameSec + } + if targetSec == nil { + targetSec = getDefaultStorageSection(rootCfg) + } else { + targetType := targetSec.Key("STORAGE_TYPE").String() + switch { + case targetType == "": + if targetSec.Key("PATH").String() == "" { + targetSec = getDefaultStorageSection(rootCfg) + } else { + targetSec.Key("STORAGE_TYPE").SetValue("local") + } + default: + newTargetSec, _ := rootCfg.GetSection(storageSectionName + "." + targetType) + if newTargetSec == nil { + if !IsValidStorageType(StorageType(targetType)) { + return nil, fmt.Errorf("invalid storage section %s.%q", storageSectionName, targetType) + } + } else { + targetSec = newTargetSec + if IsValidStorageType(StorageType(targetType)) { + tp := targetSec.Key("STORAGE_TYPE").String() + if tp == "" { + targetSec.Key("STORAGE_TYPE").SetValue(targetType) + } + } + } + } + } + + targetType := targetSec.Key("STORAGE_TYPE").String() + if !IsValidStorageType(StorageType(targetType)) { + return nil, fmt.Errorf("invalid storage type %q", targetType) } var storage Storage - storage.Section = targetSec - storage.Type = typ + storage.Type = StorageType(targetType) - overrides := make([]ConfigSection, 0, 3) - nameSec, err := rootCfg.GetSection(sectionName + "." + name) - if err == nil { - overrides = append(overrides, nameSec) - } + switch targetType { + case string(LocalStorageType): + storage.Path = ConfigSectionKeyString(targetSec, "PATH", filepath.Join(AppDataPath, name)) + if !filepath.IsAbs(storage.Path) { + storage.Path = filepath.Join(AppWorkPath, storage.Path) + } + case string(MinioStorageType): + storage.MinioConfig.BasePath = name + "/" - typeSec, err := rootCfg.GetSection(sectionName + "." + typ) - if err == nil { - overrides = append(overrides, typeSec) - nextType := typeSec.Key("STORAGE_TYPE").String() - if len(nextType) > 0 { - storage.Type = nextType // Support custom STORAGE_TYPE + if err := targetSec.MapTo(&storage.MinioConfig); err != nil { + return nil, fmt.Errorf("map minio config failed: %v", err) + } + // extra config section will be read SERVE_DIRECT, PATH, MINIO_BASE_PATH to override the targetsec + extraConfigSec := sec + if extraConfigSec == nil { + extraConfigSec = storageNameSec + } + + if extraConfigSec != nil { + storage.MinioConfig.ServeDirect = ConfigSectionKeyBool(extraConfigSec, "SERVE_DIRECT", storage.MinioConfig.ServeDirect) + storage.MinioConfig.BasePath = ConfigSectionKeyString(extraConfigSec, "MINIO_BASE_PATH", storage.MinioConfig.BasePath) + storage.MinioConfig.Bucket = ConfigSectionKeyString(extraConfigSec, "MINIO_BUCKET", storage.MinioConfig.Bucket) } } - overrides = append(overrides, sec) - for _, override := range overrides { - for _, key := range override.Keys() { - if !targetSec.HasKey(key.Name()) { - _, _ = targetSec.NewKey(key.Name(), key.Value()) - } - } - if len(storage.Type) == 0 { - storage.Type = override.Key("STORAGE_TYPE").String() - } - } - storage.ServeDirect = storage.Section.Key("SERVE_DIRECT").MustBool(false) - - // Specific defaults - storage.Path = storage.Section.Key("PATH").MustString(filepath.Join(AppDataPath, name)) - if !filepath.IsAbs(storage.Path) { - storage.Path = filepath.Join(AppWorkPath, storage.Path) - storage.Section.Key("PATH").SetValue(storage.Path) - } - storage.Section.Key("MINIO_BASE_PATH").MustString(name + "/") - - return storage + return &storage, nil } diff --git a/modules/setting/storage_test.go b/modules/setting/storage_test.go index 5e213606e3..4eda7646e8 100644 --- a/modules/setting/storage_test.go +++ b/modules/setting/storage_test.go @@ -9,106 +9,6 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_getStorageCustomType(t *testing.T) { - iniStr := ` -[attachment] -STORAGE_TYPE = my_minio -MINIO_BUCKET = gitea-attachment - -[storage.my_minio] -STORAGE_TYPE = minio -MINIO_ENDPOINT = my_minio:9000 -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "my_minio:9000", storage.Section.Key("MINIO_ENDPOINT").String()) - assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String()) -} - -func Test_getStorageNameSectionOverridesTypeSection(t *testing.T) { - iniStr := ` -[attachment] -STORAGE_TYPE = minio - -[storage.attachments] -MINIO_BUCKET = gitea-attachment - -[storage.minio] -MINIO_BUCKET = gitea -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String()) -} - -func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) { - iniStr := ` -[attachment] -STORAGE_TYPE = minio - -[storage.minio] -MINIO_BUCKET = gitea-minio - -[storage] -MINIO_BUCKET = gitea -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea-minio", storage.Section.Key("MINIO_BUCKET").String()) -} - -func Test_getStorageSpecificOverridesStorage(t *testing.T) { - iniStr := ` -[attachment] -STORAGE_TYPE = minio -MINIO_BUCKET = gitea-attachment - -[storage.attachments] -MINIO_BUCKET = gitea - -[storage] -STORAGE_TYPE = local -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String()) -} - -func Test_getStorageGetDefaults(t *testing.T) { - cfg, err := NewConfigProviderFromData("") - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "gitea", storage.Section.Key("MINIO_BUCKET").String()) -} - func Test_getStorageMultipleName(t *testing.T) { iniStr := ` [lfs] @@ -118,32 +18,20 @@ MINIO_BUCKET = gitea-lfs MINIO_BUCKET = gitea-attachment [storage] +STORAGE_TYPE = minio MINIO_BUCKET = gitea-storage ` cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) - { - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) + assert.NoError(t, loadAttachmentFrom(cfg)) + assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String()) - } - { - sec := cfg.Section("lfs") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "lfs", storageType, sec) + assert.NoError(t, loadLFSFrom(cfg)) + assert.EqualValues(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "gitea-lfs", storage.Section.Key("MINIO_BUCKET").String()) - } - { - sec := cfg.Section("avatar") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "avatars", storageType, sec) - - assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String()) - } + assert.NoError(t, loadAvatarsFrom(cfg)) + assert.EqualValues(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket) } func Test_getStorageUseOtherNameAsType(t *testing.T) { @@ -152,25 +40,17 @@ func Test_getStorageUseOtherNameAsType(t *testing.T) { STORAGE_TYPE = lfs [storage.lfs] +STORAGE_TYPE = minio MINIO_BUCKET = gitea-storage ` cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) - { - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) + assert.NoError(t, loadAttachmentFrom(cfg)) + assert.EqualValues(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String()) - } - { - sec := cfg.Section("lfs") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "lfs", storageType, sec) - - assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String()) - } + assert.NoError(t, loadLFSFrom(cfg)) + assert.EqualValues(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket) } func Test_getStorageInheritStorageType(t *testing.T) { @@ -181,24 +61,32 @@ STORAGE_TYPE = minio cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) + assert.NoError(t, loadPackagesFrom(cfg)) + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "gitea", Packages.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) - assert.EqualValues(t, "minio", storage.Type) -} - -func Test_getStorageInheritNameSectionType(t *testing.T) { - iniStr := ` -[storage.attachments] -STORAGE_TYPE = minio -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + + assert.NoError(t, loadActionsFrom(cfg)) + assert.EqualValues(t, "minio", Actions.LogStorage.Type) + assert.EqualValues(t, "gitea", Actions.LogStorage.MinioConfig.Bucket) + assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + + assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket) + assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + + assert.NoError(t, loadAvatarsFrom(cfg)) + assert.EqualValues(t, "minio", Avatar.Storage.Type) + assert.EqualValues(t, "gitea", Avatar.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) + + assert.NoError(t, loadRepoAvatarFrom(cfg)) + assert.EqualValues(t, "minio", RepoAvatar.Storage.Type) + assert.EqualValues(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath) } diff --git a/modules/storage/helper.go b/modules/storage/helper.go index d1959830b9..f8dff9e64d 100644 --- a/modules/storage/helper.go +++ b/modules/storage/helper.go @@ -8,64 +8,8 @@ import ( "io" "net/url" "os" - "reflect" - - "code.gitea.io/gitea/modules/json" ) -// Mappable represents an interface that can MapTo another interface -type Mappable interface { - MapTo(v interface{}) error -} - -// toConfig will attempt to convert a given configuration cfg into the provided exemplar type. -// -// It will tolerate the cfg being passed as a []byte or string of a json representation of the -// exemplar or the correct type of the exemplar itself -func toConfig(exemplar, cfg interface{}) (interface{}, error) { - // First of all check if we've got the same type as the exemplar - if so it's all fine. - if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) { - return cfg, nil - } - - // Now if not - does it provide a MapTo function we can try? - if mappable, ok := cfg.(Mappable); ok { - newVal := reflect.New(reflect.TypeOf(exemplar)) - if err := mappable.MapTo(newVal.Interface()); err == nil { - return newVal.Elem().Interface(), nil - } - // MapTo has failed us ... let's try the json route ... - } - - // OK we've been passed a byte array right? - configBytes, ok := cfg.([]byte) - if !ok { - // oh ... it's a string then? - var configStr string - - configStr, ok = cfg.(string) - configBytes = []byte(configStr) - } - if !ok { - // hmm ... can we marshal it to json? - var err error - configBytes, err = json.Marshal(cfg) - ok = err == nil - } - if !ok { - // no ... we've tried hard enough at this point - throw an error! - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - // OK unmarshal the byte array into a new copy of the exemplar - newVal := reflect.New(reflect.TypeOf(exemplar)) - if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil { - // If we can't unmarshal it then return an error! - return nil, ErrInvalidConfiguration{cfg: cfg, err: err} - } - return newVal.Elem().Interface(), nil -} - var uninitializedStorage = discardStorage("uninitialized storage") type discardStorage string diff --git a/modules/storage/local.go b/modules/storage/local.go index 73ef306979..9bb532f1df 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -12,20 +12,12 @@ import ( "path/filepath" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) var _ ObjectStorage = &LocalStorage{} -// LocalStorageType is the type descriptor for local storage -const LocalStorageType Type = "local" - -// LocalStorageConfig represents the configuration for a local storage -type LocalStorageConfig struct { - Path string `ini:"PATH"` - TemporaryPath string `ini:"TEMPORARY_PATH"` -} - // LocalStorage represents a local files storage type LocalStorage struct { ctx context.Context @@ -34,13 +26,7 @@ type LocalStorage struct { } // NewLocalStorage returns a local files -func NewLocalStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) { - configInterface, err := toConfig(LocalStorageConfig{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(LocalStorageConfig) - +func NewLocalStorage(ctx context.Context, config *setting.Storage) (ObjectStorage, error) { if !filepath.IsAbs(config.Path) { return nil, fmt.Errorf("LocalStorageConfig.Path should have been prepared by setting/storage.go and should be an absolute path, but not: %q", config.Path) } @@ -164,5 +150,5 @@ func (l *LocalStorage) IterateObjects(dirName string, fn func(path string, obj O } func init() { - RegisterStorageType(LocalStorageType, NewLocalStorage) + RegisterStorageType(setting.LocalStorageType, NewLocalStorage) } diff --git a/modules/storage/local_test.go b/modules/storage/local_test.go index 1c4b856ab6..e230323f67 100644 --- a/modules/storage/local_test.go +++ b/modules/storage/local_test.go @@ -8,6 +8,8 @@ import ( "path/filepath" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) @@ -55,5 +57,5 @@ func TestBuildLocalPath(t *testing.T) { func TestLocalStorageIterator(t *testing.T) { dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir") - testStorageIterator(t, string(LocalStorageType), LocalStorageConfig{Path: dir}) + testStorageIterator(t, setting.LocalStorageType, &setting.Storage{Path: dir}) } diff --git a/modules/storage/minio.go b/modules/storage/minio.go index c78f351e9c..81774fb9cf 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -16,6 +16,7 @@ import ( "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "github.com/minio/minio-go/v7" @@ -41,25 +42,9 @@ func (m *minioObject) Stat() (os.FileInfo, error) { return &minioFileInfo{oi}, nil } -// MinioStorageType is the type descriptor for minio storage -const MinioStorageType Type = "minio" - -// MinioStorageConfig represents the configuration for a minio storage -type MinioStorageConfig struct { - Endpoint string `ini:"MINIO_ENDPOINT"` - AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"` - SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"` - Bucket string `ini:"MINIO_BUCKET"` - Location string `ini:"MINIO_LOCATION"` - BasePath string `ini:"MINIO_BASE_PATH"` - UseSSL bool `ini:"MINIO_USE_SSL"` - InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"` - ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM"` -} - // MinioStorage returns a minio bucket storage type MinioStorage struct { - cfg *MinioStorageConfig + cfg *setting.MinioStorageConfig ctx context.Context client *minio.Client bucket string @@ -87,13 +72,8 @@ func convertMinioErr(err error) error { } // NewMinioStorage returns a minio storage -func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) { - configInterface, err := toConfig(MinioStorageConfig{}, cfg) - if err != nil { - return nil, convertMinioErr(err) - } - config := configInterface.(MinioStorageConfig) - +func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error) { + config := cfg.MinioConfig if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" { return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm) } @@ -258,5 +238,5 @@ func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj O } func init() { - RegisterStorageType(MinioStorageType, NewMinioStorage) + RegisterStorageType(setting.MinioStorageType, NewMinioStorage) } diff --git a/modules/storage/minio_test.go b/modules/storage/minio_test.go index bfddf33032..af392b7e22 100644 --- a/modules/storage/minio_test.go +++ b/modules/storage/minio_test.go @@ -6,6 +6,8 @@ package storage import ( "os" "testing" + + "code.gitea.io/gitea/modules/setting" ) func TestMinioStorageIterator(t *testing.T) { @@ -13,11 +15,13 @@ func TestMinioStorageIterator(t *testing.T) { t.Skip("minioStorage not present outside of CI") return } - testStorageIterator(t, string(MinioStorageType), MinioStorageConfig{ - Endpoint: "127.0.0.1:9000", - AccessKeyID: "123456", - SecretAccessKey: "12345678", - Bucket: "gitea", - Location: "us-east-1", + testStorageIterator(t, setting.MinioStorageType, &setting.Storage{ + MinioConfig: setting.MinioStorageConfig{ + Endpoint: "127.0.0.1:9000", + AccessKeyID: "123456", + SecretAccessKey: "12345678", + Bucket: "gitea", + Location: "us-east-1", + }, }) } diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 5b6efccb6a..c3396f0c7f 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -37,16 +37,15 @@ func IsErrInvalidConfiguration(err error) bool { return ok } -// Type is a type of Storage -type Type string +type Type = setting.StorageType // NewStorageFunc is a function that creates a storage -type NewStorageFunc func(ctx context.Context, cfg interface{}) (ObjectStorage, error) +type NewStorageFunc func(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error) var storageMap = map[Type]NewStorageFunc{} // RegisterStorageType registers a provided storage type with a function to create it -func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg interface{}) (ObjectStorage, error)) { +func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error)) { storageMap[typ] = fn } @@ -151,11 +150,11 @@ func Init() error { } // NewStorage takes a storage type and some config and returns an ObjectStorage or an error -func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) { +func NewStorage(typStr Type, cfg *setting.Storage) (ObjectStorage, error) { if len(typStr) == 0 { - typStr = string(LocalStorageType) + typStr = setting.LocalStorageType } - fn, ok := storageMap[Type(typStr)] + fn, ok := storageMap[typStr] if !ok { return nil, fmt.Errorf("Unsupported storage type: %s", typStr) } @@ -165,7 +164,7 @@ func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) { func initAvatars() (err error) { log.Info("Initialising Avatar storage with type: %s", setting.Avatar.Storage.Type) - Avatars, err = NewStorage(setting.Avatar.Storage.Type, &setting.Avatar.Storage) + Avatars, err = NewStorage(setting.Avatar.Storage.Type, setting.Avatar.Storage) return err } @@ -175,7 +174,7 @@ func initAttachments() (err error) { return nil } log.Info("Initialising Attachment storage with type: %s", setting.Attachment.Storage.Type) - Attachments, err = NewStorage(setting.Attachment.Storage.Type, &setting.Attachment.Storage) + Attachments, err = NewStorage(setting.Attachment.Storage.Type, setting.Attachment.Storage) return err } @@ -185,19 +184,19 @@ func initLFS() (err error) { return nil } log.Info("Initialising LFS storage with type: %s", setting.LFS.Storage.Type) - LFS, err = NewStorage(setting.LFS.Storage.Type, &setting.LFS.Storage) + LFS, err = NewStorage(setting.LFS.Storage.Type, setting.LFS.Storage) return err } func initRepoAvatars() (err error) { log.Info("Initialising Repository Avatar storage with type: %s", setting.RepoAvatar.Storage.Type) - RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, &setting.RepoAvatar.Storage) + RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, setting.RepoAvatar.Storage) return err } func initRepoArchives() (err error) { log.Info("Initialising Repository Archive storage with type: %s", setting.RepoArchive.Storage.Type) - RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage) + RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, setting.RepoArchive.Storage) return err } @@ -207,7 +206,7 @@ func initPackages() (err error) { return nil } log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type) - Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage) + Packages, err = NewStorage(setting.Packages.Storage.Type, setting.Packages.Storage) return err } @@ -218,10 +217,10 @@ func initActions() (err error) { return nil } log.Info("Initialising Actions storage with type: %s", setting.Actions.LogStorage.Type) - if Actions, err = NewStorage(setting.Actions.LogStorage.Type, &setting.Actions.LogStorage); err != nil { + if Actions, err = NewStorage(setting.Actions.LogStorage.Type, setting.Actions.LogStorage); err != nil { return err } log.Info("Initialising ActionsArtifacts storage with type: %s", setting.Actions.ArtifactStorage.Type) - ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, &setting.Actions.ArtifactStorage) + ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, setting.Actions.ArtifactStorage) return err } diff --git a/modules/storage/storage_test.go b/modules/storage/storage_test.go index b517a9e71a..5e3e9c7dba 100644 --- a/modules/storage/storage_test.go +++ b/modules/storage/storage_test.go @@ -7,10 +7,12 @@ import ( "bytes" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) -func testStorageIterator(t *testing.T, typStr string, cfg interface{}) { +func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) { l, err := NewStorage(typStr, cfg) assert.NoError(t, err) diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index e25def6df2..2b468d6e73 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -200,7 +200,7 @@ func GetRawFileOrLFS(ctx *context.APIContext) { return } - if setting.LFS.ServeDirect { + if setting.LFS.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name()) if u != nil && err == nil { @@ -320,7 +320,7 @@ func download(ctx *context.APIContext, archiveName string, archiver *repo_model. downloadName := ctx.Repo.Repository.Name + "-" + archiveName rPath := archiver.RelativePath() - if setting.RepoArchive.ServeDirect { + if setting.RepoArchive.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. u, err := storage.RepoArchives.URL(rPath, downloadName) if u != nil && err == nil { diff --git a/routers/install/install.go b/routers/install/install.go index 385954a515..4dba64df01 100644 --- a/routers/install/install.go +++ b/routers/install/install.go @@ -116,7 +116,7 @@ func Install(ctx *context.Context) { // Application general settings form.AppName = setting.AppName form.RepoRootPath = setting.RepoRootPath - form.LFSRootPath = setting.LFS.Path + form.LFSRootPath = setting.LFS.Storage.Path // Note(unknown): it's hard for Windows users change a running user, // so just use current one if config says default. diff --git a/routers/web/base.go b/routers/web/base.go index 1f6c4fbfc5..e4b7d8ce8d 100644 --- a/routers/web/base.go +++ b/routers/web/base.go @@ -19,11 +19,11 @@ import ( "code.gitea.io/gitea/modules/web/routing" ) -func storageHandler(storageSetting setting.Storage, prefix string, objStore storage.ObjectStorage) func(next http.Handler) http.Handler { +func storageHandler(storageSetting *setting.Storage, prefix string, objStore storage.ObjectStorage) func(next http.Handler) http.Handler { prefix = strings.Trim(prefix, "/") funcInfo := routing.GetFuncInfo(storageHandler, prefix) return func(next http.Handler) http.Handler { - if storageSetting.ServeDirect { + if storageSetting.MinioConfig.ServeDirect { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Method != "GET" && req.Method != "HEAD" { next.ServeHTTP(w, req) diff --git a/routers/web/repo/attachment.go b/routers/web/repo/attachment.go index dd24248024..af1842ad10 100644 --- a/routers/web/repo/attachment.go +++ b/routers/web/repo/attachment.go @@ -126,7 +126,7 @@ func ServeAttachment(ctx *context.Context, uuid string) { return } - if setting.Attachment.ServeDirect { + if setting.Attachment.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name) diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index a498180f35..fd67b82ef2 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -53,7 +53,7 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified time.Time return nil } - if setting.LFS.ServeDirect { + if setting.LFS.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name()) if u != nil && err == nil { diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 8944890f6a..a1e1346b38 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -427,7 +427,7 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep downloadName := ctx.Repo.Repository.Name + "-" + archiveName rPath := archiver.RelativePath() - if setting.RepoArchive.ServeDirect { + if setting.RepoArchive.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. u, err := storage.RepoArchives.URL(rPath, downloadName) if u != nil && err == nil { diff --git a/services/lfs/server.go b/services/lfs/server.go index 1f82aed54b..a18e752d47 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -452,7 +452,7 @@ func buildObjectResponse(rc *requestContext, pointer lfs_module.Pointer, downloa if download { var link *lfs_module.Link - if setting.LFS.ServeDirect { + if setting.LFS.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid) if u != nil && err == nil { diff --git a/templates/admin/config.tmpl b/templates/admin/config.tmpl index 0c52830ab0..2850cc8d37 100644 --- a/templates/admin/config.tmpl +++ b/templates/admin/config.tmpl @@ -102,7 +102,7 @@
{{if .LFS.StartServer}}{{svg "octicon-check"}}{{else}}{{svg "octicon-x"}}{{end}}
{{if .LFS.StartServer}}
{{.locale.Tr "admin.config.lfs_content_path"}}
-
{{.LFS.Path}}
+
{{JsonUtils.EncodeToString .LFS.Storage.ToShadowCopy}}
{{.locale.Tr "admin.config.lfs_http_auth_expiry"}}
{{.LFS.HTTPAuthExpiry}}
{{end}} diff --git a/tests/test_utils.go b/tests/test_utils.go index 6f0fb25560..5540d92e92 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -214,7 +214,9 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { // load LFS object fixtures // (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API) - lfsFixtures, err := storage.NewStorage("", storage.LocalStorageConfig{Path: path.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta")}) + lfsFixtures, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{ + Path: filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta"), + }) assert.NoError(t, err) assert.NoError(t, storage.Clean(storage.LFS)) assert.NoError(t, lfsFixtures.IterateObjects("", func(path string, _ storage.Object) error {