Fix storage Iterate bug and Add storage doctor to delete garbage attachments (#16971) (#16977)

* Fix storage Iterate bug and Add storage doctor to delete garbage attachments

* Close object when used
This commit is contained in:
Lunny Xiao 2021-09-08 02:39:05 +08:00 committed by GitHub
parent 30708d9ffe
commit 07489d0405
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 83 additions and 3 deletions

View File

@ -124,7 +124,6 @@ func runRecreateTable(ctx *cli.Context) error {
}
func runDoctor(ctx *cli.Context) error {
// Silence the default loggers
log.DelNamedLogger("console")
log.DelNamedLogger(log.DEFAULT)

View File

@ -144,6 +144,11 @@ func GetAttachmentByUUID(uuid string) (*Attachment, error) {
return getAttachmentByUUID(x, uuid)
}
// ExistAttachmentsByUUID returns true if attachment is exist by given UUID
func ExistAttachmentsByUUID(uuid string) (bool, error) {
return x.Where("`uuid`=?", uuid).Exist(new(Attachment))
}
// GetAttachmentByReleaseIDFileName returns attachment by given releaseId and fileName.
func GetAttachmentByReleaseIDFileName(releaseID int64, fileName string) (*Attachment, error) {
return getAttachmentByReleaseIDFileName(x, releaseID, fileName)

76
modules/doctor/storage.go Normal file
View File

@ -0,0 +1,76 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/storage"
)
func checkAttachmentStorageFiles(logger log.Logger, autofix bool) error {
var total, garbageNum int
var deletePaths []string
if err := storage.Attachments.IterateObjects(func(p string, obj storage.Object) error {
defer obj.Close()
total++
stat, err := obj.Stat()
if err != nil {
return err
}
exist, err := models.ExistAttachmentsByUUID(stat.Name())
if err != nil {
return err
}
if !exist {
garbageNum++
if autofix {
deletePaths = append(deletePaths, p)
}
}
return nil
}); err != nil {
logger.Error("storage.Attachments.IterateObjects failed: %v", err)
return err
}
if garbageNum > 0 {
if autofix {
var deletedNum int
for _, p := range deletePaths {
if err := storage.Attachments.Delete(p); err != nil {
log.Error("Delete attachment %s failed: %v", p, err)
} else {
deletedNum++
}
}
logger.Info("%d missed information attachment detected, %d deleted.", garbageNum, deletedNum)
} else {
logger.Warn("Checked %d attachment, %d missed information.", total, garbageNum)
}
}
return nil
}
func checkStorageFiles(logger log.Logger, autofix bool) error {
if err := storage.Init(); err != nil {
logger.Error("storage.Init failed: %v", err)
return err
}
return checkAttachmentStorageFiles(logger, autofix)
}
func init() {
Register(&Check{
Title: "Check if there is garbage storage files",
Name: "storages",
IsDefault: false,
Run: checkStorageFiles,
AbortIfFailed: false,
SkipDatabaseInitialization: false,
Priority: 1,
})
}

View File

@ -151,7 +151,7 @@ type minioFileInfo struct {
}
func (m minioFileInfo) Name() string {
return m.ObjectInfo.Key
return path.Base(m.ObjectInfo.Key)
}
func (m minioFileInfo) Size() int64 {
@ -219,7 +219,7 @@ func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) er
}
if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
defer object.Close()
return fn(strings.TrimPrefix(m.basePath, mObjInfo.Key), &minioObject{object})
return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
}(object, fn); err != nil {
return convertMinioErr(err)
}