Add `PULL_LIMIT` and `PUSH_LIMIT` to cron.update_mirror task (#17568)

This commit is contained in:
zeripath 2021-11-23 03:09:35 +00:00 committed by GitHub
parent 9450410ff7
commit 188fd2dd1a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 70 additions and 17 deletions

View File

@ -1719,6 +1719,12 @@ PATH =
;RUN_AT_START = false ;RUN_AT_START = false
;; Notice if not success ;; Notice if not success
;NO_SUCCESS_NOTICE = true ;NO_SUCCESS_NOTICE = true
;; Limit the number of mirrors added to the queue to this number
;; (negative values mean no limit, 0 will result in no result in no mirrors being queued effectively disabling pull mirror updating.)
;PULL_LIMIT=50
;; Limit the number of mirrors added to the queue to this number
;; (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating)
;PUSH_LIMIT=50
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

View File

@ -828,6 +828,8 @@ NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take ef
- `SCHEDULE`: **@every 10m**: Cron syntax for scheduling update mirrors, e.g. `@every 3h`. - `SCHEDULE`: **@every 10m**: Cron syntax for scheduling update mirrors, e.g. `@every 3h`.
- `NO_SUCCESS_NOTICE`: **true**: The cron task for update mirrors success report is not very useful - as it just means that the mirrors have been queued. Therefore this is turned off by default. - `NO_SUCCESS_NOTICE`: **true**: The cron task for update mirrors success report is not very useful - as it just means that the mirrors have been queued. Therefore this is turned off by default.
- `PULL_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling pull mirror updating).
- `PUSH_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating).
#### Cron - Repository Health Check (`cron.repo_health_check`) #### Cron - Repository Health Check (`cron.repo_health_check`)

View File

@ -119,6 +119,7 @@ func MirrorsIterate(f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext). return db.GetEngine(db.DefaultContext).
Where("next_update_unix<=?", time.Now().Unix()). Where("next_update_unix<=?", time.Now().Unix()).
And("next_update_unix!=0"). And("next_update_unix!=0").
OrderBy("updated_unix ASC").
Iterate(new(Mirror), f) Iterate(new(Mirror), f)
} }

View File

@ -107,5 +107,6 @@ func PushMirrorsIterate(f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext). return db.GetEngine(db.DefaultContext).
Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()). Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
And("`interval` != 0"). And("`interval` != 0").
OrderBy("last_update ASC").
Iterate(new(PushMirror), f) Iterate(new(PushMirror), f)
} }

View File

@ -18,13 +18,24 @@ import (
) )
func registerUpdateMirrorTask() { func registerUpdateMirrorTask() {
RegisterTaskFatal("update_mirrors", &BaseConfig{ type UpdateMirrorTaskConfig struct {
Enabled: true, BaseConfig
RunAtStart: false, PullLimit int
Schedule: "@every 10m", PushLimit int
NoSuccessNotice: true, }
}, func(ctx context.Context, _ *models.User, _ Config) error {
return mirror_service.Update(ctx) RegisterTaskFatal("update_mirrors", &UpdateMirrorTaskConfig{
BaseConfig: BaseConfig{
Enabled: true,
RunAtStart: false,
Schedule: "@every 10m",
NoSuccessNotice: true,
},
PullLimit: 50,
PushLimit: 50,
}, func(ctx context.Context, _ *models.User, cfg Config) error {
umtc := cfg.(*UpdateMirrorTaskConfig)
return mirror_service.Update(ctx, umtc.PullLimit, umtc.PushLimit)
}) })
} }

View File

@ -45,15 +45,19 @@ func doMirrorSync(ctx context.Context, req *SyncRequest) {
} }
} }
var errLimit = fmt.Errorf("reached limit")
// Update checks and updates mirror repositories. // Update checks and updates mirror repositories.
func Update(ctx context.Context) error { func Update(ctx context.Context, pullLimit, pushLimit int) error {
if !setting.Mirror.Enabled { if !setting.Mirror.Enabled {
log.Warn("Mirror feature disabled, but cron job enabled: skip update") log.Warn("Mirror feature disabled, but cron job enabled: skip update")
return nil return nil
} }
log.Trace("Doing: Update") log.Trace("Doing: Update")
handler := func(idx int, bean interface{}) error { requested := 0
handler := func(idx int, bean interface{}, limit int) error {
var item SyncRequest var item SyncRequest
if m, ok := bean.(*models.Mirror); ok { if m, ok := bean.(*models.Mirror); ok {
if m.Repo == nil { if m.Repo == nil {
@ -78,21 +82,49 @@ func Update(ctx context.Context) error {
return nil return nil
} }
// Check we've not been cancelled
select { select {
case <-ctx.Done(): case <-ctx.Done():
return fmt.Errorf("Aborted") return fmt.Errorf("aborted")
default: default:
return mirrorQueue.Push(&item)
} }
// Check if this request is already in the queue
has, err := mirrorQueue.Has(&item)
if err != nil {
return err
}
if has {
return nil
}
// Push to the Queue
if err := mirrorQueue.Push(&item); err != nil {
return err
}
requested++
if limit > 0 && requested > limit {
return errLimit
}
return nil
} }
if err := models.MirrorsIterate(handler); err != nil { if pullLimit != 0 {
log.Error("MirrorsIterate: %v", err) if err := models.MirrorsIterate(func(idx int, bean interface{}) error {
return err return handler(idx, bean, pullLimit)
}); err != nil && err != errLimit {
log.Error("MirrorsIterate: %v", err)
return err
}
} }
if err := models.PushMirrorsIterate(handler); err != nil { if pushLimit != 0 {
log.Error("PushMirrorsIterate: %v", err) if err := models.PushMirrorsIterate(func(idx int, bean interface{}) error {
return err return handler(idx, bean, pushLimit)
}); err != nil && err != errLimit {
log.Error("PushMirrorsIterate: %v", err)
return err
}
} }
log.Trace("Finished: Update") log.Trace("Finished: Update")
return nil return nil