Sleep longer if request speed is over github limitation (#9335)

* Sleep longer if request speed is over github limitation

* improve code

* remove unused code

* fix lint

* Use github's rate limit remain value to determine how long to sleep

* Save reset time when finished github api request

* fix bug

* fix lint

* Add context.Context for sleep

* fix test

* improve code

* fix bug and lint

* fix import order
This commit is contained in:
Lunny Xiao 2019-12-17 12:16:54 +08:00 committed by techknowlogick
parent d1a49977b0
commit ffc904b1e0
8 changed files with 86 additions and 16 deletions

View File

@ -6,6 +6,7 @@
package base package base
import ( import (
"context"
"time" "time"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
@ -13,6 +14,7 @@ import (
// Downloader downloads the site repo informations // Downloader downloads the site repo informations
type Downloader interface { type Downloader interface {
SetContext(context.Context)
GetRepoInfo() (*Repository, error) GetRepoInfo() (*Repository, error)
GetTopics() ([]string, error) GetTopics() ([]string, error)
GetMilestones() ([]*Milestone, error) GetMilestones() ([]*Milestone, error)
@ -30,6 +32,10 @@ type DownloaderFactory interface {
GitServiceType() structs.GitServiceType GitServiceType() structs.GitServiceType
} }
var (
_ Downloader = &RetryDownloader{}
)
// RetryDownloader retry the downloads // RetryDownloader retry the downloads
type RetryDownloader struct { type RetryDownloader struct {
Downloader Downloader
@ -46,6 +52,11 @@ func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *Retr
} }
} }
// SetContext set context
func (d *RetryDownloader) SetContext(ctx context.Context) {
d.Downloader.SetContext(ctx)
}
// GetRepoInfo returns a repository information with retry // GetRepoInfo returns a repository information with retry
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
var ( var (

View File

@ -5,6 +5,8 @@
package migrations package migrations
import ( import (
"context"
"code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/migrations/base"
) )
@ -28,6 +30,10 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl
} }
} }
// SetContext set context
func (g *PlainGitDownloader) SetContext(ctx context.Context) {
}
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
// convert github repo to stand Repo // convert github repo to stand Repo

View File

@ -6,6 +6,7 @@
package migrations package migrations
import ( import (
"context"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -35,6 +36,7 @@ var (
// GiteaLocalUploader implements an Uploader to gitea sites // GiteaLocalUploader implements an Uploader to gitea sites
type GiteaLocalUploader struct { type GiteaLocalUploader struct {
ctx context.Context
doer *models.User doer *models.User
repoOwner string repoOwner string
repoName string repoName string
@ -49,8 +51,9 @@ type GiteaLocalUploader struct {
} }
// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader { func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
return &GiteaLocalUploader{ return &GiteaLocalUploader{
ctx: ctx,
doer: doer, doer: doer,
repoOwner: repoOwner, repoOwner: repoOwner,
repoName: repoName, repoName: repoName,

View File

@ -10,6 +10,7 @@ import (
"time" "time"
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
@ -27,7 +28,7 @@ func TestGiteaUploadRepo(t *testing.T) {
var ( var (
downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder") downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder")
repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05") repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05")
uploader = NewGiteaLocalUploader(user, user.Name, repoName) uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
) )
err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{ err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{

View File

@ -11,6 +11,7 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"strings" "strings"
"time"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/migrations/base"
@ -73,6 +74,7 @@ type GithubDownloaderV3 struct {
repoName string repoName string
userName string userName string
password string password string
rate *github.Rate
} }
// NewGithubDownloaderV3 creates a github Downloader via github v3 API // NewGithubDownloaderV3 creates a github Downloader via github v3 API
@ -107,12 +109,39 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith
return &downloader return &downloader
} }
// SetContext set context
func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
g.ctx = ctx
}
func (g *GithubDownloaderV3) sleep() {
for g.rate != nil && g.rate.Remaining <= 0 {
timer := time.NewTimer(time.Until(g.rate.Reset.Time))
select {
case <-g.ctx.Done():
timer.Stop()
return
case <-timer.C:
}
rates, _, err := g.client.RateLimits(g.ctx)
if err != nil {
log.Error("g.client.RateLimits: %s", err)
}
g.rate = rates.GetCore()
}
}
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) g.sleep()
gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
g.rate = &resp.Rate
// convert github repo to stand Repo // convert github repo to stand Repo
return &base.Repository{ return &base.Repository{
Owner: g.repoOwner, Owner: g.repoOwner,
@ -126,8 +155,13 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
// GetTopics return github topics // GetTopics return github topics
func (g *GithubDownloaderV3) GetTopics() ([]string, error) { func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
r, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) g.sleep()
return r.Topics, err r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
g.rate = &resp.Rate
return r.Topics, nil
} }
// GetMilestones returns milestones // GetMilestones returns milestones
@ -135,7 +169,8 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100 var perPage = 100
var milestones = make([]*base.Milestone, 0, perPage) var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, g.sleep()
ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{ &github.MilestoneListOptions{
State: "all", State: "all",
ListOptions: github.ListOptions{ ListOptions: github.ListOptions{
@ -145,6 +180,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
g.rate = &resp.Rate
for _, m := range ms { for _, m := range ms {
var desc string var desc string
@ -189,7 +225,8 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
var perPage = 100 var perPage = 100
var labels = make([]*base.Label, 0, perPage) var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, g.sleep()
ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{ &github.ListOptions{
Page: i, Page: i,
PerPage: perPage, PerPage: perPage,
@ -197,6 +234,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
g.rate = &resp.Rate
for _, label := range ls { for _, label := range ls {
labels = append(labels, convertGithubLabel(label)) labels = append(labels, convertGithubLabel(label))
@ -260,7 +298,8 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
var perPage = 100 var perPage = 100
var releases = make([]*base.Release, 0, perPage) var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, g.sleep()
ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{ &github.ListOptions{
Page: i, Page: i,
PerPage: perPage, PerPage: perPage,
@ -268,6 +307,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
g.rate = &resp.Rate
for _, release := range ls { for _, release := range ls {
releases = append(releases, g.convertGithubRelease(release)) releases = append(releases, g.convertGithubRelease(release))
@ -304,11 +344,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
} }
var allIssues = make([]*base.Issue, 0, perPage) var allIssues = make([]*base.Issue, 0, perPage)
g.sleep()
issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err) return nil, false, fmt.Errorf("error while listing repos: %v", err)
} }
g.rate = &resp.Rate
for _, issue := range issues { for _, issue := range issues {
if issue.IsPullRequest() { if issue.IsPullRequest() {
continue continue
@ -365,10 +406,12 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
}, },
} }
for { for {
g.sleep()
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt) comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err) return nil, fmt.Errorf("error while listing repos: %v", err)
} }
g.rate = &resp.Rate
for _, comment := range comments { for _, comment := range comments {
var email string var email string
if comment.User.Email != nil { if comment.User.Email != nil {
@ -408,11 +451,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
}, },
} }
var allPRs = make([]*base.PullRequest, 0, perPage) var allPRs = make([]*base.PullRequest, 0, perPage)
g.sleep()
prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err) return nil, fmt.Errorf("error while listing repos: %v", err)
} }
g.rate = &resp.Rate
for _, pr := range prs { for _, pr := range prs {
var body string var body string
if pr.Body != nil { if pr.Body != nil {

View File

@ -6,6 +6,7 @@
package migrations package migrations
import ( import (
"context"
"fmt" "fmt"
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
@ -28,10 +29,10 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) {
} }
// MigrateRepository migrate repository according MigrateOptions // MigrateRepository migrate repository according MigrateOptions
func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
var ( var (
downloader base.Downloader downloader base.Downloader
uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName) uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
theFactory base.DownloaderFactory theFactory base.DownloaderFactory
) )
@ -69,6 +70,8 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
} }
downloader.SetContext(ctx)
if err := migrateRepository(downloader, uploader, opts); err != nil { if err := migrateRepository(downloader, uploader, opts); err != nil {
if err1 := uploader.Rollback(); err1 != nil { if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1) log.Error("rollback failed: %v", err1)

View File

@ -11,6 +11,7 @@ import (
"strings" "strings"
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/notification"
@ -95,7 +96,7 @@ func runMigrateTask(t *models.Task) (err error) {
} }
opts.MigrateToRepoID = t.RepoID opts.MigrateToRepoID = t.RepoID
repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts) repo, err := migrations.MigrateRepository(graceful.GetManager().HammerContext(), t.Doer, t.Owner.Name, *opts)
if err == nil { if err == nil {
log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name)
return nil return nil

View File

@ -18,6 +18,7 @@ import (
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/convert"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/notification"
@ -481,7 +482,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
} }
}() }()
if _, err = migrations.MigrateRepository(ctx.User, ctxUser.Name, opts); err != nil { if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, ctxUser.Name, opts); err != nil {
handleMigrateError(ctx, ctxUser, remoteAddr, err) handleMigrateError(ctx, ctxUser, remoteAddr, err)
return return
} }