From ffc904b1e0635d17e55b5fbdea4e18832ee2276d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 17 Dec 2019 12:16:54 +0800 Subject: [PATCH] Sleep longer if request speed is over github limitation (#9335) * Sleep longer if request speed is over github limitation * improve code * remove unused code * fix lint * Use github's rate limit remain value to determine how long to sleep * Save reset time when finished github api request * fix bug * fix lint * Add context.Context for sleep * fix test * improve code * fix bug and lint * fix import order --- modules/migrations/base/downloader.go | 11 +++++ modules/migrations/git.go | 6 +++ modules/migrations/gitea.go | 5 ++- modules/migrations/gitea_test.go | 3 +- modules/migrations/github.go | 64 ++++++++++++++++++++++----- modules/migrations/migrate.go | 7 ++- modules/task/migrate.go | 3 +- routers/api/v1/repo/repo.go | 3 +- 8 files changed, 86 insertions(+), 16 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index b853ec3020..87ade5c02e 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -6,6 +6,7 @@ package base import ( + "context" "time" "code.gitea.io/gitea/modules/structs" @@ -13,6 +14,7 @@ import ( // Downloader downloads the site repo informations type Downloader interface { + SetContext(context.Context) GetRepoInfo() (*Repository, error) GetTopics() ([]string, error) GetMilestones() ([]*Milestone, error) @@ -30,6 +32,10 @@ type DownloaderFactory interface { GitServiceType() structs.GitServiceType } +var ( + _ Downloader = &RetryDownloader{} +) + // RetryDownloader retry the downloads type RetryDownloader struct { Downloader @@ -46,6 +52,11 @@ func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *Retr } } +// SetContext set context +func (d *RetryDownloader) SetContext(ctx context.Context) { + d.Downloader.SetContext(ctx) +} + // GetRepoInfo returns a repository information with retry func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { var ( diff --git a/modules/migrations/git.go b/modules/migrations/git.go index 75d05976cd..f7b1e857e4 100644 --- a/modules/migrations/git.go +++ b/modules/migrations/git.go @@ -5,6 +5,8 @@ package migrations import ( + "context" + "code.gitea.io/gitea/modules/migrations/base" ) @@ -28,6 +30,10 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl } } +// SetContext set context +func (g *PlainGitDownloader) SetContext(ctx context.Context) { +} + // GetRepoInfo returns a repository information func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { // convert github repo to stand Repo diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index db2143fe7e..f52f6c585a 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -6,6 +6,7 @@ package migrations import ( + "context" "fmt" "io" "net/http" @@ -35,6 +36,7 @@ var ( // GiteaLocalUploader implements an Uploader to gitea sites type GiteaLocalUploader struct { + ctx context.Context doer *models.User repoOwner string repoName string @@ -49,8 +51,9 @@ type GiteaLocalUploader struct { } // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 -func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader { +func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader { return &GiteaLocalUploader{ + ctx: ctx, doer: doer, repoOwner: repoOwner, repoName: repoName, diff --git a/modules/migrations/gitea_test.go b/modules/migrations/gitea_test.go index 73c119a15d..438902f320 100644 --- a/modules/migrations/gitea_test.go +++ b/modules/migrations/gitea_test.go @@ -10,6 +10,7 @@ import ( "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -27,7 +28,7 @@ func TestGiteaUploadRepo(t *testing.T) { var ( downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder") repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05") - uploader = NewGiteaLocalUploader(user, user.Name, repoName) + uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) ) err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{ diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 00d137a3de..fabdb4ae44 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "strings" + "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" @@ -73,6 +74,7 @@ type GithubDownloaderV3 struct { repoName string userName string password string + rate *github.Rate } // NewGithubDownloaderV3 creates a github Downloader via github v3 API @@ -107,12 +109,39 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith return &downloader } +// SetContext set context +func (g *GithubDownloaderV3) SetContext(ctx context.Context) { + g.ctx = ctx +} + +func (g *GithubDownloaderV3) sleep() { + for g.rate != nil && g.rate.Remaining <= 0 { + timer := time.NewTimer(time.Until(g.rate.Reset.Time)) + select { + case <-g.ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + + rates, _, err := g.client.RateLimits(g.ctx) + if err != nil { + log.Error("g.client.RateLimits: %s", err) + } + + g.rate = rates.GetCore() + } +} + // GetRepoInfo returns a repository information func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { - gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) + g.sleep() + gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) if err != nil { return nil, err } + g.rate = &resp.Rate + // convert github repo to stand Repo return &base.Repository{ Owner: g.repoOwner, @@ -126,8 +155,13 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { // GetTopics return github topics func (g *GithubDownloaderV3) GetTopics() ([]string, error) { - r, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) - return r.Topics, err + g.sleep() + r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) + if err != nil { + return nil, err + } + g.rate = &resp.Rate + return r.Topics, nil } // GetMilestones returns milestones @@ -135,7 +169,8 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { var perPage = 100 var milestones = make([]*base.Milestone, 0, perPage) for i := 1; ; i++ { - ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, + g.sleep() + ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, &github.MilestoneListOptions{ State: "all", ListOptions: github.ListOptions{ @@ -145,6 +180,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { if err != nil { return nil, err } + g.rate = &resp.Rate for _, m := range ms { var desc string @@ -189,7 +225,8 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { var perPage = 100 var labels = make([]*base.Label, 0, perPage) for i := 1; ; i++ { - ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, + g.sleep() + ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, @@ -197,6 +234,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { if err != nil { return nil, err } + g.rate = &resp.Rate for _, label := range ls { labels = append(labels, convertGithubLabel(label)) @@ -260,7 +298,8 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { var perPage = 100 var releases = make([]*base.Release, 0, perPage) for i := 1; ; i++ { - ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, + g.sleep() + ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, @@ -268,6 +307,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { if err != nil { return nil, err } + g.rate = &resp.Rate for _, release := range ls { releases = append(releases, g.convertGithubRelease(release)) @@ -304,11 +344,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, } var allIssues = make([]*base.Issue, 0, perPage) - - issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) + g.sleep() + issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } + g.rate = &resp.Rate for _, issue := range issues { if issue.IsPullRequest() { continue @@ -365,10 +406,12 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er }, } for { + g.sleep() comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } + g.rate = &resp.Rate for _, comment := range comments { var email string if comment.User.Email != nil { @@ -408,11 +451,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, } var allPRs = make([]*base.PullRequest, 0, perPage) - - prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) + g.sleep() + prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } + g.rate = &resp.Rate for _, pr := range prs { var body string if pr.Body != nil { diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 957d4c85d0..ece871a857 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -6,6 +6,7 @@ package migrations import ( + "context" "fmt" "code.gitea.io/gitea/models" @@ -28,10 +29,10 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { } // MigrateRepository migrate repository according MigrateOptions -func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { +func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { var ( downloader base.Downloader - uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName) + uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) theFactory base.DownloaderFactory ) @@ -69,6 +70,8 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) } + downloader.SetContext(ctx) + if err := migrateRepository(downloader, uploader, opts); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) diff --git a/modules/task/migrate.go b/modules/task/migrate.go index 247403d7be..d3b4fa45f0 100644 --- a/modules/task/migrate.go +++ b/modules/task/migrate.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" @@ -95,7 +96,7 @@ func runMigrateTask(t *models.Task) (err error) { } opts.MigrateToRepoID = t.RepoID - repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts) + repo, err := migrations.MigrateRepository(graceful.GetManager().HammerContext(), t.Doer, t.Owner.Name, *opts) if err == nil { log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) return nil diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index cab0fc07e0..be226c3438 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" @@ -481,7 +482,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { } }() - if _, err = migrations.MigrateRepository(ctx.User, ctxUser.Name, opts); err != nil { + if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, ctxUser.Name, opts); err != nil { handleMigrateError(ctx, ctxUser, remoteAddr, err) return }