fix bugs when too many IN variables (#4594)

This commit is contained in:
Lunny Xiao 2018-08-02 21:49:05 +08:00 committed by GitHub
parent c40f5d2d4c
commit 9c354a539a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 194 additions and 95 deletions

View File

@ -9,6 +9,11 @@ import "fmt"
// IssueList defines a list of issues // IssueList defines a list of issues
type IssueList []*Issue type IssueList []*Issue
const (
// default variables number on IN () in SQL
defaultMaxInSize = 50
)
func (issues IssueList) getRepoIDs() []int64 { func (issues IssueList) getRepoIDs() []int64 {
repoIDs := make(map[int64]struct{}, len(issues)) repoIDs := make(map[int64]struct{}, len(issues))
for _, issue := range issues { for _, issue := range issues {
@ -26,12 +31,21 @@ func (issues IssueList) loadRepositories(e Engine) ([]*Repository, error) {
repoIDs := issues.getRepoIDs() repoIDs := issues.getRepoIDs()
repoMaps := make(map[int64]*Repository, len(repoIDs)) repoMaps := make(map[int64]*Repository, len(repoIDs))
var left = len(repoIDs)
for left > 0 {
var limit = defaultMaxInSize
if left < limit {
limit = left
}
err := e. err := e.
In("id", repoIDs). In("id", repoIDs[:limit]).
Find(&repoMaps) Find(&repoMaps)
if err != nil { if err != nil {
return nil, fmt.Errorf("find repository: %v", err) return nil, fmt.Errorf("find repository: %v", err)
} }
left = left - limit
repoIDs = repoIDs[limit:]
}
for _, issue := range issues { for _, issue := range issues {
issue.Repo = repoMaps[issue.RepoID] issue.Repo = repoMaps[issue.RepoID]
@ -61,12 +75,21 @@ func (issues IssueList) loadPosters(e Engine) error {
posterIDs := issues.getPosterIDs() posterIDs := issues.getPosterIDs()
posterMaps := make(map[int64]*User, len(posterIDs)) posterMaps := make(map[int64]*User, len(posterIDs))
var left = len(posterIDs)
for left > 0 {
var limit = defaultMaxInSize
if left < limit {
limit = left
}
err := e. err := e.
In("id", posterIDs). In("id", posterIDs[:limit]).
Find(&posterMaps) Find(&posterMaps)
if err != nil { if err != nil {
return err return err
} }
left = left - limit
posterIDs = posterIDs[limit:]
}
for _, issue := range issues { for _, issue := range issues {
if issue.PosterID <= 0 { if issue.PosterID <= 0 {
@ -99,24 +122,35 @@ func (issues IssueList) loadLabels(e Engine) error {
} }
var issueLabels = make(map[int64][]*Label, len(issues)*3) var issueLabels = make(map[int64][]*Label, len(issues)*3)
var issueIDs = issues.getIssueIDs()
var left = len(issueIDs)
for left > 0 {
var limit = defaultMaxInSize
if left < limit {
limit = left
}
rows, err := e.Table("label"). rows, err := e.Table("label").
Join("LEFT", "issue_label", "issue_label.label_id = label.id"). Join("LEFT", "issue_label", "issue_label.label_id = label.id").
In("issue_label.issue_id", issues.getIssueIDs()). In("issue_label.issue_id", issueIDs[:limit]).
Asc("label.name"). Asc("label.name").
Rows(new(LabelIssue)) Rows(new(LabelIssue))
if err != nil { if err != nil {
return err return err
} }
defer rows.Close()
for rows.Next() { for rows.Next() {
var labelIssue LabelIssue var labelIssue LabelIssue
err = rows.Scan(&labelIssue) err = rows.Scan(&labelIssue)
if err != nil { if err != nil {
rows.Close()
return err return err
} }
issueLabels[labelIssue.IssueLabel.IssueID] = append(issueLabels[labelIssue.IssueLabel.IssueID], labelIssue.Label) issueLabels[labelIssue.IssueLabel.IssueID] = append(issueLabels[labelIssue.IssueLabel.IssueID], labelIssue.Label)
} }
rows.Close()
left = left - limit
issueIDs = issueIDs[limit:]
}
for _, issue := range issues { for _, issue := range issues {
issue.Labels = issueLabels[issue.ID] issue.Labels = issueLabels[issue.ID]
@ -141,12 +175,21 @@ func (issues IssueList) loadMilestones(e Engine) error {
} }
milestoneMaps := make(map[int64]*Milestone, len(milestoneIDs)) milestoneMaps := make(map[int64]*Milestone, len(milestoneIDs))
var left = len(milestoneIDs)
for left > 0 {
var limit = defaultMaxInSize
if left < limit {
limit = left
}
err := e. err := e.
In("id", milestoneIDs). In("id", milestoneIDs[:limit]).
Find(&milestoneMaps) Find(&milestoneMaps)
if err != nil { if err != nil {
return err return err
} }
left = left - limit
milestoneIDs = milestoneIDs[limit:]
}
for _, issue := range issues { for _, issue := range issues {
issue.Milestone = milestoneMaps[issue.MilestoneID] issue.Milestone = milestoneMaps[issue.MilestoneID]
@ -165,24 +208,36 @@ func (issues IssueList) loadAssignees(e Engine) error {
} }
var assignees = make(map[int64][]*User, len(issues)) var assignees = make(map[int64][]*User, len(issues))
var issueIDs = issues.getIssueIDs()
var left = len(issueIDs)
for left > 0 {
var limit = defaultMaxInSize
if left < limit {
limit = left
}
rows, err := e.Table("issue_assignees"). rows, err := e.Table("issue_assignees").
Join("INNER", "`user`", "`user`.id = `issue_assignees`.assignee_id"). Join("INNER", "`user`", "`user`.id = `issue_assignees`.assignee_id").
In("`issue_assignees`.issue_id", issues.getIssueIDs()). In("`issue_assignees`.issue_id", issueIDs[:limit]).
Rows(new(AssigneeIssue)) Rows(new(AssigneeIssue))
if err != nil { if err != nil {
return err return err
} }
defer rows.Close()
for rows.Next() { for rows.Next() {
var assigneeIssue AssigneeIssue var assigneeIssue AssigneeIssue
err = rows.Scan(&assigneeIssue) err = rows.Scan(&assigneeIssue)
if err != nil { if err != nil {
rows.Close()
return err return err
} }
assignees[assigneeIssue.IssueAssignee.IssueID] = append(assignees[assigneeIssue.IssueAssignee.IssueID], assigneeIssue.Assignee) assignees[assigneeIssue.IssueAssignee.IssueID] = append(assignees[assigneeIssue.IssueAssignee.IssueID], assigneeIssue.Assignee)
} }
rows.Close()
left = left - limit
issueIDs = issueIDs[limit:]
}
for _, issue := range issues { for _, issue := range issues {
issue.Assignees = assignees[issue.ID] issue.Assignees = assignees[issue.ID]
@ -207,23 +262,34 @@ func (issues IssueList) loadPullRequests(e Engine) error {
} }
pullRequestMaps := make(map[int64]*PullRequest, len(issuesIDs)) pullRequestMaps := make(map[int64]*PullRequest, len(issuesIDs))
var left = len(issuesIDs)
for left > 0 {
var limit = defaultMaxInSize
if left < limit {
limit = left
}
rows, err := e. rows, err := e.
In("issue_id", issuesIDs). In("issue_id", issuesIDs[:limit]).
Rows(new(PullRequest)) Rows(new(PullRequest))
if err != nil { if err != nil {
return err return err
} }
defer rows.Close()
for rows.Next() { for rows.Next() {
var pr PullRequest var pr PullRequest
err = rows.Scan(&pr) err = rows.Scan(&pr)
if err != nil { if err != nil {
rows.Close()
return err return err
} }
pullRequestMaps[pr.IssueID] = &pr pullRequestMaps[pr.IssueID] = &pr
} }
rows.Close()
left = left - limit
issuesIDs = issuesIDs[limit:]
}
for _, issue := range issues { for _, issue := range issues {
issue.PullRequest = pullRequestMaps[issue.ID] issue.PullRequest = pullRequestMaps[issue.ID]
} }
@ -236,24 +302,36 @@ func (issues IssueList) loadAttachments(e Engine) (err error) {
} }
var attachments = make(map[int64][]*Attachment, len(issues)) var attachments = make(map[int64][]*Attachment, len(issues))
var issuesIDs = issues.getIssueIDs()
var left = len(issuesIDs)
for left > 0 {
var limit = defaultMaxInSize
if left < limit {
limit = left
}
rows, err := e.Table("attachment"). rows, err := e.Table("attachment").
Join("INNER", "issue", "issue.id = attachment.issue_id"). Join("INNER", "issue", "issue.id = attachment.issue_id").
In("issue.id", issues.getIssueIDs()). In("issue.id", issuesIDs[:limit]).
Rows(new(Attachment)) Rows(new(Attachment))
if err != nil { if err != nil {
return err return err
} }
defer rows.Close()
for rows.Next() { for rows.Next() {
var attachment Attachment var attachment Attachment
err = rows.Scan(&attachment) err = rows.Scan(&attachment)
if err != nil { if err != nil {
rows.Close()
return err return err
} }
attachments[attachment.IssueID] = append(attachments[attachment.IssueID], &attachment) attachments[attachment.IssueID] = append(attachments[attachment.IssueID], &attachment)
} }
rows.Close()
left = left - limit
issuesIDs = issuesIDs[limit:]
}
for _, issue := range issues { for _, issue := range issues {
issue.Attachments = attachments[issue.ID] issue.Attachments = attachments[issue.ID]
} }
@ -266,23 +344,34 @@ func (issues IssueList) loadComments(e Engine) (err error) {
} }
var comments = make(map[int64][]*Comment, len(issues)) var comments = make(map[int64][]*Comment, len(issues))
var issuesIDs = issues.getIssueIDs()
var left = len(issuesIDs)
for left > 0 {
var limit = defaultMaxInSize
if left < limit {
limit = left
}
rows, err := e.Table("comment"). rows, err := e.Table("comment").
Join("INNER", "issue", "issue.id = comment.issue_id"). Join("INNER", "issue", "issue.id = comment.issue_id").
In("issue.id", issues.getIssueIDs()). In("issue.id", issuesIDs[:limit]).
Rows(new(Comment)) Rows(new(Comment))
if err != nil { if err != nil {
return err return err
} }
defer rows.Close()
for rows.Next() { for rows.Next() {
var comment Comment var comment Comment
err = rows.Scan(&comment) err = rows.Scan(&comment)
if err != nil { if err != nil {
rows.Close()
return err return err
} }
comments[comment.IssueID] = append(comments[comment.IssueID], &comment) comments[comment.IssueID] = append(comments[comment.IssueID], &comment)
} }
rows.Close()
left = left - limit
issuesIDs = issuesIDs[limit:]
}
for _, issue := range issues { for _, issue := range issues {
issue.Comments = comments[issue.ID] issue.Comments = comments[issue.ID]
@ -307,26 +396,36 @@ func (issues IssueList) loadTotalTrackedTimes(e Engine) (err error) {
} }
} }
var left = len(ids)
for left > 0 {
var limit = defaultMaxInSize
if left < limit {
limit = left
}
// select issue_id, sum(time) from tracked_time where issue_id in (<issue ids in current page>) group by issue_id // select issue_id, sum(time) from tracked_time where issue_id in (<issue ids in current page>) group by issue_id
rows, err := e.Table("tracked_time"). rows, err := e.Table("tracked_time").
Select("issue_id, sum(time) as time"). Select("issue_id, sum(time) as time").
In("issue_id", ids). In("issue_id", ids[:limit]).
GroupBy("issue_id"). GroupBy("issue_id").
Rows(new(totalTimesByIssue)) Rows(new(totalTimesByIssue))
if err != nil { if err != nil {
return err return err
} }
defer rows.Close()
for rows.Next() { for rows.Next() {
var totalTime totalTimesByIssue var totalTime totalTimesByIssue
err = rows.Scan(&totalTime) err = rows.Scan(&totalTime)
if err != nil { if err != nil {
rows.Close()
return err return err
} }
trackedTimes[totalTime.IssueID] = totalTime.Time trackedTimes[totalTime.IssueID] = totalTime.Time
} }
rows.Close()
left = left - limit
ids = ids[limit:]
}
for _, issue := range issues { for _, issue := range issues {
issue.TotalTrackedTime = trackedTimes[issue.ID] issue.TotalTrackedTime = trackedTimes[issue.ID]