Divide GetIssueStats query in smaller chunks (#10176)

* Divide GetIssueStats query in smaller chunks

* Skip chunking if count is low enough

* Fix lint

* Define maxQueryParameters

* Remove absMaxQueryParameters because of lint

* Restart CI

* Restart CI

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lauris BH <lauris@nix.lv>
This commit is contained in:
guillep2k 2020-02-15 07:51:25 -03:00 committed by GitHub
parent 7e920703f9
commit 15614a8368
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 36 additions and 0 deletions

View File

@ -1352,6 +1352,36 @@ type IssueStatsOptions struct {
// GetIssueStats returns issue statistic information by given conditions. // GetIssueStats returns issue statistic information by given conditions.
func GetIssueStats(opts *IssueStatsOptions) (*IssueStats, error) { func GetIssueStats(opts *IssueStatsOptions) (*IssueStats, error) {
if len(opts.IssueIDs) <= maxQueryParameters {
return getIssueStatsChunk(opts, opts.IssueIDs)
}
// If too long a list of IDs is provided, we get the statistics in
// smaller chunks and get accumulates. Note: this could potentially
// get us invalid results. The alternative is to insert the list of
// ids in a temporary table and join from them.
accum := &IssueStats{}
for i := 0; i < len(opts.IssueIDs); {
chunk := i + maxQueryParameters
if chunk > len(opts.IssueIDs) {
chunk = len(opts.IssueIDs)
}
stats, err := getIssueStatsChunk(opts, opts.IssueIDs[i:chunk])
if err != nil {
return nil, err
}
accum.OpenCount += stats.OpenCount
accum.ClosedCount += stats.ClosedCount
accum.YourRepositoriesCount += stats.YourRepositoriesCount
accum.AssignCount += stats.AssignCount
accum.CreateCount += stats.CreateCount
accum.OpenCount += stats.MentionCount
i = chunk
}
return accum, nil
}
func getIssueStatsChunk(opts *IssueStatsOptions, issueIDs []int64) (*IssueStats, error) {
stats := &IssueStats{} stats := &IssueStats{}
countSession := func(opts *IssueStatsOptions) *xorm.Session { countSession := func(opts *IssueStatsOptions) *xorm.Session {

View File

@ -48,6 +48,12 @@ type Engine interface {
SumInt(bean interface{}, columnName string) (res int64, err error) SumInt(bean interface{}, columnName string) (res int64, err error)
} }
const (
// When queries are broken down in parts because of the number
// of parameters, attempt to break by this amount
maxQueryParameters = 300
)
var ( var (
x *xorm.Engine x *xorm.Engine
tables []interface{} tables []interface{}