#835: Realtime webhooks

This commit is contained in:
Unknwon 2015-07-25 21:32:04 +08:00
parent 2b1442f3df
commit fa298a2c30
13 changed files with 140 additions and 69 deletions

View File

@ -16,6 +16,7 @@ import (
"github.com/codegangsta/cli" "github.com/codegangsta/cli"
"github.com/gogits/gogs/models" "github.com/gogits/gogs/models"
"github.com/gogits/gogs/modules/httplib"
"github.com/gogits/gogs/modules/log" "github.com/gogits/gogs/modules/log"
"github.com/gogits/gogs/modules/setting" "github.com/gogits/gogs/modules/setting"
"github.com/gogits/gogs/modules/uuid" "github.com/gogits/gogs/modules/uuid"
@ -193,6 +194,12 @@ func runServ(c *cli.Context) {
} }
} }
// Send deliver hook request.
resp, err := httplib.Head(setting.AppUrl + setting.AppSubUrl + repoUserName + "/" + repoName + "/hooks/trigger").Response()
if err == nil {
resp.Body.Close()
}
// Update key activity. // Update key activity.
key, err := models.GetPublicKeyById(keyId) key, err := models.GetPublicKeyById(keyId)
if err != nil { if err != nil {

View File

@ -451,6 +451,7 @@ func runWeb(ctx *cli.Context) {
m.Get("/archive/*", repo.Download) m.Get("/archive/*", repo.Download)
m.Get("/pulls2/", repo.PullRequest2) m.Get("/pulls2/", repo.PullRequest2)
m.Get("/milestone2/", repo.Milestones2) m.Get("/milestone2/", repo.Milestones2)
m.Head("/hooks/trigger", repo.TriggerHook)
m.Group("", func() { m.Group("", func() {
m.Get("/src/*", repo.Home) m.Get("/src/*", repo.Home)

View File

@ -91,8 +91,8 @@ ENABLE_REVERSE_PROXY_AUTO_REGISTRATION = false
DISABLE_MINIMUM_KEY_SIZE_CHECK = false DISABLE_MINIMUM_KEY_SIZE_CHECK = false
[webhook] [webhook]
; Cron task interval in minutes ; Hook task queue length
TASK_INTERVAL = 1 QUEUE_LENGTH = 1000
; Deliver timeout in seconds ; Deliver timeout in seconds
DELIVER_TIMEOUT = 5 DELIVER_TIMEOUT = 5
; Allow insecure certification ; Allow insecure certification

View File

@ -17,7 +17,7 @@ import (
"github.com/gogits/gogs/modules/setting" "github.com/gogits/gogs/modules/setting"
) )
const APP_VER = "0.6.2.0725 Beta" const APP_VER = "0.6.3.0725 Beta"
func init() { func init() {
runtime.GOMAXPROCS(runtime.NumCPU()) runtime.GOMAXPROCS(runtime.NumCPU())

View File

@ -431,6 +431,8 @@ func CommitRepoAction(userId, repoUserId int64, userName, actEmail string,
} }
if err = CreateHookTask(&HookTask{ if err = CreateHookTask(&HookTask{
RepoID: repo.Id,
HookID: w.Id,
Type: w.HookTaskType, Type: w.HookTaskType,
Url: w.Url, Url: w.Url,
BasePayload: payload, BasePayload: payload,

View File

@ -9,6 +9,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"io/ioutil" "io/ioutil"
"sync"
"time" "time"
"github.com/gogits/gogs/modules/httplib" "github.com/gogits/gogs/modules/httplib"
@ -259,7 +260,9 @@ func (p Payload) GetJSONPayload() ([]byte, error) {
// HookTask represents a hook task. // HookTask represents a hook task.
type HookTask struct { type HookTask struct {
Id int64 ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX"`
HookID int64
Uuid string Uuid string
Type HookTaskType Type HookTaskType
Url string Url string
@ -269,6 +272,7 @@ type HookTask struct {
EventType HookEventType EventType HookEventType
IsSsl bool IsSsl bool
IsDelivered bool IsDelivered bool
Delivered int64
IsSucceed bool IsSucceed bool
} }
@ -287,31 +291,44 @@ func CreateHookTask(t *HookTask) error {
// UpdateHookTask updates information of hook task. // UpdateHookTask updates information of hook task.
func UpdateHookTask(t *HookTask) error { func UpdateHookTask(t *HookTask) error {
_, err := x.Id(t.Id).AllCols().Update(t) _, err := x.Id(t.ID).AllCols().Update(t)
return err return err
} }
var ( type hookQueue struct {
// Prevent duplicate deliveries. // Make sure one repository only occur once in the queue.
// This happens with massive hook tasks cannot finish delivering lock sync.Mutex
// before next shooting starts. repoIDs map[int64]bool
isShooting = false
)
// DeliverHooks checks and delivers undelivered hooks. queue chan int64
// FIXME: maybe can use goroutine to shoot a number of them at same time? }
func DeliverHooks() {
if isShooting { func (q *hookQueue) removeRepoID(id int64) {
q.lock.Lock()
defer q.lock.Unlock()
delete(q.repoIDs, id)
}
func (q *hookQueue) addRepoID(id int64) {
q.lock.Lock()
if q.repoIDs[id] {
q.lock.Unlock()
return return
} }
isShooting = true q.repoIDs[id] = true
defer func() { isShooting = false }() q.lock.Unlock()
q.queue <- id
}
tasks := make([]*HookTask, 0, 10) // AddRepoID adds repository ID to hook delivery queue.
func (q *hookQueue) AddRepoID(id int64) {
go q.addRepoID(id)
}
var HookQueue *hookQueue
func deliverHook(t *HookTask) {
timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
x.Where("is_delivered=?", false).Iterate(new(HookTask),
func(idx int, bean interface{}) error {
t := bean.(*HookTask)
req := httplib.Post(t.Url).SetTimeout(timeout, timeout). req := httplib.Post(t.Url).SetTimeout(timeout, timeout).
Header("X-Gogs-Delivery", t.Uuid). Header("X-Gogs-Delivery", t.Uuid).
Header("X-Gogs-Event", string(t.EventType)). Header("X-Gogs-Event", string(t.EventType)).
@ -330,19 +347,20 @@ func DeliverHooks() {
switch t.Type { switch t.Type {
case GOGS: case GOGS:
{ {
if _, err := req.Response(); err != nil { if resp, err := req.Response(); err != nil {
log.Error(5, "Delivery: %v", err) log.Error(5, "Delivery: %v", err)
} else { } else {
resp.Body.Close()
t.IsSucceed = true t.IsSucceed = true
} }
} }
case SLACK: case SLACK:
{ {
if res, err := req.Response(); err != nil { if resp, err := req.Response(); err != nil {
log.Error(5, "Delivery: %v", err) log.Error(5, "Delivery: %v", err)
} else { } else {
defer res.Body.Close() defer resp.Body.Close()
contents, err := ioutil.ReadAll(res.Body) contents, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
log.Error(5, "%s", err) log.Error(5, "%s", err)
} else { } else {
@ -356,18 +374,54 @@ func DeliverHooks() {
} }
} }
tasks = append(tasks, t) t.Delivered = time.Now().UTC().UnixNano()
if t.IsSucceed { if t.IsSucceed {
log.Trace("Hook delivered(%s): %s", t.Uuid, t.PayloadContent) log.Trace("Hook delivered(%s): %s", t.Uuid, t.PayloadContent)
} }
}
// DeliverHooks checks and delivers undelivered hooks.
func DeliverHooks() {
tasks := make([]*HookTask, 0, 10)
x.Where("is_delivered=?", false).Iterate(new(HookTask),
func(idx int, bean interface{}) error {
t := bean.(*HookTask)
deliverHook(t)
tasks = append(tasks, t)
return nil return nil
}) })
// Update hook task status. // Update hook task status.
for _, t := range tasks { for _, t := range tasks {
if err := UpdateHookTask(t); err != nil { if err := UpdateHookTask(t); err != nil {
log.Error(4, "UpdateHookTask(%d): %v", t.Id, err) log.Error(4, "UpdateHookTask(%d): %v", t.ID, err)
}
}
HookQueue = &hookQueue{
lock: sync.Mutex{},
repoIDs: make(map[int64]bool),
queue: make(chan int64, setting.Webhook.QueueLength),
}
// Start listening on new hook requests.
for repoID := range HookQueue.queue {
HookQueue.removeRepoID(repoID)
tasks = make([]*HookTask, 0, 5)
if err := x.Where("repo_id=? AND is_delivered=?", repoID, false).Find(&tasks); err != nil {
log.Error(4, "Get repository(%d) hook tasks: %v", repoID, err)
continue
}
for _, t := range tasks {
deliverHook(t)
if err := UpdateHookTask(t); err != nil {
log.Error(4, "UpdateHookTask(%d): %v", t.ID, err)
}
} }
} }
} }
func InitDeliverHooks() {
go DeliverHooks()
}

File diff suppressed because one or more lines are too long

View File

@ -15,7 +15,6 @@ var c = New()
func NewCronContext() { func NewCronContext() {
c.AddFunc("Update mirrors", "@every 1h", models.MirrorUpdate) c.AddFunc("Update mirrors", "@every 1h", models.MirrorUpdate)
c.AddFunc("Deliver hooks", fmt.Sprintf("@every %dm", setting.Webhook.TaskInterval), models.DeliverHooks)
if setting.Git.Fsck.Enable { if setting.Git.Fsck.Enable {
c.AddFunc("Repository health check", fmt.Sprintf("@every %dh", setting.Git.Fsck.Interval), models.GitFsck) c.AddFunc("Repository health check", fmt.Sprintf("@every %dh", setting.Git.Fsck.Interval), models.GitFsck)
} }

View File

@ -76,7 +76,7 @@ var (
// Webhook settings. // Webhook settings.
Webhook struct { Webhook struct {
TaskInterval int QueueLength int
DeliverTimeout int DeliverTimeout int
SkipTLSVerify bool SkipTLSVerify bool
} }
@ -555,7 +555,7 @@ func newNotifyMailService() {
func newWebhookService() { func newWebhookService() {
sec := Cfg.Section("webhook") sec := Cfg.Section("webhook")
Webhook.TaskInterval = sec.Key("TASK_INTERVAL").MustInt(1) Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000)
Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5) Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5)
Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool() Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool()
} }

View File

@ -68,6 +68,7 @@ func GlobalInit() {
models.HasEngine = true models.HasEngine = true
cron.NewCronContext() cron.NewCronContext()
models.InitDeliverHooks()
log.NewGitLogger(path.Join(setting.LogRootPath, "http.log")) log.NewGitLogger(path.Join(setting.LogRootPath, "http.log"))
} }
if models.EnableSQLite3 { if models.EnableSQLite3 {

View File

@ -190,7 +190,10 @@ func Http(ctx *middleware.Context) {
refName := fields[2] refName := fields[2]
// FIXME: handle error. // FIXME: handle error.
models.Update(refName, oldCommitId, newCommitId, authUsername, username, reponame, authUser.Id) if err = models.Update(refName, oldCommitId, newCommitId, authUsername, username, reponame, authUser.Id); err == nil {
models.HookQueue.AddRepoID(repo.Id)
}
} }
lastLine = lastLine + size lastLine = lastLine + size
} else { } else {

View File

@ -634,3 +634,7 @@ func GitHooksEditPost(ctx *middleware.Context) {
} }
ctx.Redirect(ctx.Repo.RepoLink + "/settings/hooks/git") ctx.Redirect(ctx.Repo.RepoLink + "/settings/hooks/git")
} }
func TriggerHook(ctx *middleware.Context) {
models.HookQueue.AddRepoID(ctx.Repo.Repository.Id)
}

View File

@ -1 +1 @@
0.6.2.0725 Beta 0.6.3.0725 Beta