Implement webhook branch filter (#7791)

* Fix validate() function to handle errors in embedded anon structs

* Implement webhook branch filter

See #2025, #3998.
pull/8148/head
WGH 3 years ago committed by Lauris BH
parent 0118b6aaf8
commit 6ddd3b0b47
  1. 1
      go.mod
  2. 2
      go.sum
  3. 7
      models/fixtures/webhook.yml
  4. 52
      models/webhook.go
  5. 34
      models/webhook_test.go
  6. 17
      modules/auth/auth.go
  7. 1
      modules/auth/repo_form.go
  8. 12
      modules/structs/hook.go
  9. 25
      modules/validation/binding.go
  10. 5
      modules/validation/binding_test.go
  11. 62
      modules/validation/glob_pattern_test.go
  12. 3
      options/locale/locale_en-US.ini
  13. 2
      routers/api/v1/utils/hook.go
  14. 1
      routers/repo/webhook.go
  15. 7
      templates/repo/settings/webhook/settings.tmpl
  16. 8
      templates/swagger/v1_json.tmpl
  17. 8
      vendor/github.com/gobwas/glob/.gitignore
  18. 9
      vendor/github.com/gobwas/glob/.travis.yml
  19. 21
      vendor/github.com/gobwas/glob/LICENSE
  20. 26
      vendor/github.com/gobwas/glob/bench.sh
  21. 525
      vendor/github.com/gobwas/glob/compiler/compiler.go
  22. 80
      vendor/github.com/gobwas/glob/glob.go
  23. 45
      vendor/github.com/gobwas/glob/match/any.go
  24. 82
      vendor/github.com/gobwas/glob/match/any_of.go
  25. 146
      vendor/github.com/gobwas/glob/match/btree.go
  26. 58
      vendor/github.com/gobwas/glob/match/contains.go
  27. 99
      vendor/github.com/gobwas/glob/match/every_of.go
  28. 49
      vendor/github.com/gobwas/glob/match/list.go
  29. 81
      vendor/github.com/gobwas/glob/match/match.go
  30. 49
      vendor/github.com/gobwas/glob/match/max.go
  31. 57
      vendor/github.com/gobwas/glob/match/min.go
  32. 27
      vendor/github.com/gobwas/glob/match/nothing.go
  33. 50
      vendor/github.com/gobwas/glob/match/prefix.go
  34. 55
      vendor/github.com/gobwas/glob/match/prefix_any.go
  35. 62
      vendor/github.com/gobwas/glob/match/prefix_suffix.go
  36. 48
      vendor/github.com/gobwas/glob/match/range.go
  37. 77
      vendor/github.com/gobwas/glob/match/row.go
  38. 91
      vendor/github.com/gobwas/glob/match/segments.go
  39. 43
      vendor/github.com/gobwas/glob/match/single.go
  40. 35
      vendor/github.com/gobwas/glob/match/suffix.go
  41. 43
      vendor/github.com/gobwas/glob/match/suffix_any.go
  42. 33
      vendor/github.com/gobwas/glob/match/super.go
  43. 45
      vendor/github.com/gobwas/glob/match/text.go
  44. 148
      vendor/github.com/gobwas/glob/readme.md
  45. 122
      vendor/github.com/gobwas/glob/syntax/ast/ast.go
  46. 157
      vendor/github.com/gobwas/glob/syntax/ast/parser.go
  47. 273
      vendor/github.com/gobwas/glob/syntax/lexer/lexer.go
  48. 88
      vendor/github.com/gobwas/glob/syntax/lexer/token.go
  49. 14
      vendor/github.com/gobwas/glob/syntax/syntax.go
  50. 154
      vendor/github.com/gobwas/glob/util/runes/runes.go
  51. 39
      vendor/github.com/gobwas/glob/util/strings/strings.go
  52. 9
      vendor/modules.txt

@ -47,6 +47,7 @@ require (
github.com/go-sql-driver/mysql v1.4.1
github.com/go-swagger/go-swagger v0.20.1
github.com/go-xorm/xorm v0.7.7-0.20190822154023-17592d96b35b
github.com/gobwas/glob v0.2.3
github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561
github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14
github.com/google/go-github/v24 v24.0.1

@ -236,6 +236,8 @@ github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:56xuuq
github.com/go-xorm/xorm v0.7.6/go.mod h1:nqz2TAsuOHWH2yk4FYWtacCGgdbrcdZ5mF1XadqEHls=
github.com/go-xorm/xorm v0.7.7-0.20190822154023-17592d96b35b h1:Y0hWUheXDHpIs7BWtJcykO4d1VOsVDKg1PsP5YJwxxM=
github.com/go-xorm/xorm v0.7.7-0.20190822154023-17592d96b35b/go.mod h1:nqz2TAsuOHWH2yk4FYWtacCGgdbrcdZ5mF1XadqEHls=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561 h1:deE7ritpK04PgtpyVOS2TYcQEld9qLCD5b5EbVNOuLA=
github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561/go.mod h1:YgYOrVn3Nj9Tq0EvjmFbphRytDj7JNRoWSStJZWDJTQ=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

@ -22,3 +22,10 @@
content_type: 1 # json
events: '{"push_only":false,"send_everything":false,"choose_events":false,"events":{"create":false,"push":true,"pull_request":true}}'
is_active: true
-
id: 4
repo_id: 2
url: www.example.com/url4
content_type: 1 # json
events: '{"push_only":true,"branch_filter":"{master,feature*}"}'
is_active: true

@ -19,12 +19,14 @@ import (
"strings"
"time"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/sync"
"code.gitea.io/gitea/modules/timeutil"
"github.com/gobwas/glob"
gouuid "github.com/satori/go.uuid"
"github.com/unknwon/com"
)
@ -84,9 +86,10 @@ type HookEvents struct {
// HookEvent represents events that will delivery hook.
type HookEvent struct {
PushOnly bool `json:"push_only"`
SendEverything bool `json:"send_everything"`
ChooseEvents bool `json:"choose_events"`
PushOnly bool `json:"push_only"`
SendEverything bool `json:"send_everything"`
ChooseEvents bool `json:"choose_events"`
BranchFilter string `json:"branch_filter"`
HookEvents `json:"events"`
}
@ -256,6 +259,21 @@ func (w *Webhook) EventsArray() []string {
return events
}
func (w *Webhook) checkBranch(branch string) bool {
if w.BranchFilter == "" || w.BranchFilter == "*" {
return true
}
g, err := glob.Compile(w.BranchFilter)
if err != nil {
// should not really happen as BranchFilter is validated
log.Error("CheckBranch failed: %s", err)
return false
}
return g.Match(branch)
}
// CreateWebhook creates a new web hook.
func CreateWebhook(w *Webhook) error {
return createWebhook(x, w)
@ -651,6 +669,25 @@ func PrepareWebhook(w *Webhook, repo *Repository, event HookEventType, p api.Pay
return prepareWebhook(x, w, repo, event, p)
}
// getPayloadBranch returns branch for hook event, if applicable.
func getPayloadBranch(p api.Payloader) string {
switch pp := p.(type) {
case *api.CreatePayload:
if pp.RefType == "branch" {
return pp.Ref
}
case *api.DeletePayload:
if pp.RefType == "branch" {
return pp.Ref
}
case *api.PushPayload:
if strings.HasPrefix(pp.Ref, git.BranchPrefix) {
return pp.Ref[len(git.BranchPrefix):]
}
}
return ""
}
func prepareWebhook(e Engine, w *Webhook, repo *Repository, event HookEventType, p api.Payloader) error {
for _, e := range w.eventCheckers() {
if event == e.typ {
@ -660,6 +697,15 @@ func prepareWebhook(e Engine, w *Webhook, repo *Repository, event HookEventType,
}
}
// If payload has no associated branch (e.g. it's a new tag, issue, etc.),
// branch filter has no effect.
if branch := getPayloadBranch(p); branch != "" {
if !w.checkBranch(branch) {
log.Info("Branch %q doesn't match branch filter %q, skipping", branch, w.BranchFilter)
return nil
}
}
var payloader api.Payloader
var err error
// Use separate objects so modifications won't be made on payload on non-Gogs/Gitea type hooks.

@ -270,6 +270,40 @@ func TestPrepareWebhooks(t *testing.T) {
}
}
func TestPrepareWebhooksBranchFilterMatch(t *testing.T) {
assert.NoError(t, PrepareTestDatabase())
repo := AssertExistsAndLoadBean(t, &Repository{ID: 2}).(*Repository)
hookTasks := []*HookTask{
{RepoID: repo.ID, HookID: 4, EventType: HookEventPush},
}
for _, hookTask := range hookTasks {
AssertNotExistsBean(t, hookTask)
}
// this test also ensures that * doesn't handle / in any special way (like shell would)
assert.NoError(t, PrepareWebhooks(repo, HookEventPush, &api.PushPayload{Ref: "refs/heads/feature/7791"}))
for _, hookTask := range hookTasks {
AssertExistsAndLoadBean(t, hookTask)
}
}
func TestPrepareWebhooksBranchFilterNoMatch(t *testing.T) {
assert.NoError(t, PrepareTestDatabase())
repo := AssertExistsAndLoadBean(t, &Repository{ID: 2}).(*Repository)
hookTasks := []*HookTask{
{RepoID: repo.ID, HookID: 4, EventType: HookEventPush},
}
for _, hookTask := range hookTasks {
AssertNotExistsBean(t, hookTask)
}
assert.NoError(t, PrepareWebhooks(repo, HookEventPush, &api.PushPayload{Ref: "refs/heads/fix_weird_bug"}))
for _, hookTask := range hookTasks {
AssertNotExistsBean(t, hookTask)
}
}
// TODO TestHookTask_deliver
// TODO TestDeliverHooks

@ -310,6 +310,10 @@ func validate(errs binding.Errors, data map[string]interface{}, f Form, l macaro
}
data["HasError"] = true
// If the field with name errs[0].FieldNames[0] is not found in form
// somehow, some code later on will panic on Data["ErrorMsg"].(string).
// So initialize it to some default.
data["ErrorMsg"] = l.Tr("form.unknown_error")
AssignForm(f, data)
typ := reflect.TypeOf(f)
@ -320,16 +324,9 @@ func validate(errs binding.Errors, data map[string]interface{}, f Form, l macaro
val = val.Elem()
}
for i := 0; i < typ.NumField(); i++ {
field := typ.Field(i)
if field, ok := typ.FieldByName(errs[0].FieldNames[0]); ok {
fieldName := field.Tag.Get("form")
// Allow ignored fields in the struct
if fieldName == "-" {
continue
}
if errs[0].FieldNames[0] == field.Name {
if fieldName != "-" {
data["Err_"+field.Name] = true
trName := field.Tag.Get("locale")
@ -360,6 +357,8 @@ func validate(errs binding.Errors, data map[string]interface{}, f Form, l macaro
data["ErrorMsg"] = trName + l.Tr("form.url_error")
case binding.ERR_INCLUDE:
data["ErrorMsg"] = trName + l.Tr("form.include_error", GetInclude(field))
case validation.ErrGlobPattern:
data["ErrorMsg"] = trName + l.Tr("form.glob_pattern_error", errs[0].Message)
default:
data["ErrorMsg"] = l.Tr("form.unknown_error") + " " + errs[0].Classification
}

@ -185,6 +185,7 @@ type WebhookForm struct {
PullRequest bool
Repository bool
Active bool
BranchFilter string `binding:"GlobPattern"`
}
// PushOnly if the hook will be triggered when push

@ -40,17 +40,19 @@ type CreateHookOption struct {
// enum: gitea,gogs,slack,discord
Type string `json:"type" binding:"Required"`
// required: true
Config map[string]string `json:"config" binding:"Required"`
Events []string `json:"events"`
Config map[string]string `json:"config" binding:"Required"`
Events []string `json:"events"`
BranchFilter string `json:"branch_filter" binding:"GlobPattern"`
// default: false
Active bool `json:"active"`
}
// EditHookOption options when modify one hook
type EditHookOption struct {
Config map[string]string `json:"config"`
Events []string `json:"events"`
Active *bool `json:"active"`
Config map[string]string `json:"config"`
Events []string `json:"events"`
BranchFilter string `json:"branch_filter" binding:"GlobPattern"`
Active *bool `json:"active"`
}
// Payloader payload is some part of one hook

@ -10,11 +10,15 @@ import (
"strings"
"gitea.com/macaron/binding"
"github.com/gobwas/glob"
)
const (
// ErrGitRefName is git reference name error
ErrGitRefName = "GitRefNameError"
// ErrGlobPattern is returned when glob pattern is invalid
ErrGlobPattern = "GlobPattern"
)
var (
@ -28,6 +32,7 @@ var (
func AddBindingRules() {
addGitRefNameBindingRule()
addValidURLBindingRule()
addGlobPatternRule()
}
func addGitRefNameBindingRule() {
@ -82,6 +87,26 @@ func addValidURLBindingRule() {
})
}
func addGlobPatternRule() {
binding.AddRule(&binding.Rule{
IsMatch: func(rule string) bool {
return rule == "GlobPattern"
},
IsValid: func(errs binding.Errors, name string, val interface{}) (bool, binding.Errors) {
str := fmt.Sprintf("%v", val)
if len(str) != 0 {
if _, err := glob.Compile(str); err != nil {
errs.Add([]string{name}, ErrGlobPattern, err.Error())
return false, errs
}
}
return true, errs
},
})
}
func portOnly(hostport string) string {
colon := strings.IndexByte(hostport, ':')
if colon == -1 {

@ -26,8 +26,9 @@ type (
}
TestForm struct {
BranchName string `form:"BranchName" binding:"GitRefName"`
URL string `form:"ValidUrl" binding:"ValidUrl"`
BranchName string `form:"BranchName" binding:"GitRefName"`
URL string `form:"ValidUrl" binding:"ValidUrl"`
GlobPattern string `form:"GlobPattern" binding:"GlobPattern"`
}
)

@ -0,0 +1,62 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package validation
import (
"testing"
"gitea.com/macaron/binding"
"github.com/gobwas/glob"
)
func getGlobPatternErrorString(pattern string) string {
// It would be unwise to rely on that glob
// compilation errors don't ever change.
if _, err := glob.Compile(pattern); err != nil {
return err.Error()
}
return ""
}
var globValidationTestCases = []validationTestCase{
{
description: "Empty glob pattern",
data: TestForm{
GlobPattern: "",
},
expectedErrors: binding.Errors{},
},
{
description: "Valid glob",
data: TestForm{
GlobPattern: "{master,release*}",
},
expectedErrors: binding.Errors{},
},
{
description: "Invalid glob",
data: TestForm{
GlobPattern: "[a-",
},
expectedErrors: binding.Errors{
binding.Error{
FieldNames: []string{"GlobPattern"},
Classification: ErrGlobPattern,
Message: getGlobPatternErrorString("[a-"),
},
},
},
}
func Test_GlobPatternValidation(t *testing.T) {
AddBindingRules()
for _, testCase := range globValidationTestCases {
t.Run(testCase.description, func(t *testing.T) {
performValidationTest(t, testCase)
})
}
}

@ -300,6 +300,7 @@ max_size_error = ` must contain at most %s characters.`
email_error = ` is not a valid email address.`
url_error = ` is not a valid URL.`
include_error = ` must contain substring '%s'.`
glob_pattern_error = ` glob pattern is invalid: %s.`
unknown_error = Unknown error:
captcha_incorrect = The CAPTCHA code is incorrect.
password_not_match = The passwords do not match.
@ -1258,6 +1259,8 @@ settings.event_pull_request = Pull Request
settings.event_pull_request_desc = Pull request opened, closed, reopened, edited, approved, rejected, review comment, assigned, unassigned, label updated, label cleared or synchronized.
settings.event_push = Push
settings.event_push_desc = Git push to a repository.
settings.branch_filter = Branch filter
settings.branch_filter_desc = Branch whitelist for push, branch creation and branch deletion events, specified as glob pattern. If empty or <code>*</code>, events for all branches are reported. See <a href="https://godoc.org/github.com/gobwas/glob#Compile">github.com/gobwas/glob</a> documentation for syntax. Examples: <code>master</code>, <code>{master,release*}</code>.
settings.event_repository = Repository
settings.event_repository_desc = Repository created or deleted.
settings.active = Active

@ -112,6 +112,7 @@ func addHook(ctx *context.APIContext, form *api.CreateHookOption, orgID, repoID
Repository: com.IsSliceContainsStr(form.Events, string(models.HookEventRepository)),
Release: com.IsSliceContainsStr(form.Events, string(models.HookEventRelease)),
},
BranchFilter: form.BranchFilter,
},
IsActive: form.Active,
HookTaskType: models.ToHookTaskType(form.Type),
@ -236,6 +237,7 @@ func editHook(ctx *context.APIContext, form *api.EditHookOption, w *models.Webho
w.PullRequest = com.IsSliceContainsStr(form.Events, string(models.HookEventPullRequest))
w.Repository = com.IsSliceContainsStr(form.Events, string(models.HookEventRepository))
w.Release = com.IsSliceContainsStr(form.Events, string(models.HookEventRelease))
w.BranchFilter = form.BranchFilter
if err := w.UpdateEvent(); err != nil {
ctx.Error(500, "UpdateEvent", err)

@ -145,6 +145,7 @@ func ParseHookEvent(form auth.WebhookForm) *models.HookEvent {
PullRequest: form.PullRequest,
Repository: form.Repository,
},
BranchFilter: form.BranchFilter,
}
}

@ -116,6 +116,13 @@
</div>
</div>
<!-- Branch filter -->
<div class="field">
<label for="branch_filter">{{.i18n.Tr "repo.settings.branch_filter"}}</label>
<input name="branch_filter" type="text" tabindex="0" value="{{or .Webhook.BranchFilter "*"}}">
<span class="help">{{.i18n.Tr "repo.settings.branch_filter_desc" | Str2html}}</span>
</div>
<div class="ui divider"></div>
<div class="inline field">

@ -7528,6 +7528,10 @@
"default": false,
"x-go-name": "Active"
},
"branch_filter": {
"type": "string",
"x-go-name": "BranchFilter"
},
"config": {
"type": "object",
"additionalProperties": {
@ -8124,6 +8128,10 @@
"type": "boolean",
"x-go-name": "Active"
},
"branch_filter": {
"type": "string",
"x-go-name": "BranchFilter"
},
"config": {
"type": "object",
"additionalProperties": {

@ -0,0 +1,8 @@
glob.iml
.idea
*.cpu
*.mem
*.test
*.dot
*.png
*.svg

@ -0,0 +1,9 @@
sudo: false
language: go
go:
- 1.5.3
script:
- go test -v ./...

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 Sergey Kamardin
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

@ -0,0 +1,26 @@
#! /bin/bash
bench() {
filename="/tmp/$1-$2.bench"
if test -e "${filename}";
then
echo "Already exists ${filename}"
else
backup=`git rev-parse --abbrev-ref HEAD`
git checkout $1
echo -n "Creating ${filename}... "
go test ./... -run=NONE -bench=$2 > "${filename}" -benchmem
echo "OK"
git checkout ${backup}
sleep 5
fi
}
to=$1
current=`git rev-parse --abbrev-ref HEAD`
bench ${to} $2
bench ${current} $2
benchcmp $3 "/tmp/${to}-$2.bench" "/tmp/${current}-$2.bench"

@ -0,0 +1,525 @@
package compiler
// TODO use constructor with all matchers, and to their structs private
// TODO glue multiple Text nodes (like after QuoteMeta)
import (
"fmt"
"reflect"
"github.com/gobwas/glob/match"
"github.com/gobwas/glob/syntax/ast"
"github.com/gobwas/glob/util/runes"
)
func optimizeMatcher(matcher match.Matcher) match.Matcher {
switch m := matcher.(type) {
case match.Any:
if len(m.Separators) == 0 {
return match.NewSuper()
}
case match.AnyOf:
if len(m.Matchers) == 1 {
return m.Matchers[0]
}
return m
case match.List:
if m.Not == false && len(m.List) == 1 {
return match.NewText(string(m.List))
}
return m
case match.BTree:
m.Left = optimizeMatcher(m.Left)
m.Right = optimizeMatcher(m.Right)
r, ok := m.Value.(match.Text)
if !ok {
return m
}
var (
leftNil = m.Left == nil
rightNil = m.Right == nil
)
if leftNil && rightNil {
return match.NewText(r.Str)
}
_, leftSuper := m.Left.(match.Super)
lp, leftPrefix := m.Left.(match.Prefix)
la, leftAny := m.Left.(match.Any)
_, rightSuper := m.Right.(match.Super)
rs, rightSuffix := m.Right.(match.Suffix)
ra, rightAny := m.Right.(match.Any)
switch {
case leftSuper && rightSuper:
return match.NewContains(r.Str, false)
case leftSuper && rightNil:
return match.NewSuffix(r.Str)
case rightSuper && leftNil:
return match.NewPrefix(r.Str)
case leftNil && rightSuffix:
return match.NewPrefixSuffix(r.Str, rs.Suffix)
case rightNil && leftPrefix:
return match.NewPrefixSuffix(lp.Prefix, r.Str)
case rightNil && leftAny:
return match.NewSuffixAny(r.Str, la.Separators)
case leftNil && rightAny:
return match.NewPrefixAny(r.Str, ra.Separators)
}
return m
}
return matcher
}
func compileMatchers(matchers []match.Matcher) (match.Matcher, error) {
if len(matchers) == 0 {
return nil, fmt.Errorf("compile error: need at least one matcher")
}
if len(matchers) == 1 {
return matchers[0], nil
}
if m := glueMatchers(matchers); m != nil {
return m, nil
}
idx := -1
maxLen := -1
var val match.Matcher
for i, matcher := range matchers {
if l := matcher.Len(); l != -1 && l >= maxLen {
maxLen = l
idx = i
val = matcher
}
}
if val == nil { // not found matcher with static length
r, err := compileMatchers(matchers[1:])
if err != nil {
return nil, err
}
return match.NewBTree(matchers[0], nil, r), nil
}
left := matchers[:idx]
var right []match.Matcher
if len(matchers) > idx+1 {
right = matchers[idx+1:]
}
var l, r match.Matcher
var err error
if len(left) > 0 {
l, err = compileMatchers(left)
if err != nil {
return nil, err
}
}
if len(right) > 0 {
r, err = compileMatchers(right)
if err != nil {
return nil, err
}
}
return match.NewBTree(val, l, r), nil
}
func glueMatchers(matchers []match.Matcher) match.Matcher {
if m := glueMatchersAsEvery(matchers); m != nil {
return m
}
if m := glueMatchersAsRow(matchers); m != nil {
return m
}
return nil
}
func glueMatchersAsRow(matchers []match.Matcher) match.Matcher {
if len(matchers) <= 1 {
return nil
}
var (
c []match.Matcher
l int
)
for _, matcher := range matchers {
if ml := matcher.Len(); ml == -1 {
return nil
} else {
c = append(c, matcher)
l += ml
}
}
return match.NewRow(l, c...)
}
func glueMatchersAsEvery(matchers []match.Matcher) match.Matcher {
if len(matchers) <= 1 {
return nil
}
var (
hasAny bool
hasSuper bool
hasSingle bool
min int
separator []rune
)
for i, matcher := range matchers {
var sep []rune
switch m := matcher.(type) {
case match.Super:
sep = []rune{}
hasSuper = true
case match.Any:
sep = m.Separators
hasAny = true
case match.Single:
sep = m.Separators
hasSingle = true
min++
case match.List:
if !m.Not {
return nil
}
sep = m.List
hasSingle = true
min++
default:
return nil
}
// initialize
if i == 0 {
separator = sep
}
if runes.Equal(sep, separator) {
continue
}
return nil
}
if hasSuper && !hasAny && !hasSingle {
return match.NewSuper()
}
if hasAny && !hasSuper && !hasSingle {
return match.NewAny(separator)
}
if (hasAny || hasSuper) && min > 0 && len(separator) == 0 {
return match.NewMin(min)
}
every := match.NewEveryOf()
if min > 0 {
every.Add(match.NewMin(min))
if !hasAny && !hasSuper {
every.Add(match.NewMax(min))
}
}
if len(separator) > 0 {
every.Add(match.NewContains(string(separator), true))
}
return every
}
func minimizeMatchers(matchers []match.Matcher) []match.Matcher {
var done match.Matcher
var left, right, count int
for l := 0; l < len(matchers); l++ {
for r := len(matchers); r > l; r-- {
if glued := glueMatchers(matchers[l:r]); glued != nil {
var swap bool
if done == nil {
swap = true
} else {
cl, gl := done.Len(), glued.Len()
swap = cl > -1 && gl > -1 && gl > cl
swap = swap || count < r-l
}
if swap {
done = glued
left = l
right = r
count = r - l
}
}
}
}
if done == nil {
return matchers
}
next := append(append([]match.Matcher{}, matchers[:left]...), done)
if right < len(matchers) {
next = append(next, matchers[right:]...)
}
if len(next) == len(matchers) {
return next
}
return minimizeMatchers(next)
}
// minimizeAnyOf tries to apply some heuristics to minimize number of nodes in given tree
func minimizeTree(tree *ast.Node) *ast.Node {
switch tree.Kind {
case ast.KindAnyOf:
return minimizeTreeAnyOf(tree)
default:
return nil
}
}
// minimizeAnyOf tries to find common children of given node of AnyOf pattern
// it searches for common children from left and from right
// if any common children are found – then it returns new optimized ast tree
// else it returns nil
func minimizeTreeAnyOf(tree *ast.Node) *ast.Node {
if !areOfSameKind(tree.Children, ast.KindPattern) {
return nil
}
commonLeft, commonRight := commonChildren(tree.Children)
commonLeftCount, commonRightCount := len(commonLeft), len(commonRight)
if commonLeftCount == 0 && commonRightCount == 0 { // there are no common parts
return nil
}
var result []*ast.Node
if commonLeftCount > 0 {
result = append(result, ast.NewNode(ast.KindPattern, nil, commonLeft...))
}
var anyOf []*ast.Node
for _, child := range tree.Children {
reuse := child.Children[commonLeftCount : len(child.Children)-commonRightCount]
var node *ast.Node
if len(reuse) == 0 {
// this pattern is completely reduced by commonLeft and commonRight patterns
// so it become nothing
node = ast.NewNode(ast.KindNothing, nil)
} else {
node = ast.NewNode(ast.KindPattern, nil, reuse...)
}
anyOf = appendIfUnique(anyOf, node)
}
switch {
case len(anyOf) == 1 && anyOf[0].Kind != ast.KindNothing:
result = append(result, anyOf[0])
case len(anyOf) > 1:
result = append(result, ast.NewNode(ast.KindAnyOf, nil, anyOf...))
}
if commonRightCount > 0 {
result = append(result, ast.NewNode(ast.KindPattern, nil, commonRight...))
}
return ast.NewNode(ast.KindPattern, nil, result...)
}
func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) {
if len(nodes) <= 1 {
return
}
// find node that has least number of children
idx := leastChildren(nodes)
if idx == -1 {
return
}
tree := nodes[idx]
treeLength := len(tree.Children)
// allocate max able size for rightCommon slice
// to get ability insert elements in reverse order (from end to start)
// without sorting
commonRight = make([]*ast.Node, treeLength)
lastRight := treeLength // will use this to get results as commonRight[lastRight:]
var (
breakLeft bool
breakRight bool
commonTotal int
)
for i, j := 0, treeLength-1; commonTotal < treeLength && j >= 0 && !(breakLeft && breakRight); i, j = i+1, j-1 {
treeLeft := tree.Children[i]
treeRight := tree.Children[j]
for k := 0; k < len(nodes) && !(breakLeft && breakRight); k++ {
// skip least children node
if k == idx {
continue
}
restLeft := nodes[k].Children[i]
restRight := nodes[k].Children[j+len(nodes[k].Children)-treeLength]
breakLeft = breakLeft || !treeLeft.Equal(restLeft)
// disable searching for right common parts, if left part is already overlapping
breakRight = breakRight || (!breakLeft && j <= i)
breakRight = breakRight || !treeRight.Equal(restRight)
}
if !breakLeft {
commonTotal++
commonLeft = append(commonLeft, treeLeft)
}
if !breakRight {
commonTotal++
lastRight = j
commonRight[j] = treeRight
}
}
commonRight = commonRight[lastRight:]
return
}
func appendIfUnique(target []*ast.Node, val *ast.Node) []*ast.Node {
for _, n := range target {
if reflect.DeepEqual(n, val) {
return target
}
}
return append(target, val)
}
func areOfSameKind(nodes []*ast.Node, kind ast.Kind) bool {
for _, n := range nodes {
if n.Kind != kind {
return false
}
}
return true
}
func leastChildren(nodes []*ast.Node) int {
min := -1
idx := -1
for i, n := range nodes {
if idx == -1 || (len(n.Children) < min) {
min = len(n.Children)
idx = i
}
}
return idx
}
func compileTreeChildren(tree *ast.Node, sep []rune) ([]match.Matcher, error) {
var matchers []match.Matcher
for _, desc := range tree.Children {
m, err := compile(desc, sep)
if err != nil {
return nil, err
}
matchers = append(matchers, optimizeMatcher(m))
}
return matchers, nil
}
func compile(tree *ast.Node, sep []rune) (m match.Matcher, err error) {
switch tree.Kind {
case ast.KindAnyOf:
// todo this could be faster on pattern_alternatives_combine_lite (see glob_test.go)
if n := minimizeTree(tree); n != nil {
return compile(n, sep)
}
matchers, err := compileTreeChildren(tree, sep)
if err != nil {
return nil, err
}
return match.NewAnyOf(matchers...), nil
case ast.KindPattern:
if len(tree.Children) == 0 {
return match.NewNothing(), nil
}
matchers, err := compileTreeChildren(tree, sep)
if err != nil {
return nil, err
}
m, err = compileMatchers(minimizeMatchers(matchers))
if err != nil {
return nil, err
}
case ast.KindAny:
m = match.NewAny(sep)
case ast.KindSuper:
m = match.NewSuper()
case ast.KindSingle:
m = match.NewSingle(sep)
case ast.KindNothing:
m = match.NewNothing()
case ast.KindList:
l := tree.Value.(ast.List)
m = match.NewList([]rune(l.Chars), l.Not)
case ast.KindRange:
r := tree.Value.(ast.Range)
m = match.NewRange(r.Lo, r.Hi, r.Not)
case ast.KindText:
t := tree.Value.(ast.Text)
m = match.NewText(t.Text)
default:
return nil, fmt.Errorf("could not compile tree: unknown node type")
}
return optimizeMatcher(m), nil
}
func Compile(tree *ast.Node, sep []rune) (match.Matcher, error) {
m, err := compile(tree, sep)
if err != nil {
return nil, err
}
return m, nil
}

@ -0,0 +1,80 @@
package glob
import (
"github.com/gobwas/glob/compiler"
"github.com/gobwas/glob/syntax"
)
// Glob represents compiled glob pattern.
type Glob interface {
Match(string) bool
}
// Compile creates Glob for given pattern and strings (if any present after pattern) as separators.
// The pattern syntax is:
//
// pattern:
// { term }
//
// term:
// `*` matches any sequence of non-separator characters
// `**` matches any sequence of characters
// `?` matches any single non-separator character
// `[` [ `!` ] { character-range } `]`
// character class (must be non-empty)
// `{` pattern-list `}`
// pattern alternatives
// c matches character c (c != `*`, `**`, `?`, `\`, `[`, `{`, `}`)
// `\` c matches character c
//
// character-range:
// c matches character c (c != `\\`, `-`, `]`)
// `\` c matches character c
// lo `-` hi matches character c for lo <= c <= hi
//
// pattern-list:
// pattern { `,` pattern }
// comma-separated (without spaces) patterns
//
func Compile(pattern string, separators ...rune) (Glob, error) {
ast, err := syntax.Parse(pattern)
if err != nil {
return nil, err
}
matcher, err := compiler.Compile(ast, separators)
if err != nil {
return nil, err
}
return matcher, nil
}
// MustCompile is the same as Compile, except that if Compile returns error, this will panic
func MustCompile(pattern string, separators ...rune) Glob {
g, err := Compile(pattern, separators...)
if err != nil {
panic(err)
}
return g
}
// QuoteMeta returns a string that quotes all glob pattern meta characters
// inside the argument text; For example, QuoteMeta(`{foo*}`) returns `\[foo\*\]`.
func QuoteMeta(s string) string {
b := make([]byte, 2*len(s))
// a byte loop is correct because all meta characters are ASCII
j := 0
for i := 0; i < len(s); i++ {
if syntax.Special(s[i]) {
b[j] = '\\'
j++
}
b[j] = s[i]
j++
}
return string(b[0:j])
}

@ -0,0 +1,45 @@
package match
import (
"fmt"
"github.com/gobwas/glob/util/strings"
)
type Any struct {
Separators []rune
}
func NewAny(s []rune) Any {
return Any{s}
}
func (self Any) Match(s string) bool {
return strings.IndexAnyRunes(s, self.Separators) == -1
}
func (self Any) Index(s string) (int, []int) {
found := strings.IndexAnyRunes(s, self.Separators)
switch found {
case -1:
case 0:
return 0, segments0
default:
s = s[:found]
}
segments := acquireSegments(len(s))
for i := range s {
segments = append(segments, i)
}
segments = append(segments, len(s))
return 0, segments
}
func (self Any) Len() int {
return lenNo
}
func (self Any) String() string {
return fmt.Sprintf("<any:![%s]>", string(self.Separators))
}

@ -0,0 +1,82 @@
package match
import "fmt"
type AnyOf struct {
Matchers Matchers
}
func NewAnyOf(m ...Matcher) AnyOf {
return AnyOf{Matchers(m)}
}
func (self *AnyOf) Add(m Matcher) error {
self.Matchers = append(self.Matchers, m)
return nil
}
func (self AnyOf) Match(s string) bool {
for _, m := range self.Matchers {
if m.Match(s) {
return true
}
}
return false
}
func (self AnyOf) Index(s string) (int, []int) {
index := -1
segments := acquireSegments(len(s))
for _, m := range self.Matchers {
idx, seg := m.Index(s)
if idx == -1 {
continue
}
if index == -1 || idx < index {
index = idx
segments = append(segments[:0], seg...)
continue
}
if idx > index {
continue
}
// here idx == index
segments = appendMerge(segments, seg)
}
if index == -1 {
releaseSegments(segments)
return -1, nil
}
return index, segments
}
func (self AnyOf) Len() (l int) {
l = -1
for _, m := range self.Matchers {
ml := m.Len()
switch {
case l == -1:
l = ml
continue
case ml == -1:
return -1
case l != ml:
return -1
}
}
return
}
func (self AnyOf) String() string {
return fmt.Sprintf("<any_of:[%s]>", self.Matchers)
}

@ -0,0 +1,146 @@
package match
import (
"fmt"
"unicode/utf8"
)
type BTree struct {
Value Matcher
Left Matcher
Right Matcher
ValueLengthRunes int
LeftLengthRunes int
RightLengthRunes int
LengthRunes int
}
func NewBTree(Value, Left, Right Matcher) (tree BTree) {
tree.Value = Value
tree.Left = Left
tree.Right = Right
lenOk := true
if tree.ValueLengthRunes = Value.Len(); tree.ValueLengthRunes == -1 {
lenOk = false
}
if Left != nil {
if tree.LeftLengthRunes = Left.Len(); tree.LeftLengthRunes == -1 {
lenOk = false
}
}
if Right != nil {
if tree.RightLengthRunes = Right.Len(); tree.RightLengthRunes == -1 {
lenOk = false
}
}
if lenOk {
tree.LengthRunes = tree.LeftLengthRunes + tree.ValueLengthRunes + tree.RightLengthRunes
} else {
tree.LengthRunes = -1
}
return tree
}
func (self BTree) Len() int {
return self.LengthRunes
}
// todo?
func (self BTree) Index(s string) (int, []int) {
return -1, nil
}
func (self BTree) Match(s string) bool {
inputLen := len(s)
// self.Length, self.RLen and self.LLen are values meaning the length of runes for each part
// here we manipulating byte length for better optimizations
// but these checks still works, cause minLen of 1-rune string is 1 byte.
if self.LengthRunes != -1 && self.LengthRunes > inputLen {
return false
}
// try to cut unnecessary parts
// by knowledge of length of right and left part
var offset, limit int
if self.LeftLengthRunes >= 0 {
offset = self.LeftLengthRunes
<