From 20ae1849676ea860286b5e988dcb73f142dc9f3b Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 6 Nov 2021 17:23:43 +0800 Subject: [PATCH] Only allow webhook to send requests to allowed hosts (#17482) (#17510) Backport #17482 * Only allow webhook to send requests to allowed hosts (backport #17482) * use ALLOWED_HOST_LIST=* for default to keep the legacy behavior in 1.15.x --- cmd/web.go | 4 + custom/conf/app.example.ini | 7 ++ .../doc/advanced/config-cheat-sheet.en-us.md | 8 ++ modules/hostmatcher/hostmatcher.go | 94 ++++++++++++++ modules/hostmatcher/hostmatcher_test.go | 119 ++++++++++++++++++ modules/migrations/migrate.go | 15 +-- modules/setting/webhook.go | 19 +-- modules/util/net.go | 19 +++ services/webhook/deliver.go | 26 +++- 9 files changed, 285 insertions(+), 26 deletions(-) create mode 100644 modules/hostmatcher/hostmatcher.go create mode 100644 modules/hostmatcher/hostmatcher_test.go create mode 100644 modules/util/net.go diff --git a/cmd/web.go b/cmd/web.go index 963c816207..8d9387e06f 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -194,6 +194,10 @@ func listen(m http.Handler, handleRedirector bool) error { listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort) } log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL) + // This can be useful for users, many users do wrong to their config and get strange behaviors behind a reverse-proxy. + // A user may fix the configuration mistake when he sees this log. + // And this is also very helpful to maintainers to provide help to users to resolve their configuration problems. + log.Info("AppURL(ROOT_URL): %s", setting.AppURL) if setting.LFS.StartServer { log.Info("LFS server enabled") diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 784ef5df37..0dfae105f9 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1388,6 +1388,13 @@ PATH = ;; Deliver timeout in seconds ;DELIVER_TIMEOUT = 5 ;; +;; Webhook can only call allowed hosts for security reasons. Comma separated list, eg: external, 192.168.1.0/24, *.mydomain.com +;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), * (for all hosts) +;; CIDR list: 1.2.3.0/8, 2001:db8::/32 +;; Wildcard hosts: *.mydomain.com, 192.168.100.* +;; Default to * for 1.15.x, external for 1.16 and later +;ALLOWED_HOST_LIST = * +;; ;; Allow insecure certification ;SKIP_TLS_VERIFY = false ;; diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 42cb29b9aa..a611ab0cd3 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -545,6 +545,14 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type - `QUEUE_LENGTH`: **1000**: Hook task queue length. Use caution when editing this value. - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks. +- `ALLOWED_HOST_LIST`: `*`: Default to `*` for 1.15.x, `external` for 1.16 and later. Webhook can only call allowed hosts for security reasons. Comma separated list. + - Built-in networks: + - `loopback`: 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. + - `private`: RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. + - `external`: A valid non-private unicast IP, you can access all hosts on public internet. + - `*`: All hosts are allowed. + - CIDR list: `1.2.3.0/8` for IPv4 and `2001:db8::/32` for IPv6 + - Wildcard hosts: `*.mydomain.com`, `192.168.100.*` - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification. - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page. - `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go new file mode 100644 index 0000000000..f8a787c575 --- /dev/null +++ b/modules/hostmatcher/hostmatcher.go @@ -0,0 +1,94 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package hostmatcher + +import ( + "net" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +// HostMatchList is used to check if a host or IP is in a list. +// If you only need to do wildcard matching, consider to use modules/matchlist +type HostMatchList struct { + hosts []string + ipNets []*net.IPNet +} + +// MatchBuiltinAll all hosts are matched +const MatchBuiltinAll = "*" + +// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched +const MatchBuiltinExternal = "external" + +// MatchBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. +const MatchBuiltinPrivate = "private" + +// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. +const MatchBuiltinLoopback = "loopback" + +// ParseHostMatchList parses the host list HostMatchList +func ParseHostMatchList(hostList string) *HostMatchList { + hl := &HostMatchList{} + for _, s := range strings.Split(hostList, ",") { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { + continue + } + _, ipNet, err := net.ParseCIDR(s) + if err == nil { + hl.ipNets = append(hl.ipNets, ipNet) + } else { + hl.hosts = append(hl.hosts, s) + } + } + return hl +} + +// MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list +func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool { + var matched bool + host = strings.ToLower(host) + ipStr := ip.String() +loop: + for _, hostInList := range hl.hosts { + switch hostInList { + case "": + continue + case MatchBuiltinAll: + matched = true + break loop + case MatchBuiltinExternal: + if matched = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); matched { + break loop + } + case MatchBuiltinPrivate: + if matched = util.IsIPPrivate(ip); matched { + break loop + } + case MatchBuiltinLoopback: + if matched = ip.IsLoopback(); matched { + break loop + } + default: + if matched, _ = filepath.Match(hostInList, host); matched { + break loop + } + if matched, _ = filepath.Match(hostInList, ipStr); matched { + break loop + } + } + } + if !matched { + for _, ipNet := range hl.ipNets { + if matched = ipNet.Contains(ip); matched { + break + } + } + } + return matched +} diff --git a/modules/hostmatcher/hostmatcher_test.go b/modules/hostmatcher/hostmatcher_test.go new file mode 100644 index 0000000000..8eaafbdbc8 --- /dev/null +++ b/modules/hostmatcher/hostmatcher_test.go @@ -0,0 +1,119 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package hostmatcher + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHostOrIPMatchesList(t *testing.T) { + type tc struct { + host string + ip net.IP + expected bool + } + + // for IPv6: "::1" is loopback, "fd00::/8" is private + + hl := ParseHostMatchList("private, External, *.myDomain.com, 169.254.1.0/24") + cases := []tc{ + {"", net.IPv4zero, false}, + {"", net.IPv6zero, false}, + + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("::1"), false}, + + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("fd00::1"), true}, + + {"", net.ParseIP("8.8.8.8"), true}, + {"", net.ParseIP("1001::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + {"sub.mydomain.com", net.IPv4zero, true}, + + {"", net.ParseIP("169.254.1.1"), true}, + {"", net.ParseIP("169.254.2.2"), false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + hl = ParseHostMatchList("loopback") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + hl = ParseHostMatchList("private") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + hl = ParseHostMatchList("external") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + hl = ParseHostMatchList("*") + cases = []tc{ + {"", net.IPv4zero, true}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, true}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } +} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 0a507d9c33..3d9b03d176 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -89,7 +89,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error { return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true} } for _, addr := range addrList { - if isIPPrivate(addr) || !addr.IsGlobalUnicast() { + if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() { return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true} } } @@ -486,16 +486,3 @@ func Init() error { return nil } - -// isIPPrivate reports whether ip is a private address, according to -// RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses). -// from https://github.com/golang/go/pull/42793 -// TODO remove if https://github.com/golang/go/issues/29146 got resolved -func isIPPrivate(ip net.IP) bool { - if ip4 := ip.To4(); ip4 != nil { - return ip4[0] == 10 || - (ip4[0] == 172 && ip4[1]&0xf0 == 16) || - (ip4[0] == 192 && ip4[1] == 168) - } - return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc -} diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index 4a0c593c8d..e25117c473 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -7,20 +7,22 @@ package setting import ( "net/url" + "code.gitea.io/gitea/modules/hostmatcher" "code.gitea.io/gitea/modules/log" ) var ( // Webhook settings Webhook = struct { - QueueLength int - DeliverTimeout int - SkipTLSVerify bool - Types []string - PagingNum int - ProxyURL string - ProxyURLFixed *url.URL - ProxyHosts []string + QueueLength int + DeliverTimeout int + SkipTLSVerify bool + AllowedHostList *hostmatcher.HostMatchList + Types []string + PagingNum int + ProxyURL string + ProxyURLFixed *url.URL + ProxyHosts []string }{ QueueLength: 1000, DeliverTimeout: 5, @@ -36,6 +38,7 @@ func newWebhookService() { Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5) Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool() + Webhook.AllowedHostList = hostmatcher.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(hostmatcher.MatchBuiltinAll)) Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix"} Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10) Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("") diff --git a/modules/util/net.go b/modules/util/net.go new file mode 100644 index 0000000000..54c0a2ca39 --- /dev/null +++ b/modules/util/net.go @@ -0,0 +1,19 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "net" +) + +// IsIPPrivate for net.IP.IsPrivate. TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17 +func IsIPPrivate(ip net.IP) bool { + if ip4 := ip.To4(); ip4 != nil { + return ip4[0] == 10 || + (ip4[0] == 172 && ip4[1]&0xf0 == 16) || + (ip4[0] == 192 && ip4[1] == 168) + } + return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc +} diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index 8243fde1bb..4fcb167efe 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -20,6 +20,7 @@ import ( "strconv" "strings" "sync" + "syscall" "time" "code.gitea.io/gitea/models" @@ -29,6 +30,8 @@ import ( "github.com/gobwas/glob" ) +var contextKeyWebhookRequest interface{} = "contextKeyWebhookRequest" + // Deliver deliver hook task func Deliver(t *models.HookTask) error { w, err := models.GetWebhookByID(t.HookID) @@ -166,7 +169,7 @@ func Deliver(t *models.HookTask) error { return fmt.Errorf("Webhook task skipped (webhooks disabled): [%d]", t.ID) } - resp, err := webhookHTTPClient.Do(req) + resp, err := webhookHTTPClient.Do(req.WithContext(context.WithValue(req.Context(), contextKeyWebhookRequest, req))) if err != nil { t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err) return err @@ -288,14 +291,29 @@ func InitDeliverHooks() { timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second webhookHTTPClient = &http.Client{ + Timeout: timeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}, Proxy: webhookProxy(), - Dial: func(netw, addr string) (net.Conn, error) { - return net.DialTimeout(netw, addr, timeout) // dial timeout + DialContext: func(ctx context.Context, network, addrOrHost string) (net.Conn, error) { + dialer := net.Dialer{ + Timeout: timeout, + Control: func(network, ipAddr string, c syscall.RawConn) error { + // in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here + tcpAddr, err := net.ResolveTCPAddr(network, ipAddr) + req := ctx.Value(contextKeyWebhookRequest).(*http.Request) + if err != nil { + return fmt.Errorf("webhook can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", req.Host, network, ipAddr, err) + } + if !setting.Webhook.AllowedHostList.MatchesHostOrIP(req.Host, tcpAddr.IP) { + return fmt.Errorf("webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny '%s(%s)'", req.Host, ipAddr) + } + return nil + }, + } + return dialer.DialContext(ctx, network, addrOrHost) }, }, - Timeout: timeout, // request timeout } go graceful.GetManager().RunWithShutdownContext(DeliverHooks)