From 599ff1c054e436daa4dc3f049aa8661d9c2395f9 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 1 Nov 2021 16:39:52 +0800 Subject: [PATCH] Only allow webhook to send requests to allowed hosts (#17482) --- cmd/web.go | 4 + custom/conf/app.example.ini | 6 + .../doc/advanced/config-cheat-sheet.en-us.md | 8 ++ modules/hostmatcher/hostmatcher.go | 94 ++++++++++++++ modules/hostmatcher/hostmatcher_test.go | 119 ++++++++++++++++++ modules/migrations/migrate.go | 12 +- modules/setting/webhook.go | 19 +-- modules/util/net.go | 19 +++ services/webhook/deliver.go | 26 +++- 9 files changed, 284 insertions(+), 23 deletions(-) create mode 100644 modules/hostmatcher/hostmatcher.go create mode 100644 modules/hostmatcher/hostmatcher_test.go create mode 100644 modules/util/net.go diff --git a/cmd/web.go b/cmd/web.go index 963c816207..8d9387e06f 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -194,6 +194,10 @@ func listen(m http.Handler, handleRedirector bool) error { listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort) } log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL) + // This can be useful for users, many users do wrong to their config and get strange behaviors behind a reverse-proxy. + // A user may fix the configuration mistake when he sees this log. + // And this is also very helpful to maintainers to provide help to users to resolve their configuration problems. + log.Info("AppURL(ROOT_URL): %s", setting.AppURL) if setting.LFS.StartServer { log.Info("LFS server enabled") diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 1753ed2330..eadc1c0d96 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1396,6 +1396,12 @@ PATH = ;; Deliver timeout in seconds ;DELIVER_TIMEOUT = 5 ;; +;; Webhook can only call allowed hosts for security reasons. Comma separated list, eg: external, 192.168.1.0/24, *.mydomain.com +;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), * (for all hosts) +;; CIDR list: 1.2.3.0/8, 2001:db8::/32 +;; Wildcard hosts: *.mydomain.com, 192.168.100.* +;ALLOWED_HOST_LIST = external +;; ;; Allow insecure certification ;SKIP_TLS_VERIFY = false ;; diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 91c62dbec3..6cc6043cae 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -581,6 +581,14 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type - `QUEUE_LENGTH`: **1000**: Hook task queue length. Use caution when editing this value. - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks. +- `ALLOWED_HOST_LIST`: **external**: Webhook can only call allowed hosts for security reasons. Comma separated list. + - Built-in networks: + - `loopback`: 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. + - `private`: RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. + - `external`: A valid non-private unicast IP, you can access all hosts on public internet. + - `*`: All hosts are allowed. + - CIDR list: `1.2.3.0/8` for IPv4 and `2001:db8::/32` for IPv6 + - Wildcard hosts: `*.mydomain.com`, `192.168.100.*` - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification. - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page. - `PROXY_URL`: **\**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting. diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go new file mode 100644 index 0000000000..f8a787c575 --- /dev/null +++ b/modules/hostmatcher/hostmatcher.go @@ -0,0 +1,94 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package hostmatcher + +import ( + "net" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +// HostMatchList is used to check if a host or IP is in a list. +// If you only need to do wildcard matching, consider to use modules/matchlist +type HostMatchList struct { + hosts []string + ipNets []*net.IPNet +} + +// MatchBuiltinAll all hosts are matched +const MatchBuiltinAll = "*" + +// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched +const MatchBuiltinExternal = "external" + +// MatchBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. +const MatchBuiltinPrivate = "private" + +// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. +const MatchBuiltinLoopback = "loopback" + +// ParseHostMatchList parses the host list HostMatchList +func ParseHostMatchList(hostList string) *HostMatchList { + hl := &HostMatchList{} + for _, s := range strings.Split(hostList, ",") { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { + continue + } + _, ipNet, err := net.ParseCIDR(s) + if err == nil { + hl.ipNets = append(hl.ipNets, ipNet) + } else { + hl.hosts = append(hl.hosts, s) + } + } + return hl +} + +// MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list +func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool { + var matched bool + host = strings.ToLower(host) + ipStr := ip.String() +loop: + for _, hostInList := range hl.hosts { + switch hostInList { + case "": + continue + case MatchBuiltinAll: + matched = true + break loop + case MatchBuiltinExternal: + if matched = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); matched { + break loop + } + case MatchBuiltinPrivate: + if matched = util.IsIPPrivate(ip); matched { + break loop + } + case MatchBuiltinLoopback: + if matched = ip.IsLoopback(); matched { + break loop + } + default: + if matched, _ = filepath.Match(hostInList, host); matched { + break loop + } + if matched, _ = filepath.Match(hostInList, ipStr); matched { + break loop + } + } + } + if !matched { + for _, ipNet := range hl.ipNets { + if matched = ipNet.Contains(ip); matched { + break + } + } + } + return matched +} diff --git a/modules/hostmatcher/hostmatcher_test.go b/modules/hostmatcher/hostmatcher_test.go new file mode 100644 index 0000000000..8eaafbdbc8 --- /dev/null +++ b/modules/hostmatcher/hostmatcher_test.go @@ -0,0 +1,119 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package hostmatcher + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHostOrIPMatchesList(t *testing.T) { + type tc struct { + host string + ip net.IP + expected bool + } + + // for IPv6: "::1" is loopback, "fd00::/8" is private + + hl := ParseHostMatchList("private, External, *.myDomain.com, 169.254.1.0/24") + cases := []tc{ + {"", net.IPv4zero, false}, + {"", net.IPv6zero, false}, + + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("::1"), false}, + + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("fd00::1"), true}, + + {"", net.ParseIP("8.8.8.8"), true}, + {"", net.ParseIP("1001::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + {"sub.mydomain.com", net.IPv4zero, true}, + + {"", net.ParseIP("169.254.1.1"), true}, + {"", net.ParseIP("169.254.2.2"), false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + hl = ParseHostMatchList("loopback") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + hl = ParseHostMatchList("private") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + hl = ParseHostMatchList("external") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + hl = ParseHostMatchList("*") + cases = []tc{ + {"", net.IPv4zero, true}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, true}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + } +} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index c5d78fba73..dbe69259f4 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -89,7 +89,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error { return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true} } for _, addr := range addrList { - if isIPPrivate(addr) || !addr.IsGlobalUnicast() { + if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() { return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true} } } @@ -474,13 +474,3 @@ func Init() error { return nil } - -// TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17 -func isIPPrivate(ip net.IP) bool { - if ip4 := ip.To4(); ip4 != nil { - return ip4[0] == 10 || - (ip4[0] == 172 && ip4[1]&0xf0 == 16) || - (ip4[0] == 192 && ip4[1] == 168) - } - return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc -} diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index 8ef54f5cbe..acd5bd0455 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -7,20 +7,22 @@ package setting import ( "net/url" + "code.gitea.io/gitea/modules/hostmatcher" "code.gitea.io/gitea/modules/log" ) var ( // Webhook settings Webhook = struct { - QueueLength int - DeliverTimeout int - SkipTLSVerify bool - Types []string - PagingNum int - ProxyURL string - ProxyURLFixed *url.URL - ProxyHosts []string + QueueLength int + DeliverTimeout int + SkipTLSVerify bool + AllowedHostList *hostmatcher.HostMatchList + Types []string + PagingNum int + ProxyURL string + ProxyURLFixed *url.URL + ProxyHosts []string }{ QueueLength: 1000, DeliverTimeout: 5, @@ -36,6 +38,7 @@ func newWebhookService() { Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5) Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool() + Webhook.AllowedHostList = hostmatcher.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(hostmatcher.MatchBuiltinExternal)) Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork"} Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10) Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("") diff --git a/modules/util/net.go b/modules/util/net.go new file mode 100644 index 0000000000..54c0a2ca39 --- /dev/null +++ b/modules/util/net.go @@ -0,0 +1,19 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "net" +) + +// IsIPPrivate for net.IP.IsPrivate. TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17 +func IsIPPrivate(ip net.IP) bool { + if ip4 := ip.To4(); ip4 != nil { + return ip4[0] == 10 || + (ip4[0] == 172 && ip4[1]&0xf0 == 16) || + (ip4[0] == 192 && ip4[1] == 168) + } + return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc +} diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index 28c3b23b2f..04cec4c1c4 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -19,6 +19,7 @@ import ( "strconv" "strings" "sync" + "syscall" "time" "code.gitea.io/gitea/models" @@ -29,6 +30,8 @@ import ( "github.com/gobwas/glob" ) +var contextKeyWebhookRequest interface{} = "contextKeyWebhookRequest" + // Deliver deliver hook task func Deliver(t *models.HookTask) error { w, err := models.GetWebhookByID(t.HookID) @@ -171,7 +174,7 @@ func Deliver(t *models.HookTask) error { return fmt.Errorf("Webhook task skipped (webhooks disabled): [%d]", t.ID) } - resp, err := webhookHTTPClient.Do(req) + resp, err := webhookHTTPClient.Do(req.WithContext(context.WithValue(req.Context(), contextKeyWebhookRequest, req))) if err != nil { t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err) return err @@ -293,14 +296,29 @@ func InitDeliverHooks() { timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second webhookHTTPClient = &http.Client{ + Timeout: timeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}, Proxy: webhookProxy(), - Dial: func(netw, addr string) (net.Conn, error) { - return net.DialTimeout(netw, addr, timeout) // dial timeout + DialContext: func(ctx context.Context, network, addrOrHost string) (net.Conn, error) { + dialer := net.Dialer{ + Timeout: timeout, + Control: func(network, ipAddr string, c syscall.RawConn) error { + // in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here + tcpAddr, err := net.ResolveTCPAddr(network, ipAddr) + req := ctx.Value(contextKeyWebhookRequest).(*http.Request) + if err != nil { + return fmt.Errorf("webhook can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", req.Host, network, ipAddr, err) + } + if !setting.Webhook.AllowedHostList.MatchesHostOrIP(req.Host, tcpAddr.IP) { + return fmt.Errorf("webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny '%s(%s)'", req.Host, ipAddr) + } + return nil + }, + } + return dialer.DialContext(ctx, network, addrOrHost) }, }, - Timeout: timeout, // request timeout } go graceful.GetManager().RunWithShutdownContext(DeliverHooks)