From 2b0b629b101561adba37fc8d34c9796fde9d5853 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 31 Oct 2019 11:37:48 +0800 Subject: [PATCH 1/5] Webhook support custom proxy --- custom/conf/app.ini.sample | 4 ++++ modules/setting/webhook.go | 21 +++++++++++++++++++++ modules/webhook/deliver.go | 18 ++++++++++++++++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 33cd0506ed469..9e8520e9c8be9 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -511,6 +511,10 @@ DELIVER_TIMEOUT = 5 SKIP_TLS_VERIFY = false ; Number of history information in each page PAGING_NUM = 10 +; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy +PROXY_URL = +; All request domains needed to proxy, +PROXY_DOMAINS = [mailer] ENABLED = false diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index b0e7d66ad2804..4a953616f1524 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -4,6 +4,12 @@ package setting +import ( + "net/url" + + "code.gitea.io/gitea/modules/log" +) + var ( // Webhook settings Webhook = struct { @@ -12,11 +18,16 @@ var ( SkipTLSVerify bool Types []string PagingNum int + ProxyURL string + ProxyURLFixed *url.URL + ProxyHosts []string }{ QueueLength: 1000, DeliverTimeout: 5, SkipTLSVerify: false, PagingNum: 10, + ProxyURL: "", + ProxyHosts: []string{}, } ) @@ -27,4 +38,14 @@ func newWebhookService() { Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool() Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams"} Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10) + Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("") + if Webhook.ProxyURL != "" { + var err error + Webhook.ProxyURLFixed, err = url.Parse(Webhook.ProxyURL) + if err != nil { + log.Error("Webhook PROXY_URL is not valid") + Webhook.ProxyURL = "" + } + } + Webhook.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",") } diff --git a/modules/webhook/deliver.go b/modules/webhook/deliver.go index 54f20171fad60..2f4263294554e 100644 --- a/modules/webhook/deliver.go +++ b/modules/webhook/deliver.go @@ -184,6 +184,21 @@ func DeliverHooks() { var webhookHTTPClient *http.Client +func webhookProxy() func(req *http.Request) (*url.URL, error) { + if setting.Webhook.ProxyURL == "" { + return http.ProxyFromEnvironment + } + + return func(req *http.Request) (*url.URL, error) { + for _, v := range setting.Webhook.ProxyHosts { + if strings.EqualFold(v, req.URL.Host) { + return http.ProxyURL(setting.Webhook.ProxyURLFixed)(req) + } + } + return http.ProxyFromEnvironment(req) + } +} + // InitDeliverHooks starts the hooks delivery thread func InitDeliverHooks() { timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second @@ -191,7 +206,7 @@ func InitDeliverHooks() { webhookHTTPClient = &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}, - Proxy: http.ProxyFromEnvironment, + Proxy: webhookProxy(), Dial: func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, timeout) if err != nil { @@ -199,7 +214,6 @@ func InitDeliverHooks() { } return conn, conn.SetDeadline(time.Now().Add(timeout)) - }, }, } From ffe3e974b0760da86fb9eb068af132827bfdbaa4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 3 Nov 2019 14:49:03 +0800 Subject: [PATCH 2/5] Add glob support on webhook proxy host rules --- modules/webhook/deliver.go | 26 +++++++++++++++++++--- modules/webhook/deliver_test.go | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 modules/webhook/deliver_test.go diff --git a/modules/webhook/deliver.go b/modules/webhook/deliver.go index 2f4263294554e..2db9c6bb8c980 100644 --- a/modules/webhook/deliver.go +++ b/modules/webhook/deliver.go @@ -12,11 +12,13 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "github.com/gobwas/glob" "github.com/unknwon/com" ) @@ -182,16 +184,34 @@ func DeliverHooks() { } } -var webhookHTTPClient *http.Client +var ( + webhookHTTPClient *http.Client + once sync.Once + hostMatchers []glob.Glob +) func webhookProxy() func(req *http.Request) (*url.URL, error) { if setting.Webhook.ProxyURL == "" { return http.ProxyFromEnvironment } + once.Do(func() { + for _, h := range setting.Webhook.ProxyHosts { + if g, err := glob.Compile(h); err == nil { + hostMatchers = append(hostMatchers, g) + } else { + log.Error("glob.Compile %s failed: %v", h, err) + } + } + }) + return func(req *http.Request) (*url.URL, error) { - for _, v := range setting.Webhook.ProxyHosts { - if strings.EqualFold(v, req.URL.Host) { + if len(setting.Webhook.ProxyHosts) == 1 && setting.Webhook.ProxyHosts[0] == "*" { + return http.ProxyURL(setting.Webhook.ProxyURLFixed)(req) + } + + for _, v := range hostMatchers { + if v.Match(req.URL.Host) { return http.ProxyURL(setting.Webhook.ProxyURLFixed)(req) } } diff --git a/modules/webhook/deliver_test.go b/modules/webhook/deliver_test.go new file mode 100644 index 0000000000000..cfc99d796a536 --- /dev/null +++ b/modules/webhook/deliver_test.go @@ -0,0 +1,39 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package webhook + +import ( + "net/http" + "net/url" + "testing" + + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" +) + +func TestWebhookProxy(t *testing.T) { + setting.Webhook.ProxyURL = "http://localhost:8080" + setting.Webhook.ProxyURLFixed, _ = url.Parse(setting.Webhook.ProxyURL) + setting.Webhook.ProxyHosts = []string{"*.discordapp.com", "discordapp.com"} + + var kases = map[string]string{ + "https://discordapp.com/api/webhooks/xxxxxxxxx/xxxxxxxxxxxxxxxxxxx": "http://localhost:8080", + "http://s.discordapp.com/assets/xxxxxx": "http://localhost:8080", + "http://github.com/a/b": "", + } + + for reqURL, proxyURL := range kases { + req, err := http.NewRequest("POST", reqURL, nil) + assert.NoError(t, err) + + u, err := webhookProxy()(req) + assert.NoError(t, err) + if proxyURL == "" { + assert.Nil(t, u) + } else { + assert.EqualValues(t, proxyURL, u.String()) + } + } +} From fb69cf294427eff461c814a26e090d68b08a65e1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 3 Nov 2019 14:52:26 +0800 Subject: [PATCH 3/5] fix app.ini.sample --- custom/conf/app.ini.sample | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 9e8520e9c8be9..922a5f2e6654c 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -513,8 +513,8 @@ SKIP_TLS_VERIFY = false PAGING_NUM = 10 ; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy PROXY_URL = -; All request domains needed to proxy, -PROXY_DOMAINS = +; All request hosts needed to proxy, you could use * to match the hosts. +PROXY_HOSTS = [mailer] ENABLED = false From 8b7d53640ce1a683ccac6cce9fd5ff76fa430645 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 3 Nov 2019 21:10:12 +0800 Subject: [PATCH 4/5] improve code and app.ini.sample --- custom/conf/app.ini.sample | 2 +- modules/webhook/deliver.go | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 922a5f2e6654c..17fcc0de23fea 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -513,7 +513,7 @@ SKIP_TLS_VERIFY = false PAGING_NUM = 10 ; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy PROXY_URL = -; All request hosts needed to proxy, you could use * to match the hosts. +; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. PROXY_HOSTS = [mailer] diff --git a/modules/webhook/deliver.go b/modules/webhook/deliver.go index 2db9c6bb8c980..b262505cead15 100644 --- a/modules/webhook/deliver.go +++ b/modules/webhook/deliver.go @@ -206,10 +206,6 @@ func webhookProxy() func(req *http.Request) (*url.URL, error) { }) return func(req *http.Request) (*url.URL, error) { - if len(setting.Webhook.ProxyHosts) == 1 && setting.Webhook.ProxyHosts[0] == "*" { - return http.ProxyURL(setting.Webhook.ProxyURLFixed)(req) - } - for _, v := range hostMatchers { if v.Match(req.URL.Host) { return http.ProxyURL(setting.Webhook.ProxyURLFixed)(req) From 6f5eb1fdbdc18ac180165232c353557e62861f95 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 8 Nov 2019 20:16:10 +0800 Subject: [PATCH 5/5] update cheetsheet about added webhook options --- docs/content/doc/advanced/config-cheat-sheet.en-us.md | 2 ++ docs/content/doc/advanced/config-cheat-sheet.zh-cn.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 1e24255d8d786..96b529c0bc158 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -312,6 +312,8 @@ relation to port exhaustion. - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks. - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification. - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page. +- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy +- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. ## Mailer (`mailer`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 53426ed9834d9..b545d9a99d1f3 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -129,6 +129,8 @@ menu: - `DELIVER_TIMEOUT`: 请求webhooks的超时时间，单位秒。 - `SKIP_TLS_VERIFY`: 是否允许不安全的证书。 - `PAGING_NUM`: 每页显示的Webhook 历史数量。 +- `PROXY_URL`: ****: 代理服务器网址，支持 http://, https//, socks://, 为空将使用环境变量中的 http_proxy/https_proxy 设置。 +- `PROXY_HOSTS`: ****: 逗号分隔的需要代理的域名或IP地址。支持 * 号匹配符，使用 ** 匹配所有域名和IP地址。 ## Mailer (`mailer`)