From 1409281642d3a89f8b38894ab9ef55620f8503f3 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 28 Jun 2022 01:23:58 +0100 Subject: [PATCH 01/14] Share HTML template renderers and create a watcher framework The recovery, API, Web and package frameworks all create their own HTML Renderers. This increases the memory requirements of Gitea unnecessarily with duplicate templates being kept in memory. Further the reloading framework in dev mode for these involves locking and recompiling all of the templates on each load. This will potentially hide concurrency issues and it is inefficient. This PR stores the templates renderer in the context and stores this context in the NormalRoutes, it then creates a fsnotify.Watcher framework to watch files. The watching framework is then extended to the mailer templates which were previously not being reloaded in dev. Then the locales are simplified to a similar structure. Fix #20210, #20211, #20217 Replace #20159 Signed-off-by: Andrew Thornton --- cmd/embedded.go | 2 +- cmd/web.go | 6 +- contrib/pr/checkout.go | 3 +- go.mod | 2 +- integrations/api_activitypub_person_test.go | 12 +- integrations/api_nodeinfo_test.go | 5 +- integrations/create_no_session_test.go | 5 +- integrations/integration_test.go | 2 +- modules/context/context.go | 4 +- modules/context/package.go | 10 +- modules/options/base.go | 34 +++ modules/options/dynamic.go | 16 +- modules/options/static.go | 10 + modules/templates/base.go | 62 ++++-- modules/templates/dynamic.go | 100 +++------ modules/templates/htmlrenderer.go | 51 +++++ modules/templates/mailer.go | 98 +++++++++ modules/templates/static.go | 104 +++------- modules/timeutil/since_test.go | 3 +- modules/translation/i18n/errors.go | 12 ++ modules/translation/i18n/format.go | 42 ++++ modules/translation/i18n/i18n.go | 219 +++----------------- modules/translation/i18n/i18n_test.go | 46 ++-- modules/translation/i18n/localestore.go | 161 ++++++++++++++ modules/translation/translation.go | 99 +++++---- modules/watcher/watcher.go | 104 ++++++++++ routers/api/packages/api.go | 9 +- routers/api/packages/pypi/pypi.go | 2 - routers/api/v1/misc/markdown_test.go | 2 +- routers/init.go | 12 +- routers/install/install.go | 65 +++--- routers/install/routes.go | 9 +- routers/install/routes_test.go | 5 +- routers/install/setting.go | 2 +- routers/web/base.go | 5 +- routers/web/web.go | 10 +- 36 files changed, 837 insertions(+), 496 deletions(-) create mode 100644 modules/options/base.go create mode 100644 modules/templates/htmlrenderer.go create mode 100644 modules/templates/mailer.go create mode 100644 modules/translation/i18n/errors.go create mode 100644 modules/translation/i18n/format.go create mode 100644 modules/translation/i18n/localestore.go create mode 100644 modules/watcher/watcher.go diff --git a/cmd/embedded.go b/cmd/embedded.go index 30fc7103d838b..ffdc3d6a6364f 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -123,7 +123,7 @@ func initEmbeddedExtractor(c *cli.Context) error { sections["public"] = §ion{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset} sections["options"] = §ion{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset} - sections["templates"] = §ion{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset} + sections["templates"] = §ion{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset} for _, sec := range sections { assets = append(assets, buildAssetList(sec, pats, c)...) diff --git a/cmd/web.go b/cmd/web.go index 43bb0ada911e7..b53e867c8e38e 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -126,8 +126,10 @@ func runWeb(ctx *cli.Context) error { return err } } - c := install.Routes() + installCtx, cancel := context.WithCancel(graceful.GetManager().HammerContext()) + c := install.Routes(installCtx) err := listen(c, false) + cancel() if err != nil { log.Critical("Unable to open listener for installer. Is Gitea already running?") graceful.GetManager().DoGracefulShutdown() @@ -174,7 +176,7 @@ func runWeb(ctx *cli.Context) error { } // Set up Chi routes - c := routers.NormalRoutes() + c := routers.NormalRoutes(graceful.GetManager().HammerContext()) err := listen(c, true) <-graceful.GetManager().Done() log.Info("PID: %d Gitea Web Finished", os.Getpid()) diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index f6d29f3c5b574..36487b2f84ee5 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -27,6 +27,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unittest" gitea_git "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/external" repo_module "code.gitea.io/gitea/modules/repository" @@ -118,7 +119,7 @@ func runPR() { // routers.GlobalInit() external.RegisterRenderers() markup.Init() - c := routers.NormalRoutes() + c := routers.NormalRoutes(graceful.GetManager().HammerContext()) log.Printf("[PR] Ready for testing !\n") log.Printf("[PR] Login with user1, user2, user3, ... with pass: password\n") diff --git a/go.mod b/go.mod index 8e0003d6ecb02..478d93c894df3 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 github.com/felixge/fgprof v0.9.2 + github.com/fsnotify/fsnotify v1.5.4 github.com/gliderlabs/ssh v0.3.4 github.com/go-ap/activitypub v0.0.0-20220615144428-48208c70483b github.com/go-ap/jsonld v0.0.0-20220615144122-1d862b15410d @@ -160,7 +161,6 @@ require ( github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/fullstorydev/grpcurl v1.8.1 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/go-ap/errors v0.0.0-20220615144307-e8bc4a40ae9f // indirect diff --git a/integrations/api_activitypub_person_test.go b/integrations/api_activitypub_person_test.go index e19da40864e92..3dc2fda3f4768 100644 --- a/integrations/api_activitypub_person_test.go +++ b/integrations/api_activitypub_person_test.go @@ -23,10 +23,10 @@ import ( func TestActivityPubPerson(t *testing.T) { setting.Federation.Enabled = true - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.TODO()) defer func() { setting.Federation.Enabled = false - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.TODO()) }() onGiteaRun(t, func(*testing.T, *url.URL) { @@ -60,10 +60,10 @@ func TestActivityPubPerson(t *testing.T) { func TestActivityPubMissingPerson(t *testing.T) { setting.Federation.Enabled = true - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.Background()) defer func() { setting.Federation.Enabled = false - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.Background()) }() onGiteaRun(t, func(*testing.T, *url.URL) { @@ -75,10 +75,10 @@ func TestActivityPubMissingPerson(t *testing.T) { func TestActivityPubPersonInbox(t *testing.T) { setting.Federation.Enabled = true - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.Background()) defer func() { setting.Federation.Enabled = false - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.Background()) }() srv := httptest.NewServer(c) diff --git a/integrations/api_nodeinfo_test.go b/integrations/api_nodeinfo_test.go index cf9ff4da1b532..bbb79120784e2 100644 --- a/integrations/api_nodeinfo_test.go +++ b/integrations/api_nodeinfo_test.go @@ -5,6 +5,7 @@ package integrations import ( + "context" "net/http" "net/url" "testing" @@ -18,10 +19,10 @@ import ( func TestNodeinfo(t *testing.T) { setting.Federation.Enabled = true - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.TODO()) defer func() { setting.Federation.Enabled = false - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.TODO()) }() onGiteaRun(t, func(*testing.T, *url.URL) { diff --git a/integrations/create_no_session_test.go b/integrations/create_no_session_test.go index 49234c1e9599c..017fe1d356ed5 100644 --- a/integrations/create_no_session_test.go +++ b/integrations/create_no_session_test.go @@ -5,6 +5,7 @@ package integrations import ( + "context" "net/http" "net/http/httptest" "os" @@ -57,7 +58,7 @@ func TestSessionFileCreation(t *testing.T) { oldSessionConfig := setting.SessionConfig.ProviderConfig defer func() { setting.SessionConfig.ProviderConfig = oldSessionConfig - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.TODO()) }() var config session.Options @@ -82,7 +83,7 @@ func TestSessionFileCreation(t *testing.T) { setting.SessionConfig.ProviderConfig = string(newConfigBytes) - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.TODO()) t.Run("NoSessionOnViewIssue", func(t *testing.T) { defer PrintCurrentTest(t)() diff --git a/integrations/integration_test.go b/integrations/integration_test.go index 8a43de7c45fa9..c3da53396585f 100644 --- a/integrations/integration_test.go +++ b/integrations/integration_test.go @@ -89,7 +89,7 @@ func TestMain(m *testing.M) { defer cancel() initIntegrationTest() - c = routers.NormalRoutes() + c = routers.NormalRoutes(context.TODO()) // integration test settings... if setting.Cfg != nil { diff --git a/modules/context/context.go b/modules/context/context.go index 68f8a1b408c1f..d988dee3fbc80 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -673,8 +673,8 @@ func Auth(authMethod auth.Method) func(*Context) { } // Contexter initializes a classic context for a request. -func Contexter() func(next http.Handler) http.Handler { - rnd := templates.HTMLRenderer() +func Contexter(ctx context.Context) func(next http.Handler) http.Handler { + _, rnd := templates.HTMLRenderer(ctx) csrfOpts := getCsrfOpts() if !setting.IsProd { CsrfTokenRegenerationInterval = 5 * time.Second // in dev, re-generate the tokens more aggressively for debug purpose diff --git a/modules/context/package.go b/modules/context/package.go index 4c52907dc529c..28210a6d6ddd9 100644 --- a/modules/context/package.go +++ b/modules/context/package.go @@ -5,6 +5,7 @@ package context import ( + gocontext "context" "fmt" "net/http" @@ -13,6 +14,7 @@ import ( "code.gitea.io/gitea/models/perm" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/templates" ) // Package contains owner, access mode and optional the package descriptor @@ -101,12 +103,14 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) { } // PackageContexter initializes a package context for a request. -func PackageContexter() func(next http.Handler) http.Handler { +func PackageContexter(ctx gocontext.Context) func(next http.Handler) http.Handler { + _, rnd := templates.HTMLRenderer(ctx) return func(next http.Handler) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { ctx := Context{ - Resp: NewResponse(resp), - Data: map[string]interface{}{}, + Resp: NewResponse(resp), + Data: map[string]interface{}{}, + Render: rnd, } defer ctx.Close() diff --git a/modules/options/base.go b/modules/options/base.go new file mode 100644 index 0000000000000..685202cef9a71 --- /dev/null +++ b/modules/options/base.go @@ -0,0 +1,34 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package options + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" +) + +func walkAssetDir(root string, callback func(path string, name string, d fs.DirEntry, err error) error) error { + if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + name := path[len(root):] + if len(name) > 0 && name[0] == '/' { + name = name[1:] + } + if err != nil { + if os.IsNotExist(err) { + return callback(path, name, d, err) + } + return err + } + if d.Name() == ".DS_Store" && d.IsDir() { // Because Macs... + return fs.SkipDir + } + return callback(path, name, d, err) + }); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("unable to get files for assets in %s: %w", root, err) + } + return nil +} diff --git a/modules/options/dynamic.go b/modules/options/dynamic.go index 5fea337e4203b..37622b1e30d9e 100644 --- a/modules/options/dynamic.go +++ b/modules/options/dynamic.go @@ -8,8 +8,10 @@ package options import ( "fmt" + "io/fs" "os" "path" + "path/filepath" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -45,7 +47,7 @@ func Dir(name string) ([]string, error) { isDir, err = util.IsDir(staticDir) if err != nil { - return []string{}, fmt.Errorf("Unabe to check if static directory %s is a directory. %v", staticDir, err) + return []string{}, fmt.Errorf("unable to check if static directory %s is a directory. %v", staticDir, err) } if isDir { files, err := util.StatDir(staticDir, true) @@ -64,6 +66,18 @@ func Locale(name string) ([]byte, error) { return fileFromDir(path.Join("locale", name)) } +// WalkLocales reads the content of a specific locale from static or custom path. +func WalkLocales(callback func(path string, name string, d fs.DirEntry, err error) error) error { + if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to walk locales. Error: %w", err) + } + + if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to walk locales. Error: %w", err) + } + return nil +} + // Readme reads the content of a specific readme from static or custom path. func Readme(name string) ([]byte, error) { return fileFromDir(path.Join("readme", name)) diff --git a/modules/options/static.go b/modules/options/static.go index 6cad88cb61bbb..b6a1ee8d3b72d 100644 --- a/modules/options/static.go +++ b/modules/options/static.go @@ -9,8 +9,10 @@ package options import ( "fmt" "io" + "io/fs" "os" "path" + "path/filepath" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -74,6 +76,14 @@ func Locale(name string) ([]byte, error) { return fileFromDir(path.Join("locale", name)) } +// WalkLocales reads the content of a specific locale from static or custom path. +func WalkLocales(callback func(path string, name string, d fs.DirEntry, err error) error) error { + if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to walk locales. Error: %w", err) + } + return nil +} + // Readme reads the content of a specific readme from bindata or custom path. func Readme(name string) ([]byte, error) { return fileFromDir(path.Join("readme", name)) diff --git a/modules/templates/base.go b/modules/templates/base.go index 282019f826c1d..0c9d6da3cf09e 100644 --- a/modules/templates/base.go +++ b/modules/templates/base.go @@ -5,15 +5,16 @@ package templates import ( + "fmt" + "io/fs" "os" + "path/filepath" "strings" "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - - "github.com/unrolled/render" ) // Vars represents variables to be render in golang templates @@ -46,8 +47,16 @@ func BaseVars() Vars { } } -func getDirAssetNames(dir string) []string { +func getDirTemplateAssetNames(dir string) []string { + return getDirAssetNames(dir, false) +} + +func getDirAssetNames(dir string, mailer bool) []string { var tmpls []string + + if mailer { + dir += filepath.Join(dir, "mail") + } f, err := os.Stat(dir) if err != nil { if os.IsNotExist(err) { @@ -66,8 +75,13 @@ func getDirAssetNames(dir string) []string { log.Warn("Failed to read %s templates dir. %v", dir, err) return tmpls } + + prefix := "templates/" + if mailer { + prefix += "mail/" + } for _, filePath := range files { - if strings.HasPrefix(filePath, "mail/") { + if !mailer && strings.HasPrefix(filePath, "mail/") { continue } @@ -75,20 +89,36 @@ func getDirAssetNames(dir string) []string { continue } - tmpls = append(tmpls, "templates/"+filePath) + tmpls = append(tmpls, prefix+filePath) } return tmpls } -// HTMLRenderer returns a render. -func HTMLRenderer() *render.Render { - return render.New(render.Options{ - Extensions: []string{".tmpl"}, - Directory: "templates", - Funcs: NewFuncMap(), - Asset: GetAsset, - AssetNames: GetAssetNames, - IsDevelopment: !setting.IsProd, - DisableHTTPErrorRendering: true, - }) +func walkAssetDir(root string, skipMail bool, callback func(path string, name string, d fs.DirEntry, err error) error) error { + mailRoot := filepath.Join(root, "mail") + if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + name := path[len(root):] + if len(name) > 0 && name[0] == '/' { + name = name[1:] + } + if err != nil { + if os.IsNotExist(err) { + return callback(path, name, d, err) + } + return err + } + if skipMail && path == mailRoot && d.IsDir() { + return fs.SkipDir + } + if d.Name() == ".DS_Store" && d.IsDir() { // Because Macs... + return fs.SkipDir + } + if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() { + return callback(path, name, d, err) + } + return nil + }); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("unable to get files for template assets in %s: %w", root, err) + } + return nil } diff --git a/modules/templates/dynamic.go b/modules/templates/dynamic.go index de6968c314a08..4896580f6249f 100644 --- a/modules/templates/dynamic.go +++ b/modules/templates/dynamic.go @@ -8,15 +8,12 @@ package templates import ( "html/template" + "io/fs" "os" - "path" "path/filepath" - "strings" texttmpl "text/template" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) var ( @@ -36,77 +33,42 @@ func GetAsset(name string) ([]byte, error) { return os.ReadFile(filepath.Join(setting.StaticRootPath, name)) } -// GetAssetNames returns assets list -func GetAssetNames() []string { - tmpls := getDirAssetNames(filepath.Join(setting.CustomPath, "templates")) - tmpls2 := getDirAssetNames(filepath.Join(setting.StaticRootPath, "templates")) - return append(tmpls, tmpls2...) -} - -// Mailer provides the templates required for sending notification mails. -func Mailer() (*texttmpl.Template, *template.Template) { - for _, funcs := range NewTextFuncMap() { - subjectTemplates.Funcs(funcs) +// walkTemplateFiles calls a callback for each template asset +func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error { + if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { + return err } - for _, funcs := range NewFuncMap() { - bodyTemplates.Funcs(funcs) + if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { + return err } + return nil +} - staticDir := path.Join(setting.StaticRootPath, "templates", "mail") - - isDir, err := util.IsDir(staticDir) - if err != nil { - log.Warn("Unable to check if templates dir %s is a directory. Error: %v", staticDir, err) - } - if isDir { - files, err := util.StatDir(staticDir) - - if err != nil { - log.Warn("Failed to read %s templates dir. %v", staticDir, err) - } else { - for _, filePath := range files { - if !strings.HasSuffix(filePath, ".tmpl") { - continue - } - - content, err := os.ReadFile(path.Join(staticDir, filePath)) - if err != nil { - log.Warn("Failed to read static %s template. %v", filePath, err) - continue - } +// GetTemplateAssetNames returns list of template names +func GetTemplateAssetNames() []string { + tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates")) + tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates")) + return append(tmpls, tmpls2...) +} - buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, strings.TrimSuffix(filePath, ".tmpl"), content) - } - } +func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error { + if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { + return err } - - customDir := path.Join(setting.CustomPath, "templates", "mail") - - isDir, err = util.IsDir(customDir) - if err != nil { - log.Warn("Unable to check if templates dir %s is a directory. Error: %v", customDir, err) + if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { + return err } - if isDir { - files, err := util.StatDir(customDir) - - if err != nil { - log.Warn("Failed to read %s templates dir. %v", customDir, err) - } else { - for _, filePath := range files { - if !strings.HasSuffix(filePath, ".tmpl") { - continue - } - - content, err := os.ReadFile(path.Join(customDir, filePath)) - if err != nil { - log.Warn("Failed to read custom %s template. %v", filePath, err) - continue - } + return nil +} - buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, strings.TrimSuffix(filePath, ".tmpl"), content) - } - } - } +// BuiltinAsset will read the provided asset from the embedded assets +// (This always returns os.ErrNotExist) +func BuiltinAsset(name string) ([]byte, error) { + return nil, os.ErrNotExist +} - return subjectTemplates, bodyTemplates +// BuiltinAssetNames returns the names of the embedded assets +// (This always returns nil) +func BuiltinAssetNames() []string { + return nil } diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go new file mode 100644 index 0000000000000..618f2835c894e --- /dev/null +++ b/modules/templates/htmlrenderer.go @@ -0,0 +1,51 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package templates + +import ( + "context" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/watcher" + "github.com/unrolled/render" +) + +var rendererKey interface{} = "templatesHtmlRendereer" + +// HTMLRenderer returns the current html renderer for the context or creates and stores one within the context for future use +func HTMLRenderer(ctx context.Context) (context.Context, *render.Render) { + rendererInterface := ctx.Value(rendererKey) + if rendererInterface != nil { + renderer, ok := rendererInterface.(*render.Render) + if ok && renderer != nil { + return ctx, renderer + } + } + + if setting.IsProd { + log.Log(1, log.DEBUG, "Creating static HTML Renderer") + } else { + log.Log(1, log.DEBUG, "Creating auto-reloading HTML Renderer") + } + + renderer := render.New(render.Options{ + Extensions: []string{".tmpl"}, + Directory: "templates", + Funcs: NewFuncMap(), + Asset: GetAsset, + AssetNames: GetTemplateAssetNames, + UseMutexLock: !setting.IsProd, + IsDevelopment: false, + DisableHTTPErrorRendering: true, + }) + if !setting.IsProd { + watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{ + PathsCallback: walkTemplateFiles, + BetweenCallback: renderer.CompileTemplates, + }) + } + return context.WithValue(ctx, rendererKey, renderer), renderer +} diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go new file mode 100644 index 0000000000000..e8696dc8e8c87 --- /dev/null +++ b/modules/templates/mailer.go @@ -0,0 +1,98 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package templates + +import ( + "context" + "html/template" + "io/fs" + "os" + "strings" + texttmpl "text/template" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/watcher" +) + +// Mailer provides the templates required for sending notification mails. +func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { + for _, funcs := range NewTextFuncMap() { + subjectTemplates.Funcs(funcs) + } + for _, funcs := range NewFuncMap() { + bodyTemplates.Funcs(funcs) + } + + refreshTemplates := func() { + for _, assetPath := range BuiltinAssetNames() { + if !strings.HasPrefix(assetPath, "mail/") { + continue + } + + if !strings.HasSuffix(assetPath, ".tmpl") { + continue + } + + content, err := BuiltinAsset(assetPath) + if err != nil { + log.Warn("Failed to read embedded %s template. %v", assetPath, err) + continue + } + + assetName := strings.TrimPrefix( + strings.TrimSuffix( + assetPath, + ".tmpl", + ), + "mail/", + ) + + log.Trace("Adding built-in mailer template for %s", assetName) + buildSubjectBodyTemplate(subjectTemplates, + bodyTemplates, + assetName, + content) + } + + if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + + content, err := os.ReadFile(path) + if err != nil { + log.Warn("Failed to read custom %s template. %v", path, err) + return nil + } + + assetName := strings.TrimSuffix(name, ".tmpl") + log.Trace("Adding mailer template for %s from %q", assetName, path) + buildSubjectBodyTemplate(subjectTemplates, + bodyTemplates, + assetName, + content) + return nil + }); err != nil && !os.IsNotExist(err) { + log.Warn("Error whilst walking mailer templates directories. %v", err) + } + } + + refreshTemplates() + + if !setting.IsProd { + // Now subjectTemplates and bodyTemplates are both synchronized + // thus it is safe to call refresh from a different goroutine + watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{ + PathsCallback: walkMailerTemplates, + BetweenCallback: refreshTemplates, + }) + } + + return subjectTemplates, bodyTemplates +} diff --git a/modules/templates/static.go b/modules/templates/static.go index 351e48b4daa9a..3265bd9cfcbc4 100644 --- a/modules/templates/static.go +++ b/modules/templates/static.go @@ -9,6 +9,7 @@ package templates import ( "html/template" "io" + "io/fs" "os" "path" "path/filepath" @@ -16,10 +17,8 @@ import ( texttmpl "text/template" "time" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" ) var ( @@ -40,95 +39,42 @@ func GetAsset(name string) ([]byte, error) { } else if err == nil { return bs, nil } - return Asset(strings.TrimPrefix(name, "templates/")) + return BuiltinAsset(strings.TrimPrefix(name, "templates/")) } -// GetAssetNames only for chi -func GetAssetNames() []string { +// GetFiles calls a callback for each template asset +func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error { + if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// GetTemplateAssetNames only for chi +func GetTemplateAssetNames() []string { realFS := Assets.(vfsgen۰FS) tmpls := make([]string, 0, len(realFS)) for k := range realFS { + if strings.HasPrefix(k, "/mail/") { + continue + } tmpls = append(tmpls, "templates/"+k[1:]) } customDir := path.Join(setting.CustomPath, "templates") - customTmpls := getDirAssetNames(customDir) + customTmpls := getDirTemplateAssetNames(customDir) return append(tmpls, customTmpls...) } -// Mailer provides the templates required for sending notification mails. -func Mailer() (*texttmpl.Template, *template.Template) { - for _, funcs := range NewTextFuncMap() { - subjectTemplates.Funcs(funcs) - } - for _, funcs := range NewFuncMap() { - bodyTemplates.Funcs(funcs) +func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error { + if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { + return err } - - for _, assetPath := range AssetNames() { - if !strings.HasPrefix(assetPath, "mail/") { - continue - } - - if !strings.HasSuffix(assetPath, ".tmpl") { - continue - } - - content, err := Asset(assetPath) - if err != nil { - log.Warn("Failed to read embedded %s template. %v", assetPath, err) - continue - } - - buildSubjectBodyTemplate(subjectTemplates, - bodyTemplates, - strings.TrimPrefix( - strings.TrimSuffix( - assetPath, - ".tmpl", - ), - "mail/", - ), - content) - } - - customDir := path.Join(setting.CustomPath, "templates", "mail") - isDir, err := util.IsDir(customDir) - if err != nil { - log.Warn("Failed to check if custom directory %s is a directory. %v", err) - } - if isDir { - files, err := util.StatDir(customDir) - - if err != nil { - log.Warn("Failed to read %s templates dir. %v", customDir, err) - } else { - for _, filePath := range files { - if !strings.HasSuffix(filePath, ".tmpl") { - continue - } - - content, err := os.ReadFile(path.Join(customDir, filePath)) - if err != nil { - log.Warn("Failed to read custom %s template. %v", filePath, err) - continue - } - - buildSubjectBodyTemplate(subjectTemplates, - bodyTemplates, - strings.TrimSuffix( - filePath, - ".tmpl", - ), - content) - } - } - } - - return subjectTemplates, bodyTemplates + return nil } -func Asset(name string) ([]byte, error) { +// BuiltinAsset reads the provided asset from the builtin embedded assets +func BuiltinAsset(name string) ([]byte, error) { f, err := Assets.Open("/" + name) if err != nil { return nil, err @@ -137,7 +83,8 @@ func Asset(name string) ([]byte, error) { return io.ReadAll(f) } -func AssetNames() []string { +// BuiltinAssetNames returns the names of the built-in embedded assets +func BuiltinAssetNames() []string { realFS := Assets.(vfsgen۰FS) results := make([]string, 0, len(realFS)) for k := range realFS { @@ -146,7 +93,8 @@ func AssetNames() []string { return results } -func AssetIsDir(name string) (bool, error) { +// BuiltinAssetIsDir returns if a provided asset is a directory +func BuiltinAssetIsDir(name string) (bool, error) { if f, err := Assets.Open("/" + name); err != nil { return false, err } else { diff --git a/modules/timeutil/since_test.go b/modules/timeutil/since_test.go index 8bdb9d7546a39..dac014ee0531b 100644 --- a/modules/timeutil/since_test.go +++ b/modules/timeutil/since_test.go @@ -5,6 +5,7 @@ package timeutil import ( + "context" "fmt" "os" "testing" @@ -31,7 +32,7 @@ func TestMain(m *testing.M) { setting.Names = []string{"english"} setting.Langs = []string{"en-US"} // setup - translation.InitLocales() + translation.InitLocales(context.Background()) BaseDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) // run the tests diff --git a/modules/translation/i18n/errors.go b/modules/translation/i18n/errors.go new file mode 100644 index 0000000000000..b485badd1d2b9 --- /dev/null +++ b/modules/translation/i18n/errors.go @@ -0,0 +1,12 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package i18n + +import "errors" + +var ( + ErrLocaleAlreadyExist = errors.New("lang already exists") + ErrUncertainArguments = errors.New("arguments to i18n should not contain uncertain slices") +) diff --git a/modules/translation/i18n/format.go b/modules/translation/i18n/format.go new file mode 100644 index 0000000000000..3fb9e6d6d05fa --- /dev/null +++ b/modules/translation/i18n/format.go @@ -0,0 +1,42 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package i18n + +import ( + "fmt" + "reflect" +) + +// Format formats provided arguments for a given translated message +func Format(format string, args ...interface{}) (msg string, err error) { + if len(args) == 0 { + return format, nil + } + + fmtArgs := make([]interface{}, 0, len(args)) + for _, arg := range args { + val := reflect.ValueOf(arg) + if val.Kind() == reflect.Slice { + // Previously, we would accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f) + // but this is an unstable behavior. + // + // So we restrict the accepted arguments to either: + // + // 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...) + // 2. Tr(lang, key, args...) as Sprintf(msg, args...) + if len(args) == 1 { + for i := 0; i < val.Len(); i++ { + fmtArgs = append(fmtArgs, val.Index(i).Interface()) + } + } else { + err = ErrUncertainArguments + break + } + } else { + fmtArgs = append(fmtArgs, arg) + } + } + return fmt.Sprintf(format, fmtArgs...), err +} diff --git a/modules/translation/i18n/i18n.go b/modules/translation/i18n/i18n.go index acce5f19fb0dc..23b4e23c76446 100644 --- a/modules/translation/i18n/i18n.go +++ b/modules/translation/i18n/i18n.go @@ -5,203 +5,48 @@ package i18n import ( - "errors" - "fmt" - "os" - "reflect" - "sync" - "time" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" - - "gopkg.in/ini.v1" -) - -var ( - ErrLocaleAlreadyExist = errors.New("lang already exists") - - DefaultLocales = NewLocaleStore(true) + "io" ) -type locale struct { - store *LocaleStore - langName string - textMap map[int]string // the map key (idx) is generated by store's textIdxMap +var DefaultLocales = NewLocaleStore() - sourceFileName string - sourceFileInfo os.FileInfo - lastReloadCheckTime time.Time +type Locale interface { + // Tr translates a given key and arguments for a language + Tr(trKey string, trArgs ...interface{}) string + // Has reports if a locale has a translation for a given key + Has(trKey string) bool } -type LocaleStore struct { - reloadMu *sync.Mutex // for non-prod(dev), use a mutex for live-reload. for prod, no mutex, no live-reload. - - langNames []string - langDescs []string +// LocaleStore provides the functions common to all locale stores +type LocaleStore interface { + io.Closer - localeMap map[string]*locale - textIdxMap map[string]int - - defaultLang string -} - -func NewLocaleStore(isProd bool) *LocaleStore { - ls := &LocaleStore{localeMap: make(map[string]*locale), textIdxMap: make(map[string]int)} - if !isProd { - ls.reloadMu = &sync.Mutex{} - } - return ls -} - -// AddLocaleByIni adds locale by ini into the store -// if source is a string, then the file is loaded. in dev mode, the file can be live-reloaded -// if source is a []byte, then the content is used -func (ls *LocaleStore) AddLocaleByIni(langName, langDesc string, source interface{}) error { - if _, ok := ls.localeMap[langName]; ok { - return ErrLocaleAlreadyExist - } - - lc := &locale{store: ls, langName: langName} - if fileName, ok := source.(string); ok { - lc.sourceFileName = fileName - lc.sourceFileInfo, _ = os.Stat(fileName) // live-reload only works for regular files. the error can be ignored - } - - ls.langNames = append(ls.langNames, langName) - ls.langDescs = append(ls.langDescs, langDesc) - ls.localeMap[lc.langName] = lc - - return ls.reloadLocaleByIni(langName, source) -} - -func (ls *LocaleStore) reloadLocaleByIni(langName string, source interface{}) error { - iniFile, err := ini.LoadSources(ini.LoadOptions{ - IgnoreInlineComment: true, - UnescapeValueCommentSymbols: true, - }, source) - if err != nil { - return fmt.Errorf("unable to load ini: %w", err) - } - iniFile.BlockMode = false - - lc := ls.localeMap[langName] - lc.textMap = make(map[int]string) - for _, section := range iniFile.Sections() { - for _, key := range section.Keys() { - var trKey string - if section.Name() == "" || section.Name() == "DEFAULT" { - trKey = key.Name() - } else { - trKey = section.Name() + "." + key.Name() - } - textIdx, ok := ls.textIdxMap[trKey] - if !ok { - textIdx = len(ls.textIdxMap) - ls.textIdxMap[trKey] = textIdx - } - lc.textMap[textIdx] = key.Value() - } - } - iniFile = nil - return nil -} - -func (ls *LocaleStore) HasLang(langName string) bool { - _, ok := ls.localeMap[langName] - return ok -} - -func (ls *LocaleStore) ListLangNameDesc() (names, desc []string) { - return ls.langNames, ls.langDescs -} - -// SetDefaultLang sets default language as a fallback -func (ls *LocaleStore) SetDefaultLang(lang string) { - ls.defaultLang = lang -} - -// Tr translates content to target language. fall back to default language. -func (ls *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string { - l, ok := ls.localeMap[lang] - if !ok { - l, ok = ls.localeMap[ls.defaultLang] - } - if ok { - return l.Tr(trKey, trArgs...) - } - return trKey -} - -// Tr translates content to locale language. fall back to default language. -func (l *locale) Tr(trKey string, trArgs ...interface{}) string { - if l.store.reloadMu != nil { - l.store.reloadMu.Lock() - defer l.store.reloadMu.Unlock() - now := time.Now() - if now.Sub(l.lastReloadCheckTime) >= time.Second && l.sourceFileInfo != nil && l.sourceFileName != "" { - l.lastReloadCheckTime = now - if sourceFileInfo, err := os.Stat(l.sourceFileName); err == nil && !sourceFileInfo.ModTime().Equal(l.sourceFileInfo.ModTime()) { - if err = l.store.reloadLocaleByIni(l.langName, l.sourceFileName); err == nil { - l.sourceFileInfo = sourceFileInfo - } else { - log.Error("unable to live-reload the locale file %q, err: %v", l.sourceFileName, err) - } - } - } - } - msg, _ := l.tryTr(trKey, trArgs...) - return msg -} - -func (l *locale) tryTr(trKey string, trArgs ...interface{}) (msg string, found bool) { - trMsg := trKey - textIdx, ok := l.store.textIdxMap[trKey] - if ok { - if msg, found = l.textMap[textIdx]; found { - trMsg = msg // use current translation - } else if l.langName != l.store.defaultLang { - if def, ok := l.store.localeMap[l.store.defaultLang]; ok { - return def.tryTr(trKey, trArgs...) - } - } else if !setting.IsProd { - log.Error("missing i18n translation key: %q", trKey) - } - } - - if len(trArgs) > 0 { - fmtArgs := make([]interface{}, 0, len(trArgs)) - for _, arg := range trArgs { - val := reflect.ValueOf(arg) - if val.Kind() == reflect.Slice { - // before, it can accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f), it's an unstable behavior - // now, we restrict the strange behavior and only support: - // 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...) - // 2. Tr(lang, key, args...) as Sprintf(msg, args...) - if len(trArgs) == 1 { - for i := 0; i < val.Len(); i++ { - fmtArgs = append(fmtArgs, val.Index(i).Interface()) - } - } else { - log.Error("the args for i18n shouldn't contain uncertain slices, key=%q, args=%v", trKey, trArgs) - break - } - } else { - fmtArgs = append(fmtArgs, arg) - } - } - return fmt.Sprintf(trMsg, fmtArgs...), found - } - return trMsg, found + // Tr translates a given key and arguments for a language + Tr(lang, trKey string, trArgs ...interface{}) string + // Has reports if a locale has a translation for a given key + Has(lang, trKey string) bool + // SetDefaultLang sets the default language to fall back to + SetDefaultLang(lang string) + // ListLangNameDesc provides paired slices of language names to descriptors + ListLangNameDesc() (names, desc []string) + // Locale return the locale for the provided language or the default language if not found + Locale(langName string) (Locale, bool) + // HasLang returns whether a given language is present in the store + HasLang(langName string) bool + // AddLocaleByIni adds a new language to the store + AddLocaleByIni(langName, langDesc string, source interface{}) error } // ResetDefaultLocales resets the current default locales // NOTE: this is not synchronized -func ResetDefaultLocales(isProd bool) { - DefaultLocales = NewLocaleStore(isProd) +func ResetDefaultLocales() { + if DefaultLocales != nil { + _ = DefaultLocales.Close() + } + DefaultLocales = NewLocaleStore() } -// Tr use default locales to translate content to target language. -func Tr(lang, trKey string, trArgs ...interface{}) string { - return DefaultLocales.Tr(lang, trKey, trArgs...) +// GetLocales returns the locale from the default locales +func GetLocale(lang string) (Locale, bool) { + return DefaultLocales.Locale(lang) } diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go index 32f7585b322e0..7940e59c940a3 100644 --- a/modules/translation/i18n/i18n_test.go +++ b/modules/translation/i18n/i18n_test.go @@ -27,36 +27,34 @@ fmt = %[2]s %[1]s sub = Changed Sub String `) - for _, isProd := range []bool{true, false} { - ls := NewLocaleStore(isProd) - assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1)) - assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2)) - ls.SetDefaultLang("lang1") + ls := NewLocaleStore() + assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1)) + assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2)) + ls.SetDefaultLang("lang1") - result := ls.Tr("lang1", "fmt", "a", "b") - assert.Equal(t, "a b", result) + result := ls.Tr("lang1", "fmt", "a", "b") + assert.Equal(t, "a b", result) - result = ls.Tr("lang2", "fmt", "a", "b") - assert.Equal(t, "b a", result) + result = ls.Tr("lang2", "fmt", "a", "b") + assert.Equal(t, "b a", result) - result = ls.Tr("lang1", "section.sub") - assert.Equal(t, "Sub String", result) + result = ls.Tr("lang1", "section.sub") + assert.Equal(t, "Sub String", result) - result = ls.Tr("lang2", "section.sub") - assert.Equal(t, "Changed Sub String", result) + result = ls.Tr("lang2", "section.sub") + assert.Equal(t, "Changed Sub String", result) - result = ls.Tr("", ".dot.name") - assert.Equal(t, "Dot Name", result) + result = ls.Tr("", ".dot.name") + assert.Equal(t, "Dot Name", result) - result = ls.Tr("lang2", "section.mixed") - assert.Equal(t, `test value; more text`, result) + result = ls.Tr("lang2", "section.mixed") + assert.Equal(t, `test value; more text`, result) - langs, descs := ls.ListLangNameDesc() - assert.Equal(t, []string{"lang1", "lang2"}, langs) - assert.Equal(t, []string{"Lang1", "Lang2"}, descs) + langs, descs := ls.ListLangNameDesc() + assert.Equal(t, []string{"lang1", "lang2"}, langs) + assert.Equal(t, []string{"Lang1", "Lang2"}, descs) - result, found := ls.localeMap["lang1"].tryTr("no-such") - assert.Equal(t, "no-such", result) - assert.False(t, found) - } + found := ls.Has("lang1", "no-such") + assert.False(t, found) + assert.NoError(t, ls.Close()) } diff --git a/modules/translation/i18n/localestore.go b/modules/translation/i18n/localestore.go new file mode 100644 index 0000000000000..4388d2c76dd7b --- /dev/null +++ b/modules/translation/i18n/localestore.go @@ -0,0 +1,161 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package i18n + +import ( + "fmt" + + "code.gitea.io/gitea/modules/log" + "gopkg.in/ini.v1" +) + +// This file implements the static LocaleStore that will not watch for changes + +type locale struct { + store *localeStore + langName string + idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap +} + +type localeStore struct { + // After initializing has finished, these fields are read-only. + langNames []string + langDescs []string + + localeMap map[string]*locale + trKeyToIdxMap map[string]int + + defaultLang string +} + +// NewLocaleStore creates a static locale store +func NewLocaleStore() LocaleStore { + return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)} +} + +// AddLocaleByIni adds locale by ini into the store +// if source is a string, then the file is loaded +// if source is a []byte, then the content is used +func (store *localeStore) AddLocaleByIni(langName, langDesc string, source interface{}) error { + if _, ok := store.localeMap[langName]; ok { + return ErrLocaleAlreadyExist + } + + store.langNames = append(store.langNames, langName) + store.langDescs = append(store.langDescs, langDesc) + + l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)} + store.localeMap[l.langName] = l + + iniFile, err := ini.LoadSources(ini.LoadOptions{ + IgnoreInlineComment: true, + UnescapeValueCommentSymbols: true, + }, source) + if err != nil { + return fmt.Errorf("unable to load ini: %w", err) + } + iniFile.BlockMode = false + + for _, section := range iniFile.Sections() { + for _, key := range section.Keys() { + var trKey string + if section.Name() == "" || section.Name() == "DEFAULT" { + trKey = key.Name() + } else { + trKey = section.Name() + "." + key.Name() + } + idx, ok := store.trKeyToIdxMap[trKey] + if !ok { + idx = len(store.trKeyToIdxMap) + store.trKeyToIdxMap[trKey] = idx + } + l.idxToMsgMap[idx] = key.Value() + } + } + iniFile = nil + + return nil +} + +func (store *localeStore) HasLang(langName string) bool { + _, ok := store.localeMap[langName] + return ok +} + +func (store *localeStore) ListLangNameDesc() (names, desc []string) { + return store.langNames, store.langDescs +} + +// SetDefaultLang sets default language as a fallback +func (store *localeStore) SetDefaultLang(lang string) { + store.defaultLang = lang +} + +// Tr translates content to target language. fall back to default language. +func (store *localeStore) Tr(lang, trKey string, trArgs ...interface{}) string { + l, _ := store.Locale(lang) + + if l != nil { + return l.Tr(trKey, trArgs...) + } + return trKey +} + +// Has returns whether the given language has a translation for the provided key +func (store *localeStore) Has(lang, trKey string) bool { + l, _ := store.Locale(lang) + + if l != nil { + return false + } + return l.Has(trKey) +} + +// Locale returns the locale for the lang or the default language +func (store *localeStore) Locale(lang string) (l Locale, found bool) { + l, found = store.localeMap[lang] + if !found { + l = store.localeMap[store.defaultLang] + } + return l, found +} + +// Close implements io.Closer +func (store *localeStore) Close() error { + return nil +} + +// Tr translates content to locale language. fall back to default language. +func (l *locale) Tr(trKey string, trArgs ...interface{}) string { + format := trKey + + idx, ok := l.store.trKeyToIdxMap[trKey] + if ok { + if msg, ok := l.idxToMsgMap[idx]; ok { + format = msg // use the found translation + } else if def, ok := l.store.localeMap[l.store.defaultLang]; ok { + // try to use default locale's translation + if msg, ok := def.idxToMsgMap[idx]; ok { + format = msg + } + } + } + + msg, err := Format(format, trArgs...) + if err != nil { + log.Error("Error whilst formatting %q in %s: %v", trKey, l.langName, err) + } + return msg +} + +// Has returns whether a key is present in this locale or not +func (l *locale) Has(trKey string) bool { + idx, ok := l.store.trKeyToIdxMap[trKey] + if !ok { + return false + } + _, ok = l.idxToMsgMap[idx] + return ok +} diff --git a/modules/translation/translation.go b/modules/translation/translation.go index fcc101d963435..e40a9357faefe 100644 --- a/modules/translation/translation.go +++ b/modules/translation/translation.go @@ -5,15 +5,16 @@ package translation import ( - "path" + "context" "sort" "strings" + "sync" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation/i18n" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/watcher" "golang.org/x/text/language" ) @@ -31,6 +32,7 @@ type LangType struct { } var ( + lock *sync.RWMutex matcher language.Matcher allLangs []*LangType allLangMap map[string]*LangType @@ -43,58 +45,53 @@ func AllLangs() []*LangType { } // InitLocales loads the locales -func InitLocales() { - i18n.ResetDefaultLocales(setting.IsProd) - localeNames, err := options.Dir("locale") - if err != nil { - log.Fatal("Failed to list locale files: %v", err) +func InitLocales(ctx context.Context) { + if lock != nil { + lock.Lock() + defer lock.Unlock() + } else if !setting.IsProd && lock == nil { + lock = &sync.RWMutex{} } - localFiles := make(map[string]interface{}, len(localeNames)) - for _, name := range localeNames { - if options.IsDynamic() { - // Try to check if CustomPath has the file, otherwise fallback to StaticRootPath - value := path.Join(setting.CustomPath, "options/locale", name) - - isFile, err := util.IsFile(value) - if err != nil { - log.Fatal("Failed to load %s locale file. %v", name, err) - } + refreshLocales := func() { + i18n.ResetDefaultLocales() + localeNames, err := options.Dir("locale") + if err != nil { + log.Fatal("Failed to list locale files: %v", err) + } - if isFile { - localFiles[name] = value - } else { - localFiles[name] = path.Join(setting.StaticRootPath, "options/locale", name) - } - } else { + localFiles := make(map[string]interface{}, len(localeNames)) + for _, name := range localeNames { localFiles[name], err = options.Locale(name) if err != nil { log.Fatal("Failed to load %s locale file. %v", name, err) } } - } - supportedTags = make([]language.Tag, len(setting.Langs)) - for i, lang := range setting.Langs { - supportedTags[i] = language.Raw.Make(lang) - } + supportedTags = make([]language.Tag, len(setting.Langs)) + for i, lang := range setting.Langs { + supportedTags[i] = language.Raw.Make(lang) + } - matcher = language.NewMatcher(supportedTags) - for i := range setting.Names { - key := "locale_" + setting.Langs[i] + ".ini" + matcher = language.NewMatcher(supportedTags) + for i := range setting.Names { + key := "locale_" + setting.Langs[i] + ".ini" - if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil { - log.Error("Failed to set messages to %s: %v", setting.Langs[i], err) + if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil { + log.Error("Failed to set messages to %s: %v", setting.Langs[i], err) + } } - } - if len(setting.Langs) != 0 { - defaultLangName := setting.Langs[0] - if defaultLangName != "en-US" { - log.Info("Use the first locale (%s) in LANGS setting option as default", defaultLangName) + if len(setting.Langs) != 0 { + defaultLangName := setting.Langs[0] + if defaultLangName != "en-US" { + log.Info("Use the first locale (%s) in LANGS setting option as default", defaultLangName) + } + i18n.DefaultLocales.SetDefaultLang(defaultLangName) } - i18n.DefaultLocales.SetDefaultLang(defaultLangName) } + refreshLocales() + langs, descs := i18n.DefaultLocales.ListLangNameDesc() allLangs = make([]*LangType, 0, len(langs)) allLangMap = map[string]*LangType{} @@ -108,6 +105,17 @@ func InitLocales() { sort.Slice(allLangs, func(i, j int) bool { return strings.ToLower(allLangs[i].Name) < strings.ToLower(allLangs[j].Name) }) + + if !setting.IsProd { + watcher.CreateWatcher(ctx, "Locales", &watcher.CreateWatcherOpts{ + PathsCallback: options.WalkLocales, + BetweenCallback: func() { + lock.Lock() + defer lock.Unlock() + refreshLocales() + }, + }) + } } // Match matches accept languages @@ -118,16 +126,24 @@ func Match(tags ...language.Tag) language.Tag { // locale represents the information of localization. type locale struct { + i18n.Locale Lang, LangName string // these fields are used directly in templates: .i18n.Lang } // NewLocale return a locale func NewLocale(lang string) Locale { + if lock != nil { + lock.RLock() + defer lock.RUnlock() + } + langName := "unknown" if l, ok := allLangMap[lang]; ok { langName = l.Name } + i18nLocale, _ := i18n.GetLocale(lang) return &locale{ + Locale: i18nLocale, Lang: lang, LangName: langName, } @@ -137,11 +153,6 @@ func (l *locale) Language() string { return l.Lang } -// Tr translates content to target language. -func (l *locale) Tr(format string, args ...interface{}) string { - return i18n.Tr(l.Lang, format, args...) -} - // Language specific rules for translating plural texts var trNLangRules = map[string]func(int64) int{ // the default rule is "en-US" if a language isn't listed here diff --git a/modules/watcher/watcher.go b/modules/watcher/watcher.go new file mode 100644 index 0000000000000..8029ce1ab96cc --- /dev/null +++ b/modules/watcher/watcher.go @@ -0,0 +1,104 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package watcher + +import ( + "context" + "io/fs" + "os" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "github.com/fsnotify/fsnotify" +) + +type CreateWatcherOpts struct { + PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error + BeforeCallback func() + BetweenCallback func() + AfterCallback func() +} + +func CreateWatcher(ctx context.Context, desc string, opts *CreateWatcherOpts) { + go run(ctx, desc, opts) +} + +func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { + if opts.BeforeCallback != nil { + opts.BeforeCallback() + } + if opts.AfterCallback != nil { + defer opts.AfterCallback() + } + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Watcher: "+desc, process.SystemProcessType, true) + defer finished() + + log.Trace("Watcher loop starting for %s", desc) + defer log.Trace("Watcher loop ended for %s", desc) + + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Error("Unable to create watcher for %s: %v", desc, err) + return + } + if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { + if err != nil && !os.IsNotExist(err) { + return err + } + log.Trace("Watcher: %s watching %q", desc, path) + _ = watcher.Add(path) + return nil + }); err != nil { + log.Error("Unable to create watcher for %s: %v", desc, err) + _ = watcher.Close() + return + } + + // Note we don't call the BetweenCallback here + + for { + select { + case event, ok := <-watcher.Events: + if !ok { + _ = watcher.Close() + return + } + log.Debug("Watched file for %s had event: %v", desc, event) + case err, ok := <-watcher.Errors: + if !ok { + _ = watcher.Close() + return + } + log.Error("Error whilst watching files for %s: %v", desc, err) + case <-ctx.Done(): + _ = watcher.Close() + return + } + + // Recreate the watcher - only call the BetweenCallback after the new watcher is set-up + _ = watcher.Close() + watcher, err = fsnotify.NewWatcher() + if err != nil { + log.Error("Unable to create watcher for %s: %v", desc, err) + return + } + if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { + if err != nil { + return err + } + _ = watcher.Add(path) + return nil + }); err != nil { + log.Error("Unable to create watcher for %s: %v", desc, err) + _ = watcher.Close() + return + } + + // Inform our BetweenCallback that there has been an event + if opts.BetweenCallback != nil { + opts.BetweenCallback() + } + } +} diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index b5fdc739d7c10..c4efae8bd2b84 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -5,6 +5,7 @@ package packages import ( + gocontext "context" "net/http" "regexp" "strings" @@ -37,10 +38,10 @@ func reqPackageAccess(accessMode perm.AccessMode) func(ctx *context.Context) { } } -func Routes() *web.Route { +func Routes(ctx gocontext.Context) *web.Route { r := web.NewRoute() - r.Use(context.PackageContexter()) + r.Use(context.PackageContexter(ctx)) authMethods := []auth.Method{ &auth.OAuth2{}, @@ -237,10 +238,10 @@ func Routes() *web.Route { return r } -func ContainerRoutes() *web.Route { +func ContainerRoutes(ctx gocontext.Context) *web.Route { r := web.NewRoute() - r.Use(context.PackageContexter()) + r.Use(context.PackageContexter(ctx)) authMethods := []auth.Method{ &auth.Basic{}, diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 9209c4edd5501..848fd9a148475 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -16,7 +16,6 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" pypi_module "code.gitea.io/gitea/modules/packages/pypi" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/validation" "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" @@ -58,7 +57,6 @@ func PackageMetadata(ctx *context.Context) { ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi" ctx.Data["PackageDescriptor"] = pds[0] ctx.Data["PackageDescriptors"] = pds - ctx.Render = templates.HTMLRenderer() ctx.HTML(http.StatusOK, "api/packages/pypi/simple") } diff --git a/routers/api/v1/misc/markdown_test.go b/routers/api/v1/misc/markdown_test.go index 9beb88be16846..7809fa5cc72a0 100644 --- a/routers/api/v1/misc/markdown_test.go +++ b/routers/api/v1/misc/markdown_test.go @@ -29,7 +29,7 @@ const ( ) func createContext(req *http.Request) (*context.Context, *httptest.ResponseRecorder) { - rnd := templates.HTMLRenderer() + _, rnd := templates.HTMLRenderer(req.Context()) resp := httptest.NewRecorder() c := &context.Context{ Req: req, diff --git a/routers/init.go b/routers/init.go index 2898c446072f1..9eb6d14d3822a 100644 --- a/routers/init.go +++ b/routers/init.go @@ -29,6 +29,7 @@ import ( "code.gitea.io/gitea/modules/ssh" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/svg" + "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" @@ -114,7 +115,7 @@ func GlobalInitInstalled(ctx context.Context) { log.Info("Run Mode: %s", util.ToTitleCase(setting.RunMode)) // Setup i18n - translation.InitLocales() + translation.InitLocales(ctx) setting.NewServices() mustInit(storage.Init) @@ -171,18 +172,19 @@ func GlobalInitInstalled(ctx context.Context) { } // NormalRoutes represents non install routes -func NormalRoutes() *web.Route { +func NormalRoutes(ctx context.Context) *web.Route { + ctx, _ = templates.HTMLRenderer(ctx) r := web.NewRoute() for _, middle := range common.Middlewares() { r.Use(middle) } - r.Mount("/", web_routers.Routes()) + r.Mount("/", web_routers.Routes(ctx)) r.Mount("/api/v1", apiv1.Routes()) r.Mount("/api/internal", private.Routes()) if setting.Packages.Enabled { - r.Mount("/api/packages", packages_router.Routes()) - r.Mount("/v2", packages_router.ContainerRoutes()) + r.Mount("/api/packages", packages_router.Routes(ctx)) + r.Mount("/v2", packages_router.ContainerRoutes(ctx)) } return r } diff --git a/routers/install/install.go b/routers/install/install.go index 27c3509fdec51..7483d14d255a3 100644 --- a/routers/install/install.go +++ b/routers/install/install.go @@ -6,6 +6,7 @@ package install import ( + goctx "context" "fmt" "net/http" "os" @@ -51,39 +52,41 @@ func getSupportedDbTypeNames() (dbTypeNames []map[string]string) { } // Init prepare for rendering installation page -func Init(next http.Handler) http.Handler { - rnd := templates.HTMLRenderer() +func Init(ctx goctx.Context) func(next http.Handler) http.Handler { + _, rnd := templates.HTMLRenderer(ctx) dbTypeNames := getSupportedDbTypeNames() - return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if setting.InstallLock { - resp.Header().Add("Refresh", "1; url="+setting.AppURL+"user/login") - _ = rnd.HTML(resp, http.StatusOK, string(tplPostInstall), nil) - return - } - locale := middleware.Locale(resp, req) - startTime := time.Now() - ctx := context.Context{ - Resp: context.NewResponse(resp), - Flash: &middleware.Flash{}, - Locale: locale, - Render: rnd, - Session: session.GetSession(req), - Data: map[string]interface{}{ - "locale": locale, - "Title": locale.Tr("install.install"), - "PageIsInstall": true, - "DbTypeNames": dbTypeNames, - "AllLangs": translation.AllLangs(), - "PageStartTime": startTime, - - "PasswordHashAlgorithms": user_model.AvailableHashAlgorithms, - }, - } - defer ctx.Close() + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if setting.InstallLock { + resp.Header().Add("Refresh", "1; url="+setting.AppURL+"user/login") + _ = rnd.HTML(resp, http.StatusOK, string(tplPostInstall), nil) + return + } + locale := middleware.Locale(resp, req) + startTime := time.Now() + ctx := context.Context{ + Resp: context.NewResponse(resp), + Flash: &middleware.Flash{}, + Locale: locale, + Render: rnd, + Session: session.GetSession(req), + Data: map[string]interface{}{ + "locale": locale, + "Title": locale.Tr("install.install"), + "PageIsInstall": true, + "DbTypeNames": dbTypeNames, + "AllLangs": translation.AllLangs(), + "PageStartTime": startTime, + + "PasswordHashAlgorithms": user_model.AvailableHashAlgorithms, + }, + } + defer ctx.Close() - ctx.Req = context.WithContext(req, &ctx) - next.ServeHTTP(resp, ctx.Req) - }) + ctx.Req = context.WithContext(req, &ctx) + next.ServeHTTP(resp, ctx.Req) + }) + } } // Install render installation page diff --git a/routers/install/routes.go b/routers/install/routes.go index 32829ede9e26f..682fa2bfb5360 100644 --- a/routers/install/routes.go +++ b/routers/install/routes.go @@ -5,6 +5,7 @@ package install import ( + goctx "context" "fmt" "net/http" "path" @@ -28,8 +29,8 @@ func (d *dataStore) GetData() map[string]interface{} { return *d } -func installRecovery() func(next http.Handler) http.Handler { - rnd := templates.HTMLRenderer() +func installRecovery(ctx goctx.Context) func(next http.Handler) http.Handler { + _, rnd := templates.HTMLRenderer(ctx) return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer func() { @@ -80,7 +81,7 @@ func installRecovery() func(next http.Handler) http.Handler { } // Routes registers the install routes -func Routes() *web.Route { +func Routes(ctx goctx.Context) *web.Route { r := web.NewRoute() for _, middle := range common.Middlewares() { r.Use(middle) @@ -103,7 +104,7 @@ func Routes() *web.Route { Domain: setting.SessionConfig.Domain, })) - r.Use(installRecovery()) + r.Use(installRecovery(ctx)) r.Use(Init) r.Get("/", Install) r.Post("/", web.Bind(forms.InstallForm{}), SubmitInstall) diff --git a/routers/install/routes_test.go b/routers/install/routes_test.go index 29003c3841beb..e69d2d15dfafe 100644 --- a/routers/install/routes_test.go +++ b/routers/install/routes_test.go @@ -5,13 +5,16 @@ package install import ( + "context" "testing" "github.com/stretchr/testify/assert" ) func TestRoutes(t *testing.T) { - routes := Routes() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + routes := Routes(ctx) assert.NotNil(t, routes) assert.EqualValues(t, "/", routes.R.Routes()[0].Pattern) assert.Nil(t, routes.R.Routes()[0].SubRoutes) diff --git a/routers/install/setting.go b/routers/install/setting.go index cf0a01ce31f57..c4912f1124f8a 100644 --- a/routers/install/setting.go +++ b/routers/install/setting.go @@ -24,7 +24,7 @@ func PreloadSettings(ctx context.Context) bool { log.Info("Log path: %s", setting.LogRootPath) log.Info("Configuration file: %s", setting.CustomConf) log.Info("Prepare to run install page") - translation.InitLocales() + translation.InitLocales(ctx) if setting.EnableSQLite3 { log.Info("SQLite3 is supported") } diff --git a/routers/web/base.go b/routers/web/base.go index c7ade55a61f6f..2dacedb21be6b 100644 --- a/routers/web/base.go +++ b/routers/web/base.go @@ -5,6 +5,7 @@ package web import ( + goctx "context" "errors" "fmt" "io" @@ -123,8 +124,8 @@ func (d *dataStore) GetData() map[string]interface{} { // Recovery returns a middleware that recovers from any panics and writes a 500 and a log if so. // This error will be created with the gitea 500 page. -func Recovery() func(next http.Handler) http.Handler { - rnd := templates.HTMLRenderer() +func Recovery(ctx goctx.Context) func(next http.Handler) http.Handler { + _, rnd := templates.HTMLRenderer(ctx) return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer func() { diff --git a/routers/web/web.go b/routers/web/web.go index 1b6dd03bc8a84..f10935b7151ca 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -98,7 +98,7 @@ func buildAuthGroup() *auth_service.Group { } // Routes returns all web routes -func Routes() *web.Route { +func Routes(ctx gocontext.Context) *web.Route { routes := web.NewRoute() routes.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{ @@ -120,7 +120,9 @@ func Routes() *web.Route { }) routes.Use(sessioner) - routes.Use(Recovery()) + ctx, _ = templates.HTMLRenderer(ctx) + + routes.Use(Recovery(ctx)) // We use r.Route here over r.Use because this prevents requests that are not for avatars having to go through this additional handler routes.Route("/avatars/*", "GET, HEAD", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars)) @@ -151,7 +153,7 @@ func Routes() *web.Route { common = append(common, h) } - mailer.InitMailRender(templates.Mailer()) + mailer.InitMailRender(templates.Mailer(ctx)) if setting.Service.EnableCaptcha { // The captcha http.Handler should only fire on /captcha/* so we can just mount this on that url @@ -195,7 +197,7 @@ func Routes() *web.Route { routes.Get("/api/healthz", healthcheck.Check) // Removed: toolbox.Toolboxer middleware will provide debug information which seems unnecessary - common = append(common, context.Contexter()) + common = append(common, context.Contexter(ctx)) group := buildAuthGroup() if err := group.Init(); err != nil { From 07dc4f2a855b47ff1b4eec81629d49b8ea9f7c05 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 3 Jul 2022 21:31:03 +0100 Subject: [PATCH 02/14] placate lint Signed-off-by: Andrew Thornton --- modules/options/base.go | 2 +- modules/options/dynamic.go | 2 +- modules/options/static.go | 2 +- modules/templates/base.go | 2 +- modules/templates/htmlrenderer.go | 1 + modules/translation/i18n/localestore.go | 1 + modules/watcher/watcher.go | 1 + 7 files changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/options/base.go b/modules/options/base.go index 685202cef9a71..eea4e054337ab 100644 --- a/modules/options/base.go +++ b/modules/options/base.go @@ -11,7 +11,7 @@ import ( "path/filepath" ) -func walkAssetDir(root string, callback func(path string, name string, d fs.DirEntry, err error) error) error { +func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error { if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { name := path[len(root):] if len(name) > 0 && name[0] == '/' { diff --git a/modules/options/dynamic.go b/modules/options/dynamic.go index 37622b1e30d9e..eeef11e8daa21 100644 --- a/modules/options/dynamic.go +++ b/modules/options/dynamic.go @@ -67,7 +67,7 @@ func Locale(name string) ([]byte, error) { } // WalkLocales reads the content of a specific locale from static or custom path. -func WalkLocales(callback func(path string, name string, d fs.DirEntry, err error) error) error { +func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error { if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to walk locales. Error: %w", err) } diff --git a/modules/options/static.go b/modules/options/static.go index b6a1ee8d3b72d..d9a6c8366405e 100644 --- a/modules/options/static.go +++ b/modules/options/static.go @@ -77,7 +77,7 @@ func Locale(name string) ([]byte, error) { } // WalkLocales reads the content of a specific locale from static or custom path. -func WalkLocales(callback func(path string, name string, d fs.DirEntry, err error) error) error { +func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error { if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to walk locales. Error: %w", err) } diff --git a/modules/templates/base.go b/modules/templates/base.go index 0c9d6da3cf09e..6a1f73078e605 100644 --- a/modules/templates/base.go +++ b/modules/templates/base.go @@ -94,7 +94,7 @@ func getDirAssetNames(dir string, mailer bool) []string { return tmpls } -func walkAssetDir(root string, skipMail bool, callback func(path string, name string, d fs.DirEntry, err error) error) error { +func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error { mailRoot := filepath.Join(root, "mail") if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { name := path[len(root):] diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index 618f2835c894e..4380b9a45cb21 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/watcher" + "github.com/unrolled/render" ) diff --git a/modules/translation/i18n/localestore.go b/modules/translation/i18n/localestore.go index 4388d2c76dd7b..fe2edaeb3df3e 100644 --- a/modules/translation/i18n/localestore.go +++ b/modules/translation/i18n/localestore.go @@ -8,6 +8,7 @@ import ( "fmt" "code.gitea.io/gitea/modules/log" + "gopkg.in/ini.v1" ) diff --git a/modules/watcher/watcher.go b/modules/watcher/watcher.go index 8029ce1ab96cc..5136c2dee8cc4 100644 --- a/modules/watcher/watcher.go +++ b/modules/watcher/watcher.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" + "github.com/fsnotify/fsnotify" ) From f265ce5ce850283f5a145b5e3cdc891e4793a0d7 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 3 Jul 2022 23:02:36 +0100 Subject: [PATCH 03/14] fix windows Signed-off-by: Andrew Thornton --- services/auth/sspi_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/auth/sspi_windows.go b/services/auth/sspi_windows.go index 7e31378b6c4d8..cc2b43417194f 100644 --- a/services/auth/sspi_windows.go +++ b/services/auth/sspi_windows.go @@ -64,7 +64,7 @@ func (s *SSPI) Init() error { Directory: "templates", Funcs: templates.NewFuncMap(), Asset: templates.GetAsset, - AssetNames: templates.GetAssetNames, + AssetNames: templates.GetTemplateAssetNames, IsDevelopment: !setting.IsProd, }) return nil From aa90128c742da41408c4fd6f56a54f4ad9b7b3c6 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 4 Jul 2022 13:53:22 +0100 Subject: [PATCH 04/14] Init needs a ctx Signed-off-by: Andrew Thornton --- routers/install/routes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/install/routes.go b/routers/install/routes.go index 682fa2bfb5360..70c38a8f81ab3 100644 --- a/routers/install/routes.go +++ b/routers/install/routes.go @@ -105,7 +105,7 @@ func Routes(ctx goctx.Context) *web.Route { })) r.Use(installRecovery(ctx)) - r.Use(Init) + r.Use(Init(ctx)) r.Get("/", Install) r.Post("/", web.Bind(forms.InstallForm{}), SubmitInstall) r.Get("/api/healthz", healthcheck.Check) From 0209fa416bb647d6d5185dbcf45f7a268781c074 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 4 Jul 2022 21:29:27 +0100 Subject: [PATCH 05/14] make SSPI also share the templates too Signed-off-by: Andrew Thornton --- routers/api/v1/api.go | 5 +++-- routers/init.go | 2 +- routers/web/web.go | 2 +- services/auth/group.go | 5 +++-- services/auth/interface.go | 2 +- services/auth/sspi_windows.go | 12 +++--------- 6 files changed, 12 insertions(+), 16 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c93606ae88308..3d6cb0d923157 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -65,6 +65,7 @@ package v1 import ( + gocontext "context" "fmt" "net/http" "reflect" @@ -605,7 +606,7 @@ func buildAuthGroup() *auth.Group { } // Routes registers all v1 APIs routes to web application. -func Routes() *web.Route { +func Routes(ctx gocontext.Context) *web.Route { m := web.NewRoute() m.Use(securityHeaders()) @@ -623,7 +624,7 @@ func Routes() *web.Route { m.Use(context.APIContexter()) group := buildAuthGroup() - if err := group.Init(); err != nil { + if err := group.Init(ctx); err != nil { log.Error("Could not initialize '%s' auth method, error: %s", group.Name(), err) } diff --git a/routers/init.go b/routers/init.go index 9eb6d14d3822a..b0867589bf52d 100644 --- a/routers/init.go +++ b/routers/init.go @@ -180,7 +180,7 @@ func NormalRoutes(ctx context.Context) *web.Route { } r.Mount("/", web_routers.Routes(ctx)) - r.Mount("/api/v1", apiv1.Routes()) + r.Mount("/api/v1", apiv1.Routes(ctx)) r.Mount("/api/internal", private.Routes()) if setting.Packages.Enabled { r.Mount("/api/packages", packages_router.Routes(ctx)) diff --git a/routers/web/web.go b/routers/web/web.go index f10935b7151ca..90e97a3c0800a 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -200,7 +200,7 @@ func Routes(ctx gocontext.Context) *web.Route { common = append(common, context.Contexter(ctx)) group := buildAuthGroup() - if err := group.Init(); err != nil { + if err := group.Init(ctx); err != nil { log.Error("Could not initialize '%s' auth method, error: %s", group.Name(), err) } diff --git a/services/auth/group.go b/services/auth/group.go index 0f40e1a76c9be..bbafe64b495cb 100644 --- a/services/auth/group.go +++ b/services/auth/group.go @@ -5,6 +5,7 @@ package auth import ( + "context" "net/http" "reflect" "strings" @@ -51,14 +52,14 @@ func (b *Group) Name() string { } // Init does nothing as the Basic implementation does not need to allocate any resources -func (b *Group) Init() error { +func (b *Group) Init(ctx context.Context) error { for _, method := range b.methods { initializable, ok := method.(Initializable) if !ok { continue } - if err := initializable.Init(); err != nil { + if err := initializable.Init(ctx); err != nil { return err } } diff --git a/services/auth/interface.go b/services/auth/interface.go index a05ece2078d1d..ecc9ad2ca6b8d 100644 --- a/services/auth/interface.go +++ b/services/auth/interface.go @@ -34,7 +34,7 @@ type Method interface { type Initializable interface { // Init should be called exactly once before using any of the other methods, // in order to allow the plugin to allocate necessary resources - Init() error + Init(ctx context.Context) error } // Named represents a named thing diff --git a/services/auth/sspi_windows.go b/services/auth/sspi_windows.go index cc2b43417194f..757d596c4c216 100644 --- a/services/auth/sspi_windows.go +++ b/services/auth/sspi_windows.go @@ -5,6 +5,7 @@ package auth import ( + "context" "errors" "net/http" "strings" @@ -52,21 +53,14 @@ type SSPI struct { } // Init creates a new global websspi.Authenticator object -func (s *SSPI) Init() error { +func (s *SSPI) Init(ctx context.Context) error { config := websspi.NewConfig() var err error sspiAuth, err = websspi.New(config) if err != nil { return err } - s.rnd = render.New(render.Options{ - Extensions: []string{".tmpl"}, - Directory: "templates", - Funcs: templates.NewFuncMap(), - Asset: templates.GetAsset, - AssetNames: templates.GetTemplateAssetNames, - IsDevelopment: !setting.IsProd, - }) + _, s.rnd = templates.HTMLRenderer(ctx) return nil } From cbdc8bce076b86aa4054c19684589d7e6934cab7 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 10 Jul 2022 15:06:40 +0100 Subject: [PATCH 06/14] use todo only Signed-off-by: Andrew Thornton --- integrations/api_activitypub_person_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integrations/api_activitypub_person_test.go b/integrations/api_activitypub_person_test.go index 3dc2fda3f4768..c0548df0bcf4f 100644 --- a/integrations/api_activitypub_person_test.go +++ b/integrations/api_activitypub_person_test.go @@ -60,10 +60,10 @@ func TestActivityPubPerson(t *testing.T) { func TestActivityPubMissingPerson(t *testing.T) { setting.Federation.Enabled = true - c = routers.NormalRoutes(context.Background()) + c = routers.NormalRoutes(context.TODO()) defer func() { setting.Federation.Enabled = false - c = routers.NormalRoutes(context.Background()) + c = routers.NormalRoutes(context.TODO()) }() onGiteaRun(t, func(*testing.T, *url.URL) { @@ -75,10 +75,10 @@ func TestActivityPubMissingPerson(t *testing.T) { func TestActivityPubPersonInbox(t *testing.T) { setting.Federation.Enabled = true - c = routers.NormalRoutes(context.Background()) + c = routers.NormalRoutes(context.TODO()) defer func() { setting.Federation.Enabled = false - c = routers.NormalRoutes(context.Background()) + c = routers.NormalRoutes(context.TODO()) }() srv := httptest.NewServer(c) From 959595b07c7b62c1f9494dd40f5646d1c8071c14 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 17 Jul 2022 00:39:51 +0100 Subject: [PATCH 07/14] Switch to use syncthing/notify instead of fsnotify/fsnotify Signed-off-by: Andrew Thornton --- go.mod | 3 ++- go.sum | 3 +++ modules/watcher/watcher.go | 46 +++++++++++++++++--------------------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 728c731707854..f2dd349c72d42 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,6 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 github.com/felixge/fgprof v0.9.2 - github.com/fsnotify/fsnotify v1.5.4 github.com/gliderlabs/ssh v0.3.4 github.com/go-ap/activitypub v0.0.0-20220615144428-48208c70483b github.com/go-ap/jsonld v0.0.0-20220615144122-1d862b15410d @@ -82,6 +81,7 @@ require ( github.com/sergi/go-diff v1.2.0 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 github.com/stretchr/testify v1.7.1 + github.com/syncthing/notify v0.0.0-20210616190510-c6b7342338d2 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 github.com/unrolled/render v1.4.1 @@ -161,6 +161,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/fullstorydev/grpcurl v1.8.1 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/go-ap/errors v0.0.0-20220615144307-e8bc4a40ae9f // indirect diff --git a/go.sum b/go.sum index 103206980776e..bdd4461a3ea22 100644 --- a/go.sum +++ b/go.sum @@ -1474,6 +1474,8 @@ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMT github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syncthing/notify v0.0.0-20210616190510-c6b7342338d2 h1:F4snRP//nIuTTW9LYEzVH4HVwDG9T3M4t8y/2nqMbiY= +github.com/syncthing/notify v0.0.0-20210616190510-c6b7342338d2/go.mod h1:J0q59IWjLtpRIJulohwqEZvjzwOfTEPp8SVhDJl+y0Y= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= @@ -1840,6 +1842,7 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/modules/watcher/watcher.go b/modules/watcher/watcher.go index 5136c2dee8cc4..4b288600db253 100644 --- a/modules/watcher/watcher.go +++ b/modules/watcher/watcher.go @@ -12,7 +12,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" - "github.com/fsnotify/fsnotify" + "github.com/syncthing/notify" ) type CreateWatcherOpts struct { @@ -39,21 +39,22 @@ func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { log.Trace("Watcher loop starting for %s", desc) defer log.Trace("Watcher loop ended for %s", desc) - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - return - } + // Make the channel buffered to ensure no event is dropped. Notify will drop + // an event if the receiver is not able to keep up the sending pace. + events := make(chan notify.EventInfo, 1) + if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { if err != nil && !os.IsNotExist(err) { return err } log.Trace("Watcher: %s watching %q", desc, path) - _ = watcher.Add(path) + if err := notify.Watch(path, events, notify.All); err != nil { + log.Trace("Watcher: %s unable to watch %q: error %v", desc, path, err) + } return nil }); err != nil { log.Error("Unable to create watcher for %s: %v", desc, err) - _ = watcher.Close() + notify.Stop(events) return } @@ -61,39 +62,34 @@ func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { for { select { - case event, ok := <-watcher.Events: + case event, ok := <-events: if !ok { - _ = watcher.Close() + notify.Stop(events) return } + log.Debug("Watched file for %s had event: %v", desc, event) - case err, ok := <-watcher.Errors: - if !ok { - _ = watcher.Close() - return - } - log.Error("Error whilst watching files for %s: %v", desc, err) case <-ctx.Done(): - _ = watcher.Close() + notify.Stop(events) return } // Recreate the watcher - only call the BetweenCallback after the new watcher is set-up - _ = watcher.Close() - watcher, err = fsnotify.NewWatcher() - if err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - return - } + notify.Stop(events) + events = make(chan notify.EventInfo, 1) + if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { if err != nil { return err } - _ = watcher.Add(path) + log.Trace("Watcher: %s watching %q", desc, path) + if err := notify.Watch(path, events, notify.All); err != nil { + log.Trace("Watcher: %s unable to watch %q: error %v", desc, path, err) + } return nil }); err != nil { log.Error("Unable to create watcher for %s: %v", desc, err) - _ = watcher.Close() + notify.Stop(events) return } From 426eb8ff5482cf966e2a50e45b4312d3969e7f15 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 17 Jul 2022 10:56:21 +0100 Subject: [PATCH 08/14] Revert "Switch to use syncthing/notify instead of fsnotify/fsnotify" This reverts commit 959595b07c7b62c1f9494dd40f5646d1c8071c14. syncthing/notify opens goroutines etc even on prod mode. This is unacceptable and therefore we should stick with fsnotify which is already a dependency because of unrolled. --- go.mod | 3 +-- go.sum | 3 --- modules/watcher/watcher.go | 46 +++++++++++++++++++++----------------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 04cdcfe18cfe7..3c72e859a5042 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 github.com/felixge/fgprof v0.9.2 + github.com/fsnotify/fsnotify v1.5.4 github.com/gliderlabs/ssh v0.3.4 github.com/go-ap/activitypub v0.0.0-20220615144428-48208c70483b github.com/go-ap/jsonld v0.0.0-20220615144122-1d862b15410d @@ -81,7 +82,6 @@ require ( github.com/sergi/go-diff v1.2.0 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 github.com/stretchr/testify v1.7.1 - github.com/syncthing/notify v0.0.0-20210616190510-c6b7342338d2 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 github.com/unrolled/render v1.4.1 @@ -161,7 +161,6 @@ require ( github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/fullstorydev/grpcurl v1.8.1 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/go-ap/errors v0.0.0-20220615144307-e8bc4a40ae9f // indirect diff --git a/go.sum b/go.sum index 84b42132ad83f..dca68d9a8e7d8 100644 --- a/go.sum +++ b/go.sum @@ -1474,8 +1474,6 @@ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMT github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syncthing/notify v0.0.0-20210616190510-c6b7342338d2 h1:F4snRP//nIuTTW9LYEzVH4HVwDG9T3M4t8y/2nqMbiY= -github.com/syncthing/notify v0.0.0-20210616190510-c6b7342338d2/go.mod h1:J0q59IWjLtpRIJulohwqEZvjzwOfTEPp8SVhDJl+y0Y= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= @@ -1842,7 +1840,6 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/modules/watcher/watcher.go b/modules/watcher/watcher.go index 4b288600db253..5136c2dee8cc4 100644 --- a/modules/watcher/watcher.go +++ b/modules/watcher/watcher.go @@ -12,7 +12,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" - "github.com/syncthing/notify" + "github.com/fsnotify/fsnotify" ) type CreateWatcherOpts struct { @@ -39,22 +39,21 @@ func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { log.Trace("Watcher loop starting for %s", desc) defer log.Trace("Watcher loop ended for %s", desc) - // Make the channel buffered to ensure no event is dropped. Notify will drop - // an event if the receiver is not able to keep up the sending pace. - events := make(chan notify.EventInfo, 1) - + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Error("Unable to create watcher for %s: %v", desc, err) + return + } if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { if err != nil && !os.IsNotExist(err) { return err } log.Trace("Watcher: %s watching %q", desc, path) - if err := notify.Watch(path, events, notify.All); err != nil { - log.Trace("Watcher: %s unable to watch %q: error %v", desc, path, err) - } + _ = watcher.Add(path) return nil }); err != nil { log.Error("Unable to create watcher for %s: %v", desc, err) - notify.Stop(events) + _ = watcher.Close() return } @@ -62,34 +61,39 @@ func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { for { select { - case event, ok := <-events: + case event, ok := <-watcher.Events: if !ok { - notify.Stop(events) + _ = watcher.Close() return } - log.Debug("Watched file for %s had event: %v", desc, event) + case err, ok := <-watcher.Errors: + if !ok { + _ = watcher.Close() + return + } + log.Error("Error whilst watching files for %s: %v", desc, err) case <-ctx.Done(): - notify.Stop(events) + _ = watcher.Close() return } // Recreate the watcher - only call the BetweenCallback after the new watcher is set-up - notify.Stop(events) - events = make(chan notify.EventInfo, 1) - + _ = watcher.Close() + watcher, err = fsnotify.NewWatcher() + if err != nil { + log.Error("Unable to create watcher for %s: %v", desc, err) + return + } if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { if err != nil { return err } - log.Trace("Watcher: %s watching %q", desc, path) - if err := notify.Watch(path, events, notify.All); err != nil { - log.Trace("Watcher: %s unable to watch %q: error %v", desc, path, err) - } + _ = watcher.Add(path) return nil }); err != nil { log.Error("Unable to create watcher for %s: %v", desc, err) - notify.Stop(events) + _ = watcher.Close() return } From 3841573370ba494e989c7e97c153a7b742cee45d Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 24 Aug 2022 15:45:35 +0100 Subject: [PATCH 09/14] as per review Signed-off-by: Andrew Thornton --- modules/options/base.go | 5 ++++- modules/templates/base.go | 2 +- modules/templates/htmlrenderer.go | 2 +- modules/templates/mailer.go | 8 +------- modules/util/path.go | 9 +++++++-- modules/watcher/watcher.go | 18 ++++++++++++++---- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/modules/options/base.go b/modules/options/base.go index eea4e054337ab..48ee209c1dbd4 100644 --- a/modules/options/base.go +++ b/modules/options/base.go @@ -9,10 +9,13 @@ import ( "io/fs" "os" "path/filepath" + + "code.gitea.io/gitea/modules/util" ) func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error { if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + // name is the path relative to the root name := path[len(root):] if len(name) > 0 && name[0] == '/' { name = name[1:] @@ -23,7 +26,7 @@ func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, e } return err } - if d.Name() == ".DS_Store" && d.IsDir() { // Because Macs... + if d.IsDir() && util.CommonSkipDir(d.Name()) { return fs.SkipDir } return callback(path, name, d, err) diff --git a/modules/templates/base.go b/modules/templates/base.go index 05644fd2a3d19..c660707d8ea61 100644 --- a/modules/templates/base.go +++ b/modules/templates/base.go @@ -111,7 +111,7 @@ func walkAssetDir(root string, skipMail bool, callback func(path, name string, d if skipMail && path == mailRoot && d.IsDir() { return fs.SkipDir } - if d.Name() == ".DS_Store" && d.IsDir() { // Because Macs... + if d.IsDir() && util.CommonSkipDir(d.Name()) { // Because Macs... return fs.SkipDir } if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() { diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index 4380b9a45cb21..39dbf67e439f9 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -21,7 +21,7 @@ func HTMLRenderer(ctx context.Context) (context.Context, *render.Render) { rendererInterface := ctx.Value(rendererKey) if rendererInterface != nil { renderer, ok := rendererInterface.(*render.Render) - if ok && renderer != nil { + if ok { return ctx, renderer } } diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go index e8696dc8e8c87..0cac1280f3446 100644 --- a/modules/templates/mailer.go +++ b/modules/templates/mailer.go @@ -42,13 +42,7 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { continue } - assetName := strings.TrimPrefix( - strings.TrimSuffix( - assetPath, - ".tmpl", - ), - "mail/", - ) + assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") log.Trace("Adding built-in mailer template for %s", assetName) buildSubjectBodyTemplate(subjectTemplates, diff --git a/modules/util/path.go b/modules/util/path.go index 0ccc7a1dc2aca..7ef6bc8265950 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -12,7 +12,6 @@ import ( "path/filepath" "regexp" "runtime" - "strings" ) // EnsureAbsolutePath ensure that a path is absolute, making it @@ -91,7 +90,7 @@ func statDir(dirPath, recPath string, includeDir, isDirOnly, followSymlinks bool statList := make([]string, 0) for _, fi := range fis { - if strings.Contains(fi.Name(), ".DS_Store") { + if fi.IsDir() && CommonSkipDir(fi.Name()) { continue } @@ -199,3 +198,9 @@ func HomeDir() (home string, err error) { return home, nil } + +// CommonSkipDir will check a provided name to see if it represents directory that should not be watched +func CommonSkipDir(name string) bool { + // Check for Mac's .DS_Store entries + return name == ".DS_Store" +} diff --git a/modules/watcher/watcher.go b/modules/watcher/watcher.go index 5136c2dee8cc4..f3b71ca704b56 100644 --- a/modules/watcher/watcher.go +++ b/modules/watcher/watcher.go @@ -15,13 +15,23 @@ import ( "github.com/fsnotify/fsnotify" ) +// CreateWatcherOpts are options to configure the watcher type CreateWatcherOpts struct { - PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error - BeforeCallback func() + // PathsCallback is used to set the required paths to watch + PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error + + // BeforeCallback is called before any files are watched + BeforeCallback func() + + // Between Callback is called between after a watched event has occured BetweenCallback func() - AfterCallback func() + + // AfterCallback is called as this watcher ends + AfterCallback func() } +// CreateWatcher creates a watcher labelled with the provided description and running with the provided options. +// The created watcher will create a subcontext from the provided ctx and register it with the process manager. func CreateWatcher(ctx context.Context, desc string, opts *CreateWatcherOpts) { go run(ctx, desc, opts) } @@ -44,7 +54,7 @@ func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { log.Error("Unable to create watcher for %s: %v", desc, err) return } - if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { + if err := opts.PathsCallback(func(path, _ string, d fs.DirEntry, err error) error { if err != nil && !os.IsNotExist(err) { return err } From 2dbfc50660bc220ed6ce204767a93e05be362889 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 24 Aug 2022 16:44:56 +0100 Subject: [PATCH 10/14] Fix tests Signed-off-by: Andrew Thornton --- modules/charset/escape_test.go | 15 +++++++++++---- modules/translation/i18n/localestore.go | 19 +++++++++---------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/modules/charset/escape_test.go b/modules/charset/escape_test.go index 8063e115424cb..a7232a4658ab2 100644 --- a/modules/charset/escape_test.go +++ b/modules/charset/escape_test.go @@ -133,11 +133,18 @@ then resh (ר), and finally heh (ה) (which should appear leftmost).`, }, } +type nullLocale struct{} + +func (nullLocale) Language() string { return "" } +func (nullLocale) Tr(key string, _ ...interface{}) string { return key } +func (nullLocale) TrN(cnt interface{}, key1, keyN string, args ...interface{}) string { return "" } + +var _ (translation.Locale) = nullLocale{} + func TestEscapeControlString(t *testing.T) { for _, tt := range escapeControlTests { t.Run(tt.name, func(t *testing.T) { - locale := translation.NewLocale("en_US") - status, result := EscapeControlString(tt.text, locale) + status, result := EscapeControlString(tt.text, nullLocale{}) if !reflect.DeepEqual(*status, tt.status) { t.Errorf("EscapeControlString() status = %v, wanted= %v", status, tt.status) } @@ -173,7 +180,7 @@ func TestEscapeControlReader(t *testing.T) { t.Run(tt.name, func(t *testing.T) { input := strings.NewReader(tt.text) output := &strings.Builder{} - status, err := EscapeControlReader(input, output, translation.NewLocale("en_US")) + status, err := EscapeControlReader(input, output, nullLocale{}) result := output.String() if err != nil { t.Errorf("EscapeControlReader(): err = %v", err) @@ -195,5 +202,5 @@ func TestEscapeControlReader_panic(t *testing.T) { for i := 0; i < 6826; i++ { bs = append(bs, []byte("—")...) } - _, _ = EscapeControlString(string(bs), translation.NewLocale("en_US")) + _, _ = EscapeControlString(string(bs), nullLocale{}) } diff --git a/modules/translation/i18n/localestore.go b/modules/translation/i18n/localestore.go index fe2edaeb3df3e..e3b88ad96eba6 100644 --- a/modules/translation/i18n/localestore.go +++ b/modules/translation/i18n/localestore.go @@ -98,27 +98,26 @@ func (store *localeStore) SetDefaultLang(lang string) { func (store *localeStore) Tr(lang, trKey string, trArgs ...interface{}) string { l, _ := store.Locale(lang) - if l != nil { - return l.Tr(trKey, trArgs...) - } - return trKey + return l.Tr(trKey, trArgs...) } // Has returns whether the given language has a translation for the provided key func (store *localeStore) Has(lang, trKey string) bool { l, _ := store.Locale(lang) - if l != nil { - return false - } return l.Has(trKey) } // Locale returns the locale for the lang or the default language -func (store *localeStore) Locale(lang string) (l Locale, found bool) { - l, found = store.localeMap[lang] +func (store *localeStore) Locale(lang string) (Locale, bool) { + l, found := store.localeMap[lang] if !found { - l = store.localeMap[store.defaultLang] + var ok bool + l, ok = store.localeMap[store.defaultLang] + if !ok { + // no default - return an empty locale + l = &locale{store: store, idxToMsgMap: make(map[int]string)} + } } return l, found } From 0ff8921f7edd91831d3f00a74a9943b0b3281da0 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 24 Aug 2022 19:01:48 +0100 Subject: [PATCH 11/14] placate lint Signed-off-by: Andrew Thornton --- modules/watcher/watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/watcher/watcher.go b/modules/watcher/watcher.go index f3b71ca704b56..d737f6ccbbcae 100644 --- a/modules/watcher/watcher.go +++ b/modules/watcher/watcher.go @@ -23,7 +23,7 @@ type CreateWatcherOpts struct { // BeforeCallback is called before any files are watched BeforeCallback func() - // Between Callback is called between after a watched event has occured + // Between Callback is called between after a watched event has occurred BetweenCallback func() // AfterCallback is called as this watcher ends From c5bdfef56180b72f4c822efa0490e3925bc4409b Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 27 Aug 2022 16:54:44 +0100 Subject: [PATCH 12/14] Update modules/templates/htmlrenderer.go Co-authored-by: delvh --- modules/templates/htmlrenderer.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index 39dbf67e439f9..80930487fd3c5 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -26,11 +26,11 @@ func HTMLRenderer(ctx context.Context) (context.Context, *render.Render) { } } - if setting.IsProd { - log.Log(1, log.DEBUG, "Creating static HTML Renderer") - } else { - log.Log(1, log.DEBUG, "Creating auto-reloading HTML Renderer") + rendererType := "static" + if !setting.IsProd { + rendererType = "auto-reloading" } + log.Log(1, log.DEBUG, "Creating " + rendererType + " HTML Renderer") renderer := render.New(render.Options{ Extensions: []string{".tmpl"}, From 55c4ec6a8589f2d7d2ea239deeefbd3248cdaac9 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 27 Aug 2022 18:09:13 +0100 Subject: [PATCH 13/14] placate lint Signed-off-by: Andrew Thornton --- modules/templates/htmlrenderer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index 80930487fd3c5..210bb5e73c7e1 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -30,7 +30,7 @@ func HTMLRenderer(ctx context.Context) (context.Context, *render.Render) { if !setting.IsProd { rendererType = "auto-reloading" } - log.Log(1, log.DEBUG, "Creating " + rendererType + " HTML Renderer") + log.Log(1, log.DEBUG, "Creating "+rendererType+" HTML Renderer") renderer := render.New(render.Options{ Extensions: []string{".tmpl"}, From f7a5cd706dd120912512521e99b129255aaf5b75 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 28 Aug 2022 10:05:11 +0100 Subject: [PATCH 14/14] as per wxiaoguang Signed-off-by: Andrew Thornton --- modules/options/base.go | 7 +++++-- modules/templates/base.go | 7 +++++-- modules/util/path.go | 22 +++++++++++++++++----- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/modules/options/base.go b/modules/options/base.go index 48ee209c1dbd4..e1d6efa7f0262 100644 --- a/modules/options/base.go +++ b/modules/options/base.go @@ -26,8 +26,11 @@ func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, e } return err } - if d.IsDir() && util.CommonSkipDir(d.Name()) { - return fs.SkipDir + if util.CommonSkip(d.Name()) { + if d.IsDir() { + return fs.SkipDir + } + return nil } return callback(path, name, d, err) }); err != nil && !os.IsNotExist(err) { diff --git a/modules/templates/base.go b/modules/templates/base.go index c660707d8ea61..d234d531f3dcc 100644 --- a/modules/templates/base.go +++ b/modules/templates/base.go @@ -111,8 +111,11 @@ func walkAssetDir(root string, skipMail bool, callback func(path, name string, d if skipMail && path == mailRoot && d.IsDir() { return fs.SkipDir } - if d.IsDir() && util.CommonSkipDir(d.Name()) { // Because Macs... - return fs.SkipDir + if util.CommonSkip(d.Name()) { + if d.IsDir() { + return fs.SkipDir + } + return nil } if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() { return callback(path, name, d, err) diff --git a/modules/util/path.go b/modules/util/path.go index 7ef6bc8265950..3d4ddec21cb29 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -90,7 +90,7 @@ func statDir(dirPath, recPath string, includeDir, isDirOnly, followSymlinks bool statList := make([]string, 0) for _, fi := range fis { - if fi.IsDir() && CommonSkipDir(fi.Name()) { + if CommonSkip(fi.Name()) { continue } @@ -199,8 +199,20 @@ func HomeDir() (home string, err error) { return home, nil } -// CommonSkipDir will check a provided name to see if it represents directory that should not be watched -func CommonSkipDir(name string) bool { - // Check for Mac's .DS_Store entries - return name == ".DS_Store" +// CommonSkip will check a provided name to see if it represents file or directory that should not be watched +func CommonSkip(name string) bool { + if name == "" { + return true + } + + switch name[0] { + case '.': + return true + case 't', 'T': + return name[1:] == "humbs.db" + case 'd', 'D': + return name[1:] == "esktop.ini" + } + + return false }