diff --git a/cmd/manager.go b/cmd/manager.go index cdfe509075f3b..a608eb07c5ac3 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -4,7 +4,9 @@ package cmd import ( + "context" "fmt" + "io" "net/http" "os" "time" @@ -26,6 +28,11 @@ var ( subcmdFlushQueues, subcmdLogging, subCmdProcesses, + subCmdCPUProfile, + subCmdFGProfile, + subCmdListNamedProfiles, + subCmdNamedProfile, + subCmdTrace, }, } subcmdShutdown = cli.Command{ @@ -95,15 +102,117 @@ var ( Name: "cancel", Usage: "Process PID to cancel. (Only available for non-system processes.)", }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "-", + }, + }, + } + subCmdCPUProfile = cli.Command{ + Name: "cpu-profile", + Usage: "Return PProf CPU profile", + Action: runCPUProfile, + Flags: []cli.Flag{ + cli.DurationFlag{ + Name: "duration", + Usage: "Duration to collect CPU Profile over", + Value: 30 * time.Second, + }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "cpu-profile", + }, + }, + } + subCmdFGProfile = cli.Command{ + Name: "fg-profile", + Usage: "Return PProf Full Go profile", + Action: runFGProfile, + Flags: []cli.Flag{ + cli.DurationFlag{ + Name: "duration", + Usage: "Duration to collect CPU Profile over", + Value: 30 * time.Second, + }, + cli.StringFlag{ + Name: "format", + Usage: "Format to return the profile in: pprof, folded", + Value: "pprof", + }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "fg-profile", + }, + }, + } + subCmdNamedProfile = cli.Command{ + Name: "named-profile", + Usage: "Return PProf named profile", + Action: runNamedProfile, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "name", + Usage: "Name of profile to run", + }, + cli.IntFlag{ + Name: "debug-level", + Usage: "Debug level for the profile", + }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + }, + }, + } + subCmdListNamedProfiles = cli.Command{ + Name: "list-named-profiles", + Usage: "Return PProf list of named profiles", + Action: runListNamedProfile, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "json", + Usage: "Output as json", + }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "-", + }, + }, + } + subCmdTrace = cli.Command{ + Name: "trace", + Usage: "Return PProf trace", + Action: runTrace, + Flags: []cli.Flag{ + cli.DurationFlag{ + Name: "duration", + Usage: "Duration to collect CPU Profile over", + Value: 30 * time.Second, + }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "trace", + }, }, } ) -func runShutdown(c *cli.Context) error { +func setupManager(c *cli.Context) (context.Context, context.CancelFunc) { ctx, cancel := installSignals() - defer cancel() setup("manager", c.Bool("debug")) + return ctx, cancel +} + +func runShutdown(c *cli.Context) error { + ctx, cancel := setupManager(c) + defer cancel() + statusCode, msg := private.Shutdown(ctx) switch statusCode { case http.StatusInternalServerError: @@ -115,10 +224,9 @@ func runShutdown(c *cli.Context) error { } func runRestart(c *cli.Context) error { - ctx, cancel := installSignals() + ctx, cancel := setupManager(c) defer cancel() - setup("manager", c.Bool("debug")) statusCode, msg := private.Restart(ctx) switch statusCode { case http.StatusInternalServerError: @@ -130,10 +238,9 @@ func runRestart(c *cli.Context) error { } func runFlushQueues(c *cli.Context) error { - ctx, cancel := installSignals() + ctx, cancel := setupManager(c) defer cancel() - setup("manager", c.Bool("debug")) statusCode, msg := private.FlushQueues(ctx, c.Duration("timeout"), c.Bool("non-blocking")) switch statusCode { case http.StatusInternalServerError: @@ -144,12 +251,34 @@ func runFlushQueues(c *cli.Context) error { return nil } -func runProcesses(c *cli.Context) error { - ctx, cancel := installSignals() +func determineOutput(c *cli.Context, defaultFilename string) (io.WriteCloser, error) { + out := os.Stdout + filename := c.String("output") + if filename == "" { + filename = defaultFilename + } + if filename != "-" { + var err error + out, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return nil, fail("Unable to open "+filename, err.Error()) + } + fmt.Printf("Writing to %s\n", filename) + } + return out, nil +} + +func wrapManagerPrivateFunc(c *cli.Context, defaultOutput string, fn func(ctx context.Context, out io.Writer) (int, string)) error { + ctx, cancel := setupManager(c) defer cancel() - setup("manager", c.Bool("debug")) - statusCode, msg := private.Processes(ctx, os.Stdout, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) + out, err := determineOutput(c, defaultOutput) + if err != nil { + return err + } + defer out.Close() + + statusCode, msg := fn(ctx, out) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) @@ -157,3 +286,39 @@ func runProcesses(c *cli.Context) error { return nil } + +func runProcesses(c *cli.Context) error { + return wrapManagerPrivateFunc(c, "-", func(ctx context.Context, out io.Writer) (int, string) { + return private.Processes(ctx, out, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) + }) +} + +func runCPUProfile(c *cli.Context) error { + return wrapManagerPrivateFunc(c, "cpu-profile", func(ctx context.Context, out io.Writer) (int, string) { + return private.CPUProfile(ctx, out, c.Duration("duration")) + }) +} + +func runFGProfile(c *cli.Context) error { + return wrapManagerPrivateFunc(c, "fg-profile", func(ctx context.Context, out io.Writer) (int, string) { + return private.FGProfile(ctx, out, c.Duration("duration"), c.String("format")) + }) +} + +func runNamedProfile(c *cli.Context) error { + return wrapManagerPrivateFunc(c, c.String("name")+"-profile", func(ctx context.Context, out io.Writer) (int, string) { + return private.NamedProfile(ctx, out, c.String("name"), c.Int("debug-level")) + }) +} + +func runListNamedProfile(c *cli.Context) error { + return wrapManagerPrivateFunc(c, "-", func(ctx context.Context, out io.Writer) (int, string) { + return private.ListNamedProfiles(ctx, out, c.Bool("json")) + }) +} + +func runTrace(c *cli.Context) error { + return wrapManagerPrivateFunc(c, "trace", func(ctx context.Context, out io.Writer) (int, string) { + return private.Trace(ctx, out, c.Duration("duration")) + }) +} diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index 70efebd203618..40c24230f840c 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -526,6 +526,29 @@ Manage running server operations: - `--stacktraces`: Show stacktraces for goroutines associated with processes - `--json`: Output as json - `--cancel PID`: Send cancel to process with PID. (Only for non-system processes.) + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) + - `cpu-profile`: Return the PProf CPU profile + - Options: + - `--duration`: Duration of time to run profile (default: 30s) + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) + - `fg-profile`: Returns the PProf Full Go profile + - Options: + - `--duration`: Duration of time to run profile (default: 30s) + - `--format`: Format of profile (default: pprof) + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) + - `list-named-profiles`: Returns a list of named profiles + - Options: + - `--json`: Set to true to return a json output + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) + - `named-profile`: Returns the output of a named profile + - Options: + - `--name`: Name of the profile + - `--debug-level`: Debug level for the profile + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) + - `trace`: Return the PProf trace + - Options: + - `--duration`: Duration of time to run profile (default: 30s) + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) ### dump-repo diff --git a/modules/context/private.go b/modules/context/private.go index 24f50fa4713e6..012a50897276b 100644 --- a/modules/context/private.go +++ b/modules/context/private.go @@ -47,7 +47,7 @@ var privateContextKey interface{} = "default_private_context" // WithPrivateContext set up private context in request func WithPrivateContext(req *http.Request, ctx *PrivateContext) *http.Request { - return req.WithContext(context.WithValue(req.Context(), privateContextKey, ctx)) + return req.WithContext(context.WithValue(context.WithValue(req.Context(), privateContextKey, ctx), contextKey, ctx.Context)) } // GetPrivateContext returns a context for Private routes diff --git a/modules/private/manager.go b/modules/private/manager.go index bbf470cd7ad30..bf84d0f10552a 100644 --- a/modules/private/manager.go +++ b/modules/private/manager.go @@ -230,3 +230,47 @@ func Processes(ctx context.Context, out io.Writer, flat, noSystem, stacktraces, } return http.StatusOK, "" } + +// CPUProfile returns a cpu profile from Gitea +func CPUProfile(ctx context.Context, out io.Writer, duration time.Duration) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/cpu-profile?duration=%s", url.QueryEscape(duration.String())) + return commonGet(ctx, out, reqURL) +} + +// FGProfile returns the full go profile from Gitea +func FGProfile(ctx context.Context, out io.Writer, duration time.Duration, format string) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/fgprof?duration=%s&format=%s", url.QueryEscape(duration.String()), url.QueryEscape(format)) + return commonGet(ctx, out, reqURL) +} + +// NamedProfile returns the named profile from Gitea +func NamedProfile(ctx context.Context, out io.Writer, name string, debugLevel int) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/profile?name=%s&debug=%d", url.QueryEscape(name), debugLevel) + return commonGet(ctx, out, reqURL) +} + +// ListNamedProfiles returns a list of named profiles +func ListNamedProfiles(ctx context.Context, out io.Writer, json bool) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/list-profiles?json=%t", json) + return commonGet(ctx, out, reqURL) +} + +// Trace returns a trace from Gitea +func Trace(ctx context.Context, out io.Writer, duration time.Duration) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/trace?duration=%s", url.QueryEscape(duration.String())) + return commonGet(ctx, out, reqURL) +} + +func commonGet(ctx context.Context, out io.Writer, reqURL string) (int, string) { + req := newInternalRequest(ctx, reqURL, "GET") + resp, err := req.Response() + if err != nil { + return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) + } + defer resp.Body.Close() + _, err = io.Copy(out, resp.Body) + if err != nil { + return http.StatusInternalServerError, err.Error() + } + return resp.StatusCode, "" +} diff --git a/modules/process/stacktraces_processlist.go b/modules/process/stacktraces_processlist.go new file mode 100644 index 0000000000000..7a31d38b974ce --- /dev/null +++ b/modules/process/stacktraces_processlist.go @@ -0,0 +1,100 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package process + +import ( + "bytes" + "fmt" + "io" +) + +// WriteProcesses writes out processes to a provided writer +func WriteProcesses(out io.Writer, processes []*Process, processCount int, goroutineCount int64, indent string, flat bool) error { + if goroutineCount > 0 { + if _, err := fmt.Fprintf(out, "%sTotal Number of Goroutines: %d\n", indent, goroutineCount); err != nil { + return err + } + } + if _, err := fmt.Fprintf(out, "%sTotal Number of Processes: %d\n", indent, processCount); err != nil { + return err + } + if len(processes) > 0 { + if err := WriteProcess(out, processes[0], indent+" ", flat); err != nil { + return err + } + } + if len(processes) > 1 { + for _, process := range processes[1:] { + if _, err := fmt.Fprintf(out, "%s |\n", indent); err != nil { + return err + } + if err := WriteProcess(out, process, indent+" ", flat); err != nil { + return err + } + } + } + return nil +} + +// WriteProcess writes out a process to a provided writer +func WriteProcess(out io.Writer, process *Process, indent string, flat bool) error { + sb := &bytes.Buffer{} + if flat { + if process.ParentPID != "" { + _, _ = fmt.Fprintf(sb, "%s+ PID: %s\t\tType: %s\n", indent, process.PID, process.Type) + } else { + _, _ = fmt.Fprintf(sb, "%s+ PID: %s:%s\tType: %s\n", indent, process.ParentPID, process.PID, process.Type) + } + } else { + _, _ = fmt.Fprintf(sb, "%s+ PID: %s\tType: %s\n", indent, process.PID, process.Type) + } + indent += "| " + + _, _ = fmt.Fprintf(sb, "%sDescription: %s\n", indent, process.Description) + _, _ = fmt.Fprintf(sb, "%sStart: %s\n", indent, process.Start) + + if len(process.Stacks) > 0 { + _, _ = fmt.Fprintf(sb, "%sGoroutines:\n", indent) + for _, stack := range process.Stacks { + indent := indent + " " + _, _ = fmt.Fprintf(sb, "%s+ Description: %s", indent, stack.Description) + if stack.Count > 1 { + _, _ = fmt.Fprintf(sb, "* %d", stack.Count) + } + _, _ = fmt.Fprintf(sb, "\n") + indent += "| " + if len(stack.Labels) > 0 { + _, _ = fmt.Fprintf(sb, "%sLabels: %q:%q", indent, stack.Labels[0].Name, stack.Labels[0].Value) + + if len(stack.Labels) > 1 { + for _, label := range stack.Labels[1:] { + _, _ = fmt.Fprintf(sb, ", %q:%q", label.Name, label.Value) + } + } + _, _ = fmt.Fprintf(sb, "\n") + } + _, _ = fmt.Fprintf(sb, "%sStack:\n", indent) + indent += " " + for _, entry := range stack.Entry { + _, _ = fmt.Fprintf(sb, "%s+ %s\n", indent, entry.Function) + _, _ = fmt.Fprintf(sb, "%s| %s:%d\n", indent, entry.File, entry.Line) + } + } + } + if _, err := out.Write(sb.Bytes()); err != nil { + return err + } + sb.Reset() + if len(process.Children) > 0 { + if _, err := fmt.Fprintf(out, "%sChildren:\n", indent); err != nil { + return err + } + for _, child := range process.Children { + if err := WriteProcess(out, child, indent+" ", flat); err != nil { + return err + } + } + } + return nil +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 411a585c81d60..0aa09dcf75779 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2954,6 +2954,7 @@ monitor.previous = Previous Time monitor.execute_times = Executions monitor.process = Running Processes monitor.stacktrace = Stacktraces +monitor.stacktrace.download_stacktrace = Download Stacktrace monitor.goroutines = %d Goroutines monitor.desc = Description monitor.start = Start Time @@ -2963,6 +2964,39 @@ monitor.process.cancel = Cancel process monitor.process.cancel_desc = Cancelling a process may cause data loss monitor.process.cancel_notices = Cancel: %s? monitor.process.children = Children + +monitor.pprof = PProf Profiles +monitor.pprof.description = PProf profiles provide runtime profiling data. +monitor.pprof.description_2 = Text formats can be read directly but most data is in the format expected by the golang pprof visualization tool + +monitor.pprof.download = Download +monitor.pprof.duration = Duration +monitor.pprof.duration_placeholder = e.g. 30s +monitor.pprof.duration_invalid = Invalid duration - duration must be a golang duration string + +monitor.pprof.cpuprofile = CPU Profile +monitor.pprof.cpuprofile.description = CPU profile determines where Gitea spends its time while actively consuming CPU cycles + +monitor.pprof.fgprof = Full Go Profile +monitor.pprof.fgprof.description = Full Go profile provides wall clock profiling combining the CPU Profile with IO time +monitor.pprof.fgprof.format = Format + +monitor.pprof.named_profiles = Named Profiles +monitor.pprof.named_profiles.description = Go provides a number of named profiles for other profiling +monitor.pprof.named_profiles.name = Name +monitor.pprof.named_profiles.debug = Format +monitor.pprof.named_profiles.format_pprof = pprof +monitor.pprof.named_profiles.format_text = text +monitor.pprof.named_profiles.format_goroutine = custom/goroutine + +monitor.pprof.stacktrace.description = Stacktraces provides stacktraces for all current goroutines mapped with Gitea's internal processes +monitor.pprof.stacktrace.flat = Do not nest processes under their parents +monitor.pprof.stacktrace.no-system = Do not include go-routines associated with system processes +monitor.pprof.stacktrace.format = Format + +monitor.pprof.trace = Trace +monitor.pprof.trace.description = Trace provides tracing that can be used by the go tracing tool + monitor.queues = Queues monitor.queue = Queue: %s monitor.queue.name = Name diff --git a/routers/private/internal.go b/routers/private/internal.go index 306e4ffb0040f..06d1a98cef228 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/web/admin" "gitea.com/go-chi/binding" chi_middleware "github.com/go-chi/chi/v5/middleware" @@ -75,6 +76,11 @@ func Routes() *web.Route { r.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger) r.Post("/manager/remove-logger/{group}/{name}", RemoveLogger) r.Get("/manager/processes", Processes) + r.Get("/manager/cpu-profile", admin.PProfCPUProfile) + r.Get("/manager/profile", admin.PProfNamedProfile) + r.Get("/manager/fgprof", admin.PProfFGProfile) + r.Get("/manager/list-profiles", ListProfiles) + r.Get("/manager/trace", admin.Trace) r.Post("/mail/send", SendEmail) r.Post("/restore_repo", RestoreRepo) diff --git a/routers/private/manager_process.go b/routers/private/manager_process.go index a5993bf3718d3..58873eb65d275 100644 --- a/routers/private/manager_process.go +++ b/routers/private/manager_process.go @@ -4,11 +4,10 @@ package private import ( - "bytes" "fmt" - "io" "net/http" "runtime" + "runtime/pprof" "time" "code.gitea.io/gitea/modules/context" @@ -60,7 +59,7 @@ func Processes(ctx *context.PrivateContext) { ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") ctx.Resp.WriteHeader(http.StatusOK) - if err := writeProcesses(ctx.Resp, processes, processCount, goroutineCount, "", flat); err != nil { + if err := process_module.WriteProcesses(ctx.Resp, processes, processCount, goroutineCount, "", flat); err != nil { log.Error("Unable to write out process stacktrace: %v", err) if !ctx.Written() { ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -71,90 +70,33 @@ func Processes(ctx *context.PrivateContext) { } } -func writeProcesses(out io.Writer, processes []*process_module.Process, processCount int, goroutineCount int64, indent string, flat bool) error { - if goroutineCount > 0 { - if _, err := fmt.Fprintf(out, "%sTotal Number of Goroutines: %d\n", indent, goroutineCount); err != nil { - return err - } - } - if _, err := fmt.Fprintf(out, "%sTotal Number of Processes: %d\n", indent, processCount); err != nil { - return err - } - if len(processes) > 0 { - if err := writeProcess(out, processes[0], " ", flat); err != nil { - return err - } - } - if len(processes) > 1 { - for _, process := range processes[1:] { - if _, err := fmt.Fprintf(out, "%s | \n", indent); err != nil { - return err - } - if err := writeProcess(out, process, " ", flat); err != nil { - return err - } +// ListProfiles lists the available named pprof profiles +func ListProfiles(ctx *context.PrivateContext) { + json := ctx.FormBool("json") + profiles := pprof.Profiles() + if json { + names := make([]string, len(profiles)) + for _, profile := range profiles { + names = append(names, profile.Name()) } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "Names": names, + }) } - return nil -} -func writeProcess(out io.Writer, process *process_module.Process, indent string, flat bool) error { - sb := &bytes.Buffer{} - if flat { - if process.ParentPID != "" { - _, _ = fmt.Fprintf(sb, "%s+ PID: %s\t\tType: %s\n", indent, process.PID, process.Type) - } else { - _, _ = fmt.Fprintf(sb, "%s+ PID: %s:%s\tType: %s\n", indent, process.ParentPID, process.PID, process.Type) + ctx.Status(http.StatusOK) + for _, profile := range profiles { + if _, err := ctx.Resp.Write([]byte(profile.Name())); err != nil { + log.Error("Unable to write out profile name: %v", err) + ctx.Error(http.StatusInternalServerError, err.Error()) + return } - } else { - _, _ = fmt.Fprintf(sb, "%s+ PID: %s\tType: %s\n", indent, process.PID, process.Type) - } - indent += "| " - - _, _ = fmt.Fprintf(sb, "%sDescription: %s\n", indent, process.Description) - _, _ = fmt.Fprintf(sb, "%sStart: %s\n", indent, process.Start) - - if len(process.Stacks) > 0 { - _, _ = fmt.Fprintf(sb, "%sGoroutines:\n", indent) - for _, stack := range process.Stacks { - indent := indent + " " - _, _ = fmt.Fprintf(sb, "%s+ Description: %s", indent, stack.Description) - if stack.Count > 1 { - _, _ = fmt.Fprintf(sb, "* %d", stack.Count) - } - _, _ = fmt.Fprintf(sb, "\n") - indent += "| " - if len(stack.Labels) > 0 { - _, _ = fmt.Fprintf(sb, "%sLabels: %q:%q", indent, stack.Labels[0].Name, stack.Labels[0].Value) - if len(stack.Labels) > 1 { - for _, label := range stack.Labels[1:] { - _, _ = fmt.Fprintf(sb, ", %q:%q", label.Name, label.Value) - } - } - _, _ = fmt.Fprintf(sb, "\n") - } - _, _ = fmt.Fprintf(sb, "%sStack:\n", indent) - indent += " " - for _, entry := range stack.Entry { - _, _ = fmt.Fprintf(sb, "%s+ %s\n", indent, entry.Function) - _, _ = fmt.Fprintf(sb, "%s| %s:%d\n", indent, entry.File, entry.Line) - } - } - } - if _, err := out.Write(sb.Bytes()); err != nil { - return err - } - sb.Reset() - if len(process.Children) > 0 { - if _, err := fmt.Fprintf(out, "%sChildren:\n", indent); err != nil { - return err - } - for _, child := range process.Children { - if err := writeProcess(out, child, indent+" ", flat); err != nil { - return err - } + if _, err := ctx.Resp.Write([]byte("\n")); err != nil { + log.Error("Unable to write out profile name: %v", err) + ctx.Error(http.StatusInternalServerError, err.Error()) + return } } - return nil + ctx.Resp.Flush() } diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 0a51000c70c90..d195a43b79277 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -8,7 +8,9 @@ import ( "fmt" "net/http" "runtime" + "runtime/pprof" "strconv" + "strings" "time" activities_model "code.gitea.io/gitea/models/activities" @@ -163,6 +165,8 @@ func Monitor(ctx *context.Context) { ctx.Data["Entries"] = cron.ListTasks() ctx.Data["Queues"] = queue.GetManager().ManagedQueues() + ctx.Data["Profiles"] = pprof.Profiles() + ctx.HTML(http.StatusOK, tplMonitor) } @@ -182,6 +186,14 @@ func GoroutineStacktrace(ctx *context.Context) { ctx.Data["GoroutineCount"] = goroutineCount ctx.Data["ProcessCount"] = processCount + sb := new(strings.Builder) + + if err := process.WriteProcesses(sb, processStacks, processCount, goroutineCount, "", false); err != nil { + ctx.ServerError("WriteProcesses", err) + return + } + + ctx.Data["StacktraceString"] = sb.String() ctx.HTML(http.StatusOK, tplStacktrace) } diff --git a/routers/web/admin/pprof.go b/routers/web/admin/pprof.go new file mode 100644 index 0000000000000..fcf188c993907 --- /dev/null +++ b/routers/web/admin/pprof.go @@ -0,0 +1,187 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package admin + +import ( + "fmt" + "net/http" + "runtime/pprof" + "runtime/trace" + "strconv" + "time" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" + + "github.com/felixge/fgprof" +) + +// PProfProcessStacktrace returns the stacktrace similar to GoroutineStacktrace but without rendering it +func PProfProcessStacktrace(ctx *context.Context) { + flat := ctx.FormBool("flat") + noSystem := ctx.FormBool("no-system") + + format := ctx.FormString("format") + jsonFormat := format == "json" + + start := time.Now() + filename := "process-stacktrace-" + strconv.FormatInt(start.Unix(), 10) + if jsonFormat { + filename += ".json" + } + + processStacks, processCount, goroutineCount, err := process.GetManager().ProcessStacktraces(flat, noSystem) + if err != nil { + ctx.ServerError("ProcessStacktraces", err) + } + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: filename, + LastModified: start, + }) + + if jsonFormat { + ctx.JSON(http.StatusOK, map[string]interface{}{ + "TotalNumberOfGoroutines": goroutineCount, + "TotalNumberOfProcesses": processCount, + "Processes": processStacks, + }) + return + } + + if err := process.WriteProcesses(ctx.Resp, processStacks, processCount, goroutineCount, "", flat); err != nil { + ctx.ServerError("WriteProcesses", err) + } +} + +// PProfFGProfile returns the Full Go Profile from fgprof +func PProfFGProfile(ctx *context.Context) { + durationStr := ctx.FormString("duration") + duration := 30 * time.Second + if durationStr != "" { + var err error + duration, err = time.ParseDuration(durationStr) + if err != nil { + ctx.Flash.Error(ctx.Tr("admin.monitor.pprof.duration_invalid")) + ctx.Redirect(setting.AppSubURL + "/admin/monitor") + return + } + } + + format := fgprof.Format(ctx.FormString("format")) + if format != fgprof.FormatFolded { + format = fgprof.FormatPprof + } + + start := time.Now() + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: "fgprof-profile-" + strconv.FormatInt(start.Unix(), 10), + LastModified: start, + }) + + fn := fgprof.Start(ctx.Resp, format) + + select { + case <-time.After(duration): + case <-ctx.Done(): + } + + err := fn() + if err != nil { + ctx.ServerError("fgprof.Write", err) + } +} + +// PProfCPUProfile returns the PProf CPU Profile +func PProfCPUProfile(ctx *context.Context) { + durationStr := ctx.FormString("duration") + duration := 30 * time.Second + if durationStr != "" { + var err error + duration, err = time.ParseDuration(durationStr) + if err != nil { + ctx.Flash.Error(ctx.Tr("admin.monitor.pprof.duration_invalid")) + ctx.Redirect(setting.AppSubURL + "/admin/monitor") + return + } + } + + start := time.Now() + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: "cpu-profile-" + strconv.FormatInt(start.Unix(), 10), + LastModified: start, + }) + + err := pprof.StartCPUProfile(ctx.Resp) + if err != nil { + ctx.ServerError("StartCPUProfile", err) + return + } + + select { + case <-time.After(duration): + case <-ctx.Done(): + } + pprof.StopCPUProfile() +} + +// PProfNamedProfile returns the PProf Profile +func PProfNamedProfile(ctx *context.Context) { + name := ctx.FormString("name") + profile := pprof.Lookup(name) + if profile == nil { + ctx.ServerError(fmt.Sprintf("pprof.Lookup(%s)", name), fmt.Errorf("missing profile: %s", name)) + return + } + + debug := ctx.FormInt("debug") + + start := time.Now() + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: name + "-profile-" + strconv.FormatInt(start.Unix(), 10), + LastModified: start, + }) + if err := profile.WriteTo(ctx.Resp, debug); err != nil { + ctx.ServerError(fmt.Sprintf("PProfNamedProfile(%s).WriteTo", name), err) + return + } +} + +// Trace returns a trace +func Trace(ctx *context.Context) { + durationStr := ctx.FormString("duration") + duration := 30 * time.Second + if durationStr != "" { + var err error + duration, err = time.ParseDuration(durationStr) + if err != nil { + ctx.Flash.Error(ctx.Tr("admin.monitor.pprof.duration_invalid")) + ctx.Redirect(setting.AppSubURL + "/admin/monitor") + return + } + } + + start := time.Now() + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: "trace-" + strconv.FormatInt(start.Unix(), 10), + LastModified: start, + }) + + err := trace.Start(ctx.Resp) + if err != nil { + ctx.ServerError("StartCPUProfile", err) + return + } + + select { + case <-time.After(duration): + case <-ctx.Done(): + } + trace.Stop() +} diff --git a/routers/web/web.go b/routers/web/web.go index 88e27ad678992..a6155a680a957 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -527,6 +527,11 @@ func RegisterRoutes(m *web.Route) { m.Group("/monitor", func() { m.Get("", admin.Monitor) m.Get("/stacktrace", admin.GoroutineStacktrace) + m.Get("/cpu-profile", admin.PProfCPUProfile) + m.Get("/profile", admin.PProfNamedProfile) + m.Get("/fgprof", admin.PProfFGProfile) + m.Get("/stacktrace-profile", admin.PProfProcessStacktrace) + m.Get("/trace", admin.Trace) m.Post("/cancel/{pid}", admin.MonitorCancel) m.Group("/queue/{qid}", func() { m.Get("", admin.Queue) diff --git a/templates/admin/monitor.tmpl b/templates/admin/monitor.tmpl index d53e9e18dcc8c..bca89ae8be2fe 100644 --- a/templates/admin/monitor.tmpl +++ b/templates/admin/monitor.tmpl @@ -35,6 +35,7 @@ {{template "admin/process" .}} + {{template "admin/pprof" .}}
diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl new file mode 100644 index 0000000000000..c1c1b55f48224 --- /dev/null +++ b/templates/admin/pprof.tmpl @@ -0,0 +1,96 @@ +

+ {{.locale.Tr "admin.monitor.pprof"}} +

+
+

{{.locale.Tr "admin.monitor.pprof.description"}}

+

{{.locale.Tr "admin.monitor.pprof.description_2"}}

+

{{.locale.Tr "admin.monitor.pprof.cpuprofile"}}

+
+
+
{{.locale.Tr "admin.monitor.pprof.cpuprofile.description"}}
+
+ + +
+ +
+
+

{{.locale.Tr "admin.monitor.pprof.fgprof"}}

+
+
+
{{.locale.Tr "admin.monitor.pprof.fgprof.description"}}
+
+ + +
+
+ + +
+ +
+
+

{{.locale.Tr "admin.monitor.stacktrace"}}

+
+
+
{{.locale.Tr "admin.monitor.pprof.stacktrace.description"}}
+
+
+ + +
+
+
+
+ + +
+
+
+ + +
+ +
+
+

{{.locale.Tr "admin.monitor.pprof.named_profiles"}}

+
+
+
{{.locale.Tr "admin.monitor.pprof.named_profiles.description"}}
+
+ + +
+
+ + +
+ +
+
+

{{.locale.Tr "admin.monitor.pprof.trace"}}

+
+
+
{{.locale.Tr "admin.monitor.pprof.trace.description"}}
+
+ + +
+ +
+
+
diff --git a/templates/admin/stacktrace.tmpl b/templates/admin/stacktrace.tmpl index 4e16036ae3a3c..8198cdf21ac58 100644 --- a/templates/admin/stacktrace.tmpl +++ b/templates/admin/stacktrace.tmpl @@ -7,6 +7,7 @@ {{.locale.Tr "admin.monitor.stacktrace"}}: {{.locale.Tr "admin.monitor.goroutines" .GoroutineCount}}
{{.locale.Tr "admin.monitor"}} + {{.locale.Tr "admin.monitor.stacktrace.download_stacktrace"}}
@@ -18,6 +19,16 @@
+
{{svg "octicon-x" 16 "close inside"}} diff --git a/web_src/js/features/admin/common.js b/web_src/js/features/admin/common.js index 092cf770be41d..fb936df9a1d20 100644 --- a/web_src/js/features/admin/common.js +++ b/web_src/js/features/admin/common.js @@ -215,4 +215,18 @@ export function initAdminCommon() { }); }); } + + $('#download-stacktrace').on('click', () => { + $('#download-stacktrace-modal').modal({ + closable: false, + onApprove() { + const textToSave = document.getElementById('stacktrace-to-download').innerText; + const hiddenElement = document.createElement('a'); + hiddenElement.href = `data:attachment/text,${encodeURI(textToSave)}`; + hiddenElement.target = '_blank'; + hiddenElement.download = 'stacktraces.txt'; + hiddenElement.click(); + } + }).modal('show'); + }); }