From 5093d7d19b89d630f4b0680b6b00408f2b77fcb1 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 3 Feb 2023 22:20:26 +0000 Subject: [PATCH 01/32] Add PProf to admin pages The PProf endpoint is too difficult for many users to use - this PR will attempt to create a nicer UI to help in the collection PProf data from Gitea. Signed-off-by: Andrew Thornton --- modules/process/stacktraces_processlist.go | 100 +++++++++++++++++++++ options/locale/locale_en-US.ini | 20 +++++ routers/private/manager_process.go | 92 +------------------ routers/web/admin/admin.go | 16 +++- routers/web/admin/pprof.go | 71 +++++++++++++++ routers/web/web.go | 2 + templates/admin/monitor.tmpl | 1 + templates/admin/pprof.tmpl | 28 ++++++ templates/admin/stacktrace.tmpl | 11 +++ web_src/js/features/admin/common.js | 14 +++ 10 files changed, 263 insertions(+), 92 deletions(-) create mode 100644 modules/process/stacktraces_processlist.go create mode 100644 routers/web/admin/pprof.go create mode 100644 templates/admin/pprof.tmpl diff --git a/modules/process/stacktraces_processlist.go b/modules/process/stacktraces_processlist.go new file mode 100644 index 0000000000000..33a451dc7bc1a --- /dev/null +++ b/modules/process/stacktraces_processlist.go @@ -0,0 +1,100 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package process + +import ( + "bytes" + "fmt" + "io" +) + +// WriteProcesses writes out processes to a provided writer +func WriteProcesses(out io.Writer, processes []*Process, processCount int, goroutineCount int64, indent string, flat bool) error { + if goroutineCount > 0 { + if _, err := fmt.Fprintf(out, "%sTotal Number of Goroutines: %d\n", indent, goroutineCount); err != nil { + return err + } + } + if _, err := fmt.Fprintf(out, "%sTotal Number of Processes: %d\n", indent, processCount); err != nil { + return err + } + if len(processes) > 0 { + if err := WriteProcess(out, processes[0], " ", flat); err != nil { + return err + } + } + if len(processes) > 1 { + for _, process := range processes[1:] { + if _, err := fmt.Fprintf(out, "%s | \n", indent); err != nil { + return err + } + if err := WriteProcess(out, process, " ", flat); err != nil { + return err + } + } + } + return nil +} + +// WriteProcess writes out a process to a provided writer +func WriteProcess(out io.Writer, process *Process, indent string, flat bool) error { + sb := &bytes.Buffer{} + if flat { + if process.ParentPID != "" { + _, _ = fmt.Fprintf(sb, "%s+ PID: %s\t\tType: %s\n", indent, process.PID, process.Type) + } else { + _, _ = fmt.Fprintf(sb, "%s+ PID: %s:%s\tType: %s\n", indent, process.ParentPID, process.PID, process.Type) + } + } else { + _, _ = fmt.Fprintf(sb, "%s+ PID: %s\tType: %s\n", indent, process.PID, process.Type) + } + indent += "| " + + _, _ = fmt.Fprintf(sb, "%sDescription: %s\n", indent, process.Description) + _, _ = fmt.Fprintf(sb, "%sStart: %s\n", indent, process.Start) + + if len(process.Stacks) > 0 { + _, _ = fmt.Fprintf(sb, "%sGoroutines:\n", indent) + for _, stack := range process.Stacks { + indent := indent + " " + _, _ = fmt.Fprintf(sb, "%s+ Description: %s", indent, stack.Description) + if stack.Count > 1 { + _, _ = fmt.Fprintf(sb, "* %d", stack.Count) + } + _, _ = fmt.Fprintf(sb, "\n") + indent += "| " + if len(stack.Labels) > 0 { + _, _ = fmt.Fprintf(sb, "%sLabels: %q:%q", indent, stack.Labels[0].Name, stack.Labels[0].Value) + + if len(stack.Labels) > 1 { + for _, label := range stack.Labels[1:] { + _, _ = fmt.Fprintf(sb, ", %q:%q", label.Name, label.Value) + } + } + _, _ = fmt.Fprintf(sb, "\n") + } + _, _ = fmt.Fprintf(sb, "%sStack:\n", indent) + indent += " " + for _, entry := range stack.Entry { + _, _ = fmt.Fprintf(sb, "%s+ %s\n", indent, entry.Function) + _, _ = fmt.Fprintf(sb, "%s| %s:%d\n", indent, entry.File, entry.Line) + } + } + } + if _, err := out.Write(sb.Bytes()); err != nil { + return err + } + sb.Reset() + if len(process.Children) > 0 { + if _, err := fmt.Fprintf(out, "%sChildren:\n", indent); err != nil { + return err + } + for _, child := range process.Children { + if err := WriteProcess(out, child, indent+" ", flat); err != nil { + return err + } + } + } + return nil +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 26217293a5efa..2e767b0067b4e 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2947,6 +2947,7 @@ monitor.previous = Previous Time monitor.execute_times = Executions monitor.process = Running Processes monitor.stacktrace = Stacktraces +monitor.stacktrace.download_stacktrace = Download Stacktrace monitor.goroutines = %d Goroutines monitor.desc = Description monitor.start = Start Time @@ -2956,6 +2957,25 @@ monitor.process.cancel = Cancel process monitor.process.cancel_desc = Cancelling a process may cause data loss monitor.process.cancel_notices = Cancel: %s? monitor.process.children = Children + +monitor.pprof = PProf Profiles +monitor.pprof.download = Download + +monitor.pprof.cpuprofile = CPU Profile +monitor.pprof.cpuprofile.duration = Duration +monitor.pprof.cpuprofile.duration_placeholder = e.g. 30s +monitor.pprof.cpuprofile.duration_invalid = Invalid duration - duration must be a golang duration string + +monitor.pprof.debug = Debug level +monitor.pprof.debug_placeholder = e.g. 0 + +monitor.pprof.allocs = allocs - Memory Allocations +monitor.pprof.block = block - Blocking Stack Traces +monitor.pprof.goroutine = goroutine - Goroutine Stack Traces +monitor.pprof.heap = heap - Memory Allocations of live objects on the heap +monitor.pprof.mutex = mutex - Stack traces of holding contended mutexes +monitor.pprof.threadcreate = threadcreate - Stack traces that led to new OS threads + monitor.queues = Queues monitor.queue = Queue: %s monitor.queue.name = Name diff --git a/routers/private/manager_process.go b/routers/private/manager_process.go index a5993bf3718d3..99246b603b145 100644 --- a/routers/private/manager_process.go +++ b/routers/private/manager_process.go @@ -4,9 +4,7 @@ package private import ( - "bytes" "fmt" - "io" "net/http" "runtime" "time" @@ -60,7 +58,7 @@ func Processes(ctx *context.PrivateContext) { ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") ctx.Resp.WriteHeader(http.StatusOK) - if err := writeProcesses(ctx.Resp, processes, processCount, goroutineCount, "", flat); err != nil { + if err := process_module.WriteProcesses(ctx.Resp, processes, processCount, goroutineCount, "", flat); err != nil { log.Error("Unable to write out process stacktrace: %v", err) if !ctx.Written() { ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -70,91 +68,3 @@ func Processes(ctx *context.PrivateContext) { return } } - -func writeProcesses(out io.Writer, processes []*process_module.Process, processCount int, goroutineCount int64, indent string, flat bool) error { - if goroutineCount > 0 { - if _, err := fmt.Fprintf(out, "%sTotal Number of Goroutines: %d\n", indent, goroutineCount); err != nil { - return err - } - } - if _, err := fmt.Fprintf(out, "%sTotal Number of Processes: %d\n", indent, processCount); err != nil { - return err - } - if len(processes) > 0 { - if err := writeProcess(out, processes[0], " ", flat); err != nil { - return err - } - } - if len(processes) > 1 { - for _, process := range processes[1:] { - if _, err := fmt.Fprintf(out, "%s | \n", indent); err != nil { - return err - } - if err := writeProcess(out, process, " ", flat); err != nil { - return err - } - } - } - return nil -} - -func writeProcess(out io.Writer, process *process_module.Process, indent string, flat bool) error { - sb := &bytes.Buffer{} - if flat { - if process.ParentPID != "" { - _, _ = fmt.Fprintf(sb, "%s+ PID: %s\t\tType: %s\n", indent, process.PID, process.Type) - } else { - _, _ = fmt.Fprintf(sb, "%s+ PID: %s:%s\tType: %s\n", indent, process.ParentPID, process.PID, process.Type) - } - } else { - _, _ = fmt.Fprintf(sb, "%s+ PID: %s\tType: %s\n", indent, process.PID, process.Type) - } - indent += "| " - - _, _ = fmt.Fprintf(sb, "%sDescription: %s\n", indent, process.Description) - _, _ = fmt.Fprintf(sb, "%sStart: %s\n", indent, process.Start) - - if len(process.Stacks) > 0 { - _, _ = fmt.Fprintf(sb, "%sGoroutines:\n", indent) - for _, stack := range process.Stacks { - indent := indent + " " - _, _ = fmt.Fprintf(sb, "%s+ Description: %s", indent, stack.Description) - if stack.Count > 1 { - _, _ = fmt.Fprintf(sb, "* %d", stack.Count) - } - _, _ = fmt.Fprintf(sb, "\n") - indent += "| " - if len(stack.Labels) > 0 { - _, _ = fmt.Fprintf(sb, "%sLabels: %q:%q", indent, stack.Labels[0].Name, stack.Labels[0].Value) - - if len(stack.Labels) > 1 { - for _, label := range stack.Labels[1:] { - _, _ = fmt.Fprintf(sb, ", %q:%q", label.Name, label.Value) - } - } - _, _ = fmt.Fprintf(sb, "\n") - } - _, _ = fmt.Fprintf(sb, "%sStack:\n", indent) - indent += " " - for _, entry := range stack.Entry { - _, _ = fmt.Fprintf(sb, "%s+ %s\n", indent, entry.Function) - _, _ = fmt.Fprintf(sb, "%s| %s:%d\n", indent, entry.File, entry.Line) - } - } - } - if _, err := out.Write(sb.Bytes()); err != nil { - return err - } - sb.Reset() - if len(process.Children) > 0 { - if _, err := fmt.Fprintf(out, "%sChildren:\n", indent); err != nil { - return err - } - for _, child := range process.Children { - if err := writeProcess(out, child, indent+" ", flat); err != nil { - return err - } - } - } - return nil -} diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 0a51000c70c90..1539fe224a7e5 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -8,7 +8,9 @@ import ( "fmt" "net/http" "runtime" + "runtime/pprof" "strconv" + "strings" "time" activities_model "code.gitea.io/gitea/models/activities" @@ -159,10 +161,14 @@ func Monitor(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.monitor") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminMonitor"] = true - ctx.Data["Processes"], ctx.Data["ProcessCount"] = process.GetManager().Processes(false, true) + processes, processCount := process.GetManager().Processes(false, true) ctx.Data["Entries"] = cron.ListTasks() ctx.Data["Queues"] = queue.GetManager().ManagedQueues() + ctx.Data["Processes"], ctx.Data["ProcessCount"] = processes, processCount + + ctx.Data["Profiles"] = pprof.Profiles() + ctx.HTML(http.StatusOK, tplMonitor) } @@ -182,6 +188,14 @@ func GoroutineStacktrace(ctx *context.Context) { ctx.Data["GoroutineCount"] = goroutineCount ctx.Data["ProcessCount"] = processCount + sb := new(strings.Builder) + + if err := process.WriteProcesses(sb, processStacks, processCount, goroutineCount, "", false); err != nil { + ctx.ServerError("WriteProcesses", err) + return + } + + ctx.Data["StacktraceString"] = sb.String() ctx.HTML(http.StatusOK, tplStacktrace) } diff --git a/routers/web/admin/pprof.go b/routers/web/admin/pprof.go new file mode 100644 index 0000000000000..acf740b9e7ddd --- /dev/null +++ b/routers/web/admin/pprof.go @@ -0,0 +1,71 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package admin + +import ( + "fmt" + "runtime/pprof" + "strconv" + "time" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" +) + +// PProfCPUProfile returns the PProf CPU Profile +func PProfCPUProfile(ctx *context.Context) { + durationStr := ctx.FormString("duration") + duration := 30 * time.Second + if durationStr != "" { + var err error + duration, err = time.ParseDuration(durationStr) + if err != nil { + ctx.Flash.Error(ctx.Tr("monitor.pprof.cpuprofile.duration_invalid")) + ctx.Redirect(setting.AppSubURL + "/admin/monitor") + return + } + } + + start := time.Now() + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: "cpu-profile-" + strconv.FormatInt(start.Unix(), 10), + LastModified: start, + }) + + err := pprof.StartCPUProfile(ctx.Resp) + if err != nil { + ctx.ServerError("StartCPUProfile", err) + return + } + + select { + case <-time.After(duration): + case <-ctx.Done(): + } + pprof.StopCPUProfile() +} + +// PProfNamedProfile returns the PProf Profile +func PProfNamedProfile(ctx *context.Context) { + name := ctx.FormString("name") + profile := pprof.Lookup(name) + if profile == nil { + ctx.ServerError(fmt.Sprintf("pprof.Lookup(%s)", name), fmt.Errorf("missing profile: %s", name)) + return + } + + debug := ctx.FormInt("debug") + + start := time.Now() + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: name + "-profile-" + strconv.FormatInt(start.Unix(), 10), + LastModified: start, + }) + if err := profile.WriteTo(ctx.Resp, debug); err != nil { + ctx.ServerError(fmt.Sprintf("PProfNamedProfile(%s).WriteTo", name), err) + return + } +} diff --git a/routers/web/web.go b/routers/web/web.go index b7128fc3a9fcc..5df2a904aee33 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -522,6 +522,8 @@ func RegisterRoutes(m *web.Route) { m.Group("/monitor", func() { m.Get("", admin.Monitor) m.Get("/stacktrace", admin.GoroutineStacktrace) + m.Get("/cpu-profile", admin.PProfCPUProfile) + m.Get("/profile", admin.PProfNamedProfile) m.Post("/cancel/{pid}", admin.MonitorCancel) m.Group("/queue/{qid}", func() { m.Get("", admin.Queue) diff --git a/templates/admin/monitor.tmpl b/templates/admin/monitor.tmpl index d53e9e18dcc8c..bca89ae8be2fe 100644 --- a/templates/admin/monitor.tmpl +++ b/templates/admin/monitor.tmpl @@ -35,6 +35,7 @@ {{template "admin/process" .}} + {{template "admin/pprof" .}}
diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl new file mode 100644 index 0000000000000..2d1f212389c9c --- /dev/null +++ b/templates/admin/pprof.tmpl @@ -0,0 +1,28 @@ +

+ {{.locale.Tr "admin.monitor.pprof"}} +

+
+

{{.locale.Tr "admin.monitor.pprof.cpuprofile"}}

+
+
+
+ + +
+ +
+
+ {{range .Profiles}} +

{{$.locale.Tr (printf "admin.monitor.pprof.%s" .Name)}}

+
+
+ +
+ + +
+ +
+
+ {{end}} +
diff --git a/templates/admin/stacktrace.tmpl b/templates/admin/stacktrace.tmpl index 4e16036ae3a3c..8198cdf21ac58 100644 --- a/templates/admin/stacktrace.tmpl +++ b/templates/admin/stacktrace.tmpl @@ -7,6 +7,7 @@ {{.locale.Tr "admin.monitor.stacktrace"}}: {{.locale.Tr "admin.monitor.goroutines" .GoroutineCount}}
{{.locale.Tr "admin.monitor"}} + {{.locale.Tr "admin.monitor.stacktrace.download_stacktrace"}}
@@ -18,6 +19,16 @@
+
{{svg "octicon-x" 16 "close inside"}} diff --git a/web_src/js/features/admin/common.js b/web_src/js/features/admin/common.js index 2438fcf62b465..ee38d4dc62907 100644 --- a/web_src/js/features/admin/common.js +++ b/web_src/js/features/admin/common.js @@ -209,4 +209,18 @@ export function initAdminCommon() { }); }); } + + $('#download-stacktrace').on('click', () => { + $('#download-stacktrace-modal').modal({ + closable: false, + onApprove() { + const textToSave = document.getElementById('stacktrace-to-download').innerText; + const hiddenElement = document.createElement('a'); + hiddenElement.href = `data:attachment/text,${encodeURI(textToSave)}`; + hiddenElement.target = '_blank'; + hiddenElement.download = 'stacktraces.txt'; + hiddenElement.click(); + } + }).modal('show'); + }); } From 9cfcbb44f6af81a16acd2c136dcb62862ed61788 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 11:19:32 +0000 Subject: [PATCH 02/32] Update pprof.tmpl --- templates/admin/pprof.tmpl | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 2d1f212389c9c..6281e9adc8209 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -12,17 +12,22 @@
- {{range .Profiles}} -

{{$.locale.Tr (printf "admin.monitor.pprof.%s" .Name)}}

-
-
- -
- - -
- +

{{.locale.Tr "admin.monitor.pprof.named_profiles"}}

+ +
+
+ + +
+
+ +
- - {{end}} + +
+
From f22aa21f8a851fb9f5ba258ba38a33b17f9c89df Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 11:21:03 +0000 Subject: [PATCH 03/32] Update locale_en-US.ini --- options/locale/locale_en-US.ini | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 2e767b0067b4e..2fc2eb879bdf2 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2966,15 +2966,10 @@ monitor.pprof.cpuprofile.duration = Duration monitor.pprof.cpuprofile.duration_placeholder = e.g. 30s monitor.pprof.cpuprofile.duration_invalid = Invalid duration - duration must be a golang duration string -monitor.pprof.debug = Debug level -monitor.pprof.debug_placeholder = e.g. 0 - -monitor.pprof.allocs = allocs - Memory Allocations -monitor.pprof.block = block - Blocking Stack Traces -monitor.pprof.goroutine = goroutine - Goroutine Stack Traces -monitor.pprof.heap = heap - Memory Allocations of live objects on the heap -monitor.pprof.mutex = mutex - Stack traces of holding contended mutexes -monitor.pprof.threadcreate = threadcreate - Stack traces that led to new OS threads +monitor.pprof.named_profiles = Named Profiles +monitor.pprof.named_profiles.name = Name +monitor.pprof.named_profiles.debug = Debug level +monitor.pprof.named_profiles.debug_placeholder = e.g. 0 monitor.queues = Queues monitor.queue = Queue: %s From b9989b140a851ef7ce2d8fb636a84d004fbbfa44 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 11:41:39 +0000 Subject: [PATCH 04/32] Add FGProf handler --- routers/web/admin/pprof.go | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/routers/web/admin/pprof.go b/routers/web/admin/pprof.go index acf740b9e7ddd..3326732bc9f36 100644 --- a/routers/web/admin/pprof.go +++ b/routers/web/admin/pprof.go @@ -11,8 +11,50 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" + + "github.com/felixge/fgprof" ) +// PProfFGProfile returns the Full Go Profile from fgprof +func PProfFGProfile(ctx *context.Context) { + durationStr := ctx.FormString("duration") + duration := 30 * time.Second + if durationStr != "" { + var err error + duration, err = time.ParseDuration(durationStr) + if err != nil { + ctx.Flash.Error(ctx.Tr("monitor.pprof.cpuprofile.duration_invalid")) + ctx.Redirect(setting.AppSubURL + "/admin/monitor") + return + } + } + + format := ctx.FormString("format") + if format != fgprof.FormatFolded { + format = fgprof.FormatPprof + } + + start := time.Now() + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: "fgprof-profile-" + strconv.FormatInt(start.Unix(), 10), + LastModified: start, + }) + + fn := fgprof.Start(ctx.Resp, format) + + select { + case <-time.After(duration): + case <-ctx.Done(): + } + + err := fn() + if err != nil { + ctx.ServerError("fgprof.Write", err) + } +} + + // PProfCPUProfile returns the PProf CPU Profile func PProfCPUProfile(ctx *context.Context) { durationStr := ctx.FormString("duration") From 6451d01b58ae15e1504fcec5826d9db8fd0e38bc Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 11:42:53 +0000 Subject: [PATCH 05/32] wire in PProfFGProfile --- routers/web/web.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/web/web.go b/routers/web/web.go index 5df2a904aee33..f7704ebc753d4 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -524,6 +524,7 @@ func RegisterRoutes(m *web.Route) { m.Get("/stacktrace", admin.GoroutineStacktrace) m.Get("/cpu-profile", admin.PProfCPUProfile) m.Get("/profile", admin.PProfNamedProfile) + m.Get("/fgprof", admin.PProfFGProfile) m.Post("/cancel/{pid}", admin.MonitorCancel) m.Group("/queue/{qid}", func() { m.Get("", admin.Queue) From 351149b8d2599e9c292a9bbfc5f5c83d2f09b2de Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 11:56:02 +0000 Subject: [PATCH 06/32] Wire in FGProf --- templates/admin/pprof.tmpl | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 6281e9adc8209..1e51fb940acf7 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -6,8 +6,25 @@
- - + + +
+ +
+
+

{{.locale.Tr "admin.monitor.pprof.fgprofile"}}

+
+
+
+ + +
+
+ +
From 7941fbbbee452208afa702cbaf29a3dce33256e9 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 11:59:22 +0000 Subject: [PATCH 07/32] Update locale_en-US.ini --- options/locale/locale_en-US.ini | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 2fc2eb879bdf2..83cdc0d21fc84 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2960,11 +2960,14 @@ monitor.process.children = Children monitor.pprof = PProf Profiles monitor.pprof.download = Download +monitor.pprof.duration = Duration +monitor.pprof.duration_placeholder = e.g. 30s +monitor.pprof.duration_invalid = Invalid duration - duration must be a golang duration string monitor.pprof.cpuprofile = CPU Profile -monitor.pprof.cpuprofile.duration = Duration -monitor.pprof.cpuprofile.duration_placeholder = e.g. 30s -monitor.pprof.cpuprofile.duration_invalid = Invalid duration - duration must be a golang duration string + +monitor.pprof.fgprof = Full Go Profile +monitor.pprof.fgprof.format = Format monitor.pprof.named_profiles = Named Profiles monitor.pprof.named_profiles.name = Name From 0ef07f68cc3dea0d3af5cfc4d4a3fb688fccb1f3 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 11:59:43 +0000 Subject: [PATCH 08/32] Update pprof.tmpl --- templates/admin/pprof.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 1e51fb940acf7..3a57b853c2efe 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -12,7 +12,7 @@
-

{{.locale.Tr "admin.monitor.pprof.fgprofile"}}

+

{{.locale.Tr "admin.monitor.pprof.fgprof"}}

From 8d0d93e82e44adaf1dccb7d39420a5b80031ff6e Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 12:00:25 +0000 Subject: [PATCH 09/32] Update pprof.go --- routers/web/admin/pprof.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/web/admin/pprof.go b/routers/web/admin/pprof.go index 3326732bc9f36..5a4096606e55e 100644 --- a/routers/web/admin/pprof.go +++ b/routers/web/admin/pprof.go @@ -23,7 +23,7 @@ func PProfFGProfile(ctx *context.Context) { var err error duration, err = time.ParseDuration(durationStr) if err != nil { - ctx.Flash.Error(ctx.Tr("monitor.pprof.cpuprofile.duration_invalid")) + ctx.Flash.Error(ctx.Tr("monitor.pprof.duration_invalid")) ctx.Redirect(setting.AppSubURL + "/admin/monitor") return } @@ -63,7 +63,7 @@ func PProfCPUProfile(ctx *context.Context) { var err error duration, err = time.ParseDuration(durationStr) if err != nil { - ctx.Flash.Error(ctx.Tr("monitor.pprof.cpuprofile.duration_invalid")) + ctx.Flash.Error(ctx.Tr("monitor.pprof.duration_invalid")) ctx.Redirect(setting.AppSubURL + "/admin/monitor") return } From 768a9607be40e1f8c80c3f1d96dede1dd921e8d1 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 14:51:12 +0000 Subject: [PATCH 10/32] Update pprof.go --- routers/web/admin/pprof.go | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/routers/web/admin/pprof.go b/routers/web/admin/pprof.go index 5a4096606e55e..097b5ee4ed44d 100644 --- a/routers/web/admin/pprof.go +++ b/routers/web/admin/pprof.go @@ -15,6 +15,45 @@ import ( "github.com/felixge/fgprof" ) +// PProfProcessStacktrace returns the stacktrace similar to GoroutineStacktrace but without rendering it +func PProfProcessStacktrace(ctx *context.Context) { + flat := ctx.FormBool("flat") + + format := ctx.FormString("format") + jsonFormat := format == "json" + + start := time.Now() + filename := "process-stacktrace-" + strconv.FormatInt(start.Unix(), 10) + if jsonFormat { + filename += ".json" + } + + processStacks, processCount, goroutineCount, err := process.GetManager().ProcessStacktraces(false, false) + if err != nil { + ctx.ServerError("ProcessStacktraces", err) + } + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: filename, + LastModified: start, + }) + + if jsonFormat { + ctx.JSON(http.StatusOK, map[string]interface{}{ + "TotalNumberOfGoroutines": goroutineCount, + "TotalNumberOfProcesses": processCount, + "Processes": processes, + }) + return + } + + if err := process.WriteProcesses(ctx.Resp, processStacks, processCount, goroutineCount, "", false); err != nil { + ctx.ServerError("WriteProcesses", err) + return + } +} + + // PProfFGProfile returns the Full Go Profile from fgprof func PProfFGProfile(ctx *context.Context) { durationStr := ctx.FormString("duration") From 82169ca0d7a6891c4543149145149b5394ba5a39 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 17:18:01 +0000 Subject: [PATCH 11/32] Update web.go --- routers/web/web.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/web/web.go b/routers/web/web.go index f7704ebc753d4..11e189f3f5db7 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -525,6 +525,7 @@ func RegisterRoutes(m *web.Route) { m.Get("/cpu-profile", admin.PProfCPUProfile) m.Get("/profile", admin.PProfNamedProfile) m.Get("/fgprof", admin.PProfFGProfile) + m.Get("/stacktrace-profile", admin.PProfProcessStacktrace) m.Post("/cancel/{pid}", admin.MonitorCancel) m.Group("/queue/{qid}", func() { m.Get("", admin.Queue) From ba894a10a6e5ae2166c2d3c35d31b2d559de4502 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 17:22:37 +0000 Subject: [PATCH 12/32] Update pprof.go --- routers/web/admin/pprof.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routers/web/admin/pprof.go b/routers/web/admin/pprof.go index 097b5ee4ed44d..f58989e6d6a4b 100644 --- a/routers/web/admin/pprof.go +++ b/routers/web/admin/pprof.go @@ -18,6 +18,7 @@ import ( // PProfProcessStacktrace returns the stacktrace similar to GoroutineStacktrace but without rendering it func PProfProcessStacktrace(ctx *context.Context) { flat := ctx.FormBool("flat") + noSystem := ctx.FormBool("no-system") format := ctx.FormString("format") jsonFormat := format == "json" @@ -28,7 +29,7 @@ func PProfProcessStacktrace(ctx *context.Context) { filename += ".json" } - processStacks, processCount, goroutineCount, err := process.GetManager().ProcessStacktraces(false, false) + processStacks, processCount, goroutineCount, err := process.GetManager().ProcessStacktraces(flat, noSystem) if err != nil { ctx.ServerError("ProcessStacktraces", err) } @@ -47,7 +48,7 @@ func PProfProcessStacktrace(ctx *context.Context) { return } - if err := process.WriteProcesses(ctx.Resp, processStacks, processCount, goroutineCount, "", false); err != nil { + if err := process.WriteProcesses(ctx.Resp, processStacks, processCount, goroutineCount, "", flat); err != nil { ctx.ServerError("WriteProcesses", err) return } From 75d96e8cf2f7f02dea8f16817158f862e7536801 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 17:26:09 +0000 Subject: [PATCH 13/32] Update pprof.tmpl --- templates/admin/pprof.tmpl | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 3a57b853c2efe..0dc87721941b5 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -29,6 +29,31 @@
+

{{.locale.Tr "admin.monitor.stacktraces"}}

+
+
+
+
+ + +
+
+
+
+ + +
+
+
+ + +
+ +
+

{{.locale.Tr "admin.monitor.pprof.named_profiles"}}

From 381f12a9e26ae8d42163489b8f05fdc3fc7bbda0 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 17:30:00 +0000 Subject: [PATCH 14/32] Update pprof.tmpl --- templates/admin/pprof.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 0dc87721941b5..bbe7d2da4cddc 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -35,13 +35,13 @@
- +
- +
From 2fc5d72dd02dfbe6274a1d532940e730f81029d5 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 4 Feb 2023 17:30:32 +0000 Subject: [PATCH 15/32] Update locale_en-US.ini --- options/locale/locale_en-US.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 83cdc0d21fc84..6fe631bda0c99 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2974,6 +2974,10 @@ monitor.pprof.named_profiles.name = Name monitor.pprof.named_profiles.debug = Debug level monitor.pprof.named_profiles.debug_placeholder = e.g. 0 +monitor.pprof.stacktraces.flat = Do not nest processes under their parents +monitor.pprof.stacktraces.no-system = Do not include go-routines associated with system processes +monitor.pprof.stacktraces.format = Format + monitor.queues = Queues monitor.queue = Queue: %s monitor.queue.name = Name From 79756cd5ab08ff4319cd7c9712fe2c34cb533633 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 4 Feb 2023 19:28:35 +0000 Subject: [PATCH 16/32] placate lint Signed-off-by: Andrew Thornton --- options/locale/locale_en-US.ini | 6 +++--- routers/web/admin/pprof.go | 20 +++++++++--------- templates/admin/pprof.tmpl | 36 ++++++++++++++++----------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 6fe631bda0c99..1be045454bd85 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2974,9 +2974,9 @@ monitor.pprof.named_profiles.name = Name monitor.pprof.named_profiles.debug = Debug level monitor.pprof.named_profiles.debug_placeholder = e.g. 0 -monitor.pprof.stacktraces.flat = Do not nest processes under their parents -monitor.pprof.stacktraces.no-system = Do not include go-routines associated with system processes -monitor.pprof.stacktraces.format = Format +monitor.pprof.stacktrace.flat = Do not nest processes under their parents +monitor.pprof.stacktrace.no-system = Do not include go-routines associated with system processes +monitor.pprof.stacktrace.format = Format monitor.queues = Queues monitor.queue = Queue: %s diff --git a/routers/web/admin/pprof.go b/routers/web/admin/pprof.go index f58989e6d6a4b..75c3dff7fbc7e 100644 --- a/routers/web/admin/pprof.go +++ b/routers/web/admin/pprof.go @@ -5,13 +5,15 @@ package admin import ( "fmt" + "net/http" "runtime/pprof" "strconv" "time" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" - + "github.com/felixge/fgprof" ) @@ -19,7 +21,7 @@ import ( func PProfProcessStacktrace(ctx *context.Context) { flat := ctx.FormBool("flat") noSystem := ctx.FormBool("no-system") - + format := ctx.FormString("format") jsonFormat := format == "json" @@ -28,7 +30,7 @@ func PProfProcessStacktrace(ctx *context.Context) { if jsonFormat { filename += ".json" } - + processStacks, processCount, goroutineCount, err := process.GetManager().ProcessStacktraces(flat, noSystem) if err != nil { ctx.ServerError("ProcessStacktraces", err) @@ -43,7 +45,7 @@ func PProfProcessStacktrace(ctx *context.Context) { ctx.JSON(http.StatusOK, map[string]interface{}{ "TotalNumberOfGoroutines": goroutineCount, "TotalNumberOfProcesses": processCount, - "Processes": processes, + "Processes": processStacks, }) return } @@ -54,7 +56,6 @@ func PProfProcessStacktrace(ctx *context.Context) { } } - // PProfFGProfile returns the Full Go Profile from fgprof func PProfFGProfile(ctx *context.Context) { durationStr := ctx.FormString("duration") @@ -68,8 +69,8 @@ func PProfFGProfile(ctx *context.Context) { return } } - - format := ctx.FormString("format") + + format := fgprof.Format(ctx.FormString("format")) if format != fgprof.FormatFolded { format = fgprof.FormatPprof } @@ -82,19 +83,18 @@ func PProfFGProfile(ctx *context.Context) { }) fn := fgprof.Start(ctx.Resp, format) - + select { case <-time.After(duration): case <-ctx.Done(): } - + err := fn() if err != nil { ctx.ServerError("fgprof.Write", err) } } - // PProfCPUProfile returns the PProf CPU Profile func PProfCPUProfile(ctx *context.Context) { durationStr := ctx.FormString("duration") diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index bbe7d2da4cddc..23ad1f4ff2182 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -29,23 +29,23 @@
-

{{.locale.Tr "admin.monitor.stacktraces"}}

+

{{.locale.Tr "admin.monitor.stacktrace"}}

-
-
- - -
-
-
-
- - -
-
+
+
+ + +
+
+
+
+ + +
+
- + + +
- - + +
From 6247e72648f57ab382820ac2d8d0731bdfecf846 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 4 Feb 2023 20:44:13 +0000 Subject: [PATCH 17/32] Add pprof endpoints to manager too Signed-off-by: Andrew Thornton --- cmd/manager.go | 107 +++++++++++++++++++++++++++++ modules/context/private.go | 2 +- modules/private/manager.go | 38 ++++++++++ routers/private/internal.go | 5 ++ routers/private/manager_process.go | 32 +++++++++ 5 files changed, 183 insertions(+), 1 deletion(-) diff --git a/cmd/manager.go b/cmd/manager.go index cdfe509075f3b..b2e3494699713 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -26,6 +26,10 @@ var ( subcmdFlushQueues, subcmdLogging, subCmdProcesses, + subCmdCPUProfile, + subCmdFGProfile, + subCmdListNamedProfiles, + subCmdNamedProfile, }, } subcmdShutdown = cli.Command{ @@ -97,6 +101,61 @@ var ( }, }, } + subCmdCPUProfile = cli.Command{ + Name: "cpu-profile", + Usage: "Return PProf CPU profile", + Action: runCPUProfile, + Flags: []cli.Flag{ + cli.DurationFlag{ + Name: "duration", + Usage: "Duration to collect CPU Profile over", + Value: 30 * time.Second, + }, + }, + } + subCmdFGProfile = cli.Command{ + Name: "fg-profile", + Usage: "Return PProf Full Go profile", + Action: runFGProfile, + Flags: []cli.Flag{ + cli.DurationFlag{ + Name: "duration", + Usage: "Duration to collect CPU Profile over", + Value: 30 * time.Second, + }, + cli.StringFlag{ + Name: "format", + Usage: "Format to return the profile in: pprof, folded", + Value: "pprof", + }, + }, + } + subCmdNamedProfile = cli.Command{ + Name: "named-profile", + Usage: "Return PProf named profile", + Action: runNamedProfile, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "name", + Usage: "Name of profile to run", + }, + cli.IntFlag{ + Name: "debug-level", + Usage: "Debug level for the profile", + }, + }, + } + subCmdListNamedProfiles = cli.Command{ + Name: "list-named-profiles", + Usage: "Return PProf list of named profiles", + Action: runListNamedProfile, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "json", + Usage: "Output as json", + }, + }, + } ) func runShutdown(c *cli.Context) error { @@ -157,3 +216,51 @@ func runProcesses(c *cli.Context) error { return nil } + +func runCPUProfile(c *cli.Context) error { + ctx, cancel := installSignals() + defer cancel() + setup("manager", c.Bool("debug")) + statusCode, msg := private.CPUProfile(ctx, os.Stdout, c.Duration("duration")) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + return nil +} + +func runFGProfile(c *cli.Context) error { + ctx, cancel := installSignals() + defer cancel() + setup("manager", c.Bool("debug")) + statusCode, msg := private.FGProfile(ctx, os.Stdout, c.Duration("duration"), c.String("format")) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + return nil +} + +func runNamedProfile(c *cli.Context) error { + ctx, cancel := installSignals() + defer cancel() + setup("manager", c.Bool("debug")) + statusCode, msg := private.NamedProfile(ctx, os.Stdout, c.String("name"), c.Int("debug-level")) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + return nil +} + +func runListNamedProfile(c *cli.Context) error { + ctx, cancel := installSignals() + defer cancel() + setup("manager", c.Bool("debug")) + statusCode, msg := private.ListNamedProfiles(ctx, os.Stdout, c.Bool("json")) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + return nil +} diff --git a/modules/context/private.go b/modules/context/private.go index 24f50fa4713e6..012a50897276b 100644 --- a/modules/context/private.go +++ b/modules/context/private.go @@ -47,7 +47,7 @@ var privateContextKey interface{} = "default_private_context" // WithPrivateContext set up private context in request func WithPrivateContext(req *http.Request, ctx *PrivateContext) *http.Request { - return req.WithContext(context.WithValue(req.Context(), privateContextKey, ctx)) + return req.WithContext(context.WithValue(context.WithValue(req.Context(), privateContextKey, ctx), contextKey, ctx.Context)) } // GetPrivateContext returns a context for Private routes diff --git a/modules/private/manager.go b/modules/private/manager.go index bbf470cd7ad30..4733798b1ea93 100644 --- a/modules/private/manager.go +++ b/modules/private/manager.go @@ -230,3 +230,41 @@ func Processes(ctx context.Context, out io.Writer, flat, noSystem, stacktraces, } return http.StatusOK, "" } + +// CPUProfile returns a cpu profile from Gitea +func CPUProfile(ctx context.Context, out io.Writer, duration time.Duration) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/cpu-profile?duration=%s", url.QueryEscape(duration.String())) + return commonGet(ctx, out, reqURL) +} + +// FGProfile returns the full go profile from Gitea +func FGProfile(ctx context.Context, out io.Writer, duration time.Duration, format string) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/fgprof?duration=%s&format=%s", url.QueryEscape(duration.String()), url.QueryEscape(format)) + return commonGet(ctx, out, reqURL) +} + +// NamedProfile returns the named profile from Gitea +func NamedProfile(ctx context.Context, out io.Writer, name string, debugLevel int) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/profile?name=%s&debug=%d", url.QueryEscape(name), debugLevel) + return commonGet(ctx, out, reqURL) +} + +// ListNamedProfiles returns a list of named profiles +func ListNamedProfiles(ctx context.Context, out io.Writer, json bool) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/list-profiles?json=%t", json) + return commonGet(ctx, out, reqURL) +} + +func commonGet(ctx context.Context, out io.Writer, reqURL string) (int, string) { + req := newInternalRequest(ctx, reqURL, "GET") + resp, err := req.Response() + if err != nil { + return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) + } + defer resp.Body.Close() + _, err = io.Copy(out, resp.Body) + if err != nil { + return http.StatusInternalServerError, err.Error() + } + return resp.StatusCode, "" +} diff --git a/routers/private/internal.go b/routers/private/internal.go index 306e4ffb0040f..71506e96a2178 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/web/admin" "gitea.com/go-chi/binding" chi_middleware "github.com/go-chi/chi/v5/middleware" @@ -75,6 +76,10 @@ func Routes() *web.Route { r.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger) r.Post("/manager/remove-logger/{group}/{name}", RemoveLogger) r.Get("/manager/processes", Processes) + r.Get("/manager/cpu-profile", admin.PProfCPUProfile) + r.Get("/manager/profile", admin.PProfNamedProfile) + r.Get("/manager/fgprof", admin.PProfFGProfile) + r.Get("/manager/list-profiles", ListProfiles) r.Post("/mail/send", SendEmail) r.Post("/restore_repo", RestoreRepo) diff --git a/routers/private/manager_process.go b/routers/private/manager_process.go index 99246b603b145..58873eb65d275 100644 --- a/routers/private/manager_process.go +++ b/routers/private/manager_process.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "runtime" + "runtime/pprof" "time" "code.gitea.io/gitea/modules/context" @@ -68,3 +69,34 @@ func Processes(ctx *context.PrivateContext) { return } } + +// ListProfiles lists the available named pprof profiles +func ListProfiles(ctx *context.PrivateContext) { + json := ctx.FormBool("json") + profiles := pprof.Profiles() + if json { + names := make([]string, len(profiles)) + for _, profile := range profiles { + names = append(names, profile.Name()) + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "Names": names, + }) + } + + ctx.Status(http.StatusOK) + for _, profile := range profiles { + if _, err := ctx.Resp.Write([]byte(profile.Name())); err != nil { + log.Error("Unable to write out profile name: %v", err) + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + + if _, err := ctx.Resp.Write([]byte("\n")); err != nil { + log.Error("Unable to write out profile name: %v", err) + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + } + ctx.Resp.Flush() +} From d9eac18b8b0c1b185ee418d3f683dbe5533240b7 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 4 Feb 2023 20:49:20 +0000 Subject: [PATCH 18/32] add documentation Signed-off-by: Andrew Thornton --- docs/content/doc/usage/command-line.en-us.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index d9d397df31b3d..54b4b03245313 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -515,6 +515,20 @@ Manage running server operations: - `--stacktraces`: Show stacktraces for goroutines associated with processes - `--json`: Output as json - `--cancel PID`: Send cancel to process with PID. (Only for non-system processes.) + - `cpu-profile`: Return the PProf CPU profile + - Options: + - `--duration`: Duration of time to run profile (default: 30s) + - `fg-profile`: Returns the PProf Full Go profile + - Options: + - `--duration`: Duration of time to run profile (default: 30s) + - `--format`: Format of profile (default: pprof) + - `list-named-profiles`: Returns a list of named profiles + - Options: + - `json`: Set to true to return a json output + - `named-profile`: Returns the output of a named profile + - Options: + - `name`: Name of the profile + - `debug-level`: Debug level for the profile ### dump-repo From 7decd93787fd75f582b65754bcd7bbb4f080b615 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 4 Feb 2023 21:09:04 +0000 Subject: [PATCH 19/32] Add output option Signed-off-by: Andrew Thornton --- cmd/manager.go | 83 ++++++++++++++++++-- docs/content/doc/usage/command-line.en-us.md | 11 ++- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/cmd/manager.go b/cmd/manager.go index b2e3494699713..039283209fa0a 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -5,6 +5,7 @@ package cmd import ( "fmt" + "io" "net/http" "os" "time" @@ -99,6 +100,11 @@ var ( Name: "cancel", Usage: "Process PID to cancel. (Only available for non-system processes.)", }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "-", + }, }, } subCmdCPUProfile = cli.Command{ @@ -111,6 +117,11 @@ var ( Usage: "Duration to collect CPU Profile over", Value: 30 * time.Second, }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "cpu-profile", + }, }, } subCmdFGProfile = cli.Command{ @@ -128,6 +139,11 @@ var ( Usage: "Format to return the profile in: pprof, folded", Value: "pprof", }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "fg-profile", + }, }, } subCmdNamedProfile = cli.Command{ @@ -143,6 +159,10 @@ var ( Name: "debug-level", Usage: "Debug level for the profile", }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + }, }, } subCmdListNamedProfiles = cli.Command{ @@ -154,6 +174,11 @@ var ( Name: "json", Usage: "Output as json", }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "-", + }, }, } ) @@ -203,12 +228,35 @@ func runFlushQueues(c *cli.Context) error { return nil } +func determineOutput(c *cli.Context, defaultFilename string) (io.WriteCloser, error) { + out := os.Stdout + filename := c.String("output") + if filename == "" { + filename = defaultFilename + } + if filename != "-" { + var err error + out, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return nil, fail("Unable to open "+filename, err.Error()) + } + fmt.Printf("Writing to %s\n", filename) + } + return out, nil +} + func runProcesses(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() setup("manager", c.Bool("debug")) - statusCode, msg := private.Processes(ctx, os.Stdout, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) + out, err := determineOutput(c, "-") + if err != nil { + return err + } + defer out.Close() + + statusCode, msg := private.Processes(ctx, out, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) @@ -221,7 +269,14 @@ func runCPUProfile(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() setup("manager", c.Bool("debug")) - statusCode, msg := private.CPUProfile(ctx, os.Stdout, c.Duration("duration")) + + out, err := determineOutput(c, "cpu-profile") + if err != nil { + return err + } + defer out.Close() + + statusCode, msg := private.CPUProfile(ctx, out, c.Duration("duration")) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) @@ -233,7 +288,13 @@ func runFGProfile(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() setup("manager", c.Bool("debug")) - statusCode, msg := private.FGProfile(ctx, os.Stdout, c.Duration("duration"), c.String("format")) + out, err := determineOutput(c, "fg-profile") + if err != nil { + return err + } + defer out.Close() + + statusCode, msg := private.FGProfile(ctx, out, c.Duration("duration"), c.String("format")) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) @@ -245,7 +306,12 @@ func runNamedProfile(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() setup("manager", c.Bool("debug")) - statusCode, msg := private.NamedProfile(ctx, os.Stdout, c.String("name"), c.Int("debug-level")) + out, err := determineOutput(c, c.String("name")+"-profile") + if err != nil { + return err + } + defer out.Close() + statusCode, msg := private.NamedProfile(ctx, out, c.String("name"), c.Int("debug-level")) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) @@ -257,7 +323,14 @@ func runListNamedProfile(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() setup("manager", c.Bool("debug")) - statusCode, msg := private.ListNamedProfiles(ctx, os.Stdout, c.Bool("json")) + + out, err := determineOutput(c, "-") + if err != nil { + return err + } + defer out.Close() + + statusCode, msg := private.ListNamedProfiles(ctx, out, c.Bool("json")) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index 54b4b03245313..9e9c2fc34f4c8 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -515,20 +515,25 @@ Manage running server operations: - `--stacktraces`: Show stacktraces for goroutines associated with processes - `--json`: Output as json - `--cancel PID`: Send cancel to process with PID. (Only for non-system processes.) + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) - `cpu-profile`: Return the PProf CPU profile - Options: - `--duration`: Duration of time to run profile (default: 30s) + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) - `fg-profile`: Returns the PProf Full Go profile - Options: - `--duration`: Duration of time to run profile (default: 30s) - `--format`: Format of profile (default: pprof) + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) - `list-named-profiles`: Returns a list of named profiles - Options: - - `json`: Set to true to return a json output + - `--json`: Set to true to return a json output + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) - `named-profile`: Returns the output of a named profile - Options: - - `name`: Name of the profile - - `debug-level`: Debug level for the profile + - `--name`: Name of the profile + - `--debug-level`: Debug level for the profile + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) ### dump-repo From 4d916b45dd03ce9b10d9b10369815ee8de29460d Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 4 Feb 2023 21:20:01 +0000 Subject: [PATCH 20/32] fix missing name Signed-off-by: Andrew Thornton --- templates/admin/pprof.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 23ad1f4ff2182..79745aa47cc2a 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -59,7 +59,7 @@
- {{range .Profiles}} {{end}} From b9942cb9a6e51bbec79ff8e9209b44b6ef62baf5 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 4 Feb 2023 21:36:08 +0000 Subject: [PATCH 21/32] add trace and fix format Signed-off-by: Andrew Thornton --- cmd/manager.go | 37 +++++++++++++++++++ docs/content/doc/usage/command-line.en-us.md | 4 ++ modules/private/manager.go | 6 +++ options/locale/locale_en-US.ini | 2 + routers/private/internal.go | 1 + routers/web/admin/pprof.go | 39 +++++++++++++++++++- routers/web/web.go | 1 + templates/admin/pprof.tmpl | 12 +++++- 8 files changed, 99 insertions(+), 3 deletions(-) diff --git a/cmd/manager.go b/cmd/manager.go index 039283209fa0a..bcc02aafafbb1 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -31,6 +31,7 @@ var ( subCmdFGProfile, subCmdListNamedProfiles, subCmdNamedProfile, + subCmdTrace, }, } subcmdShutdown = cli.Command{ @@ -181,6 +182,23 @@ var ( }, }, } + subCmdTrace = cli.Command{ + Name: "trace", + Usage: "Return PProf trace", + Action: runCPUProfile, + Flags: []cli.Flag{ + cli.DurationFlag{ + Name: "duration", + Usage: "Duration to collect CPU Profile over", + Value: 30 * time.Second, + }, + cli.StringFlag{ + Name: "output,o", + Usage: "File to output to (set to \"-\" for stdout)", + Value: "trace", + }, + }, + } ) func runShutdown(c *cli.Context) error { @@ -337,3 +355,22 @@ func runListNamedProfile(c *cli.Context) error { } return nil } + +func runTrace(c *cli.Context) error { + ctx, cancel := installSignals() + defer cancel() + setup("manager", c.Bool("debug")) + + out, err := determineOutput(c, "trace") + if err != nil { + return err + } + defer out.Close() + + statusCode, msg := private.Trace(ctx, out, c.Duration("duration")) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + return nil +} diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index 9e9c2fc34f4c8..285fb848937f1 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -534,6 +534,10 @@ Manage running server operations: - `--name`: Name of the profile - `--debug-level`: Debug level for the profile - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) + - `trace`: Return the PProf trace + - Options: + - `--duration`: Duration of time to run profile (default: 30s) + - `--output filename`, `-o filename`: Filename to output to. (Set to `-` to use stdout.) ### dump-repo diff --git a/modules/private/manager.go b/modules/private/manager.go index 4733798b1ea93..bf84d0f10552a 100644 --- a/modules/private/manager.go +++ b/modules/private/manager.go @@ -255,6 +255,12 @@ func ListNamedProfiles(ctx context.Context, out io.Writer, json bool) (int, stri return commonGet(ctx, out, reqURL) } +// Trace returns a trace from Gitea +func Trace(ctx context.Context, out io.Writer, duration time.Duration) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/trace?duration=%s", url.QueryEscape(duration.String())) + return commonGet(ctx, out, reqURL) +} + func commonGet(ctx context.Context, out io.Writer, reqURL string) (int, string) { req := newInternalRequest(ctx, reqURL, "GET") resp, err := req.Response() diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 1be045454bd85..ce92fcacb9569 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2978,6 +2978,8 @@ monitor.pprof.stacktrace.flat = Do not nest processes under their parents monitor.pprof.stacktrace.no-system = Do not include go-routines associated with system processes monitor.pprof.stacktrace.format = Format +monitor.pprof.trace = Trace + monitor.queues = Queues monitor.queue = Queue: %s monitor.queue.name = Name diff --git a/routers/private/internal.go b/routers/private/internal.go index 71506e96a2178..06d1a98cef228 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -80,6 +80,7 @@ func Routes() *web.Route { r.Get("/manager/profile", admin.PProfNamedProfile) r.Get("/manager/fgprof", admin.PProfFGProfile) r.Get("/manager/list-profiles", ListProfiles) + r.Get("/manager/trace", admin.Trace) r.Post("/mail/send", SendEmail) r.Post("/restore_repo", RestoreRepo) diff --git a/routers/web/admin/pprof.go b/routers/web/admin/pprof.go index 75c3dff7fbc7e..b0b1d1e44eaef 100644 --- a/routers/web/admin/pprof.go +++ b/routers/web/admin/pprof.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "runtime/pprof" + "runtime/trace" "strconv" "time" @@ -64,7 +65,7 @@ func PProfFGProfile(ctx *context.Context) { var err error duration, err = time.ParseDuration(durationStr) if err != nil { - ctx.Flash.Error(ctx.Tr("monitor.pprof.duration_invalid")) + ctx.Flash.Error(ctx.Tr("admin.monitor.pprof.duration_invalid")) ctx.Redirect(setting.AppSubURL + "/admin/monitor") return } @@ -103,7 +104,7 @@ func PProfCPUProfile(ctx *context.Context) { var err error duration, err = time.ParseDuration(durationStr) if err != nil { - ctx.Flash.Error(ctx.Tr("monitor.pprof.duration_invalid")) + ctx.Flash.Error(ctx.Tr("admin.monitor.pprof.duration_invalid")) ctx.Redirect(setting.AppSubURL + "/admin/monitor") return } @@ -151,3 +152,37 @@ func PProfNamedProfile(ctx *context.Context) { return } } + +// Trace returns a trace +func Trace(ctx *context.Context) { + durationStr := ctx.FormString("duration") + duration := 30 * time.Second + if durationStr != "" { + var err error + duration, err = time.ParseDuration(durationStr) + if err != nil { + ctx.Flash.Error(ctx.Tr("admin.monitor.pprof.duration_invalid")) + ctx.Redirect(setting.AppSubURL + "/admin/monitor") + return + } + } + + start := time.Now() + + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: "trace-" + strconv.FormatInt(start.Unix(), 10), + LastModified: start, + }) + + err := trace.Start(ctx.Resp) + if err != nil { + ctx.ServerError("StartCPUProfile", err) + return + } + + select { + case <-time.After(duration): + case <-ctx.Done(): + } + trace.Stop() +} diff --git a/routers/web/web.go b/routers/web/web.go index 11e189f3f5db7..a249bdb60309a 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -526,6 +526,7 @@ func RegisterRoutes(m *web.Route) { m.Get("/profile", admin.PProfNamedProfile) m.Get("/fgprof", admin.PProfFGProfile) m.Get("/stacktrace-profile", admin.PProfProcessStacktrace) + m.Get("/trace", admin.Trace) m.Post("/cancel/{pid}", admin.MonitorCancel) m.Group("/queue/{qid}", func() { m.Get("", admin.Queue) diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 79745aa47cc2a..ddfd2eb38cae2 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -21,7 +21,7 @@
- @@ -54,6 +54,16 @@
+

{{.locale.Tr "admin.monitor.pprof.trace"}}

+
+
+
+ + +
+ +
+

{{.locale.Tr "admin.monitor.pprof.named_profiles"}}

From 7c4be9c1ab7ae8abbd238a0ddc1be339d07f7ff9 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 4 Feb 2023 21:37:08 +0000 Subject: [PATCH 22/32] fix format on stacktraces Signed-off-by: Andrew Thornton --- templates/admin/pprof.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index ddfd2eb38cae2..7e80adaaf0b7b 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -46,7 +46,7 @@
- From 6c2017d6cffae9dc21cc89736f8cec5fce1c3bc1 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 4 Feb 2023 21:41:58 +0000 Subject: [PATCH 23/32] fix trace Signed-off-by: Andrew Thornton --- cmd/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/manager.go b/cmd/manager.go index bcc02aafafbb1..53c77dac735cd 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -185,7 +185,7 @@ var ( subCmdTrace = cli.Command{ Name: "trace", Usage: "Return PProf trace", - Action: runCPUProfile, + Action: runTrace, Flags: []cli.Flag{ cli.DurationFlag{ Name: "duration", From d37978e3dc5aef1b3d53dd4fc7f7d6c592cc9b84 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sun, 5 Feb 2023 06:56:41 +0000 Subject: [PATCH 24/32] Apply suggestions from code review --- routers/web/admin/admin.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 1539fe224a7e5..08e1b263338ec 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -161,11 +161,10 @@ func Monitor(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.monitor") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminMonitor"] = true - processes, processCount := process.GetManager().Processes(false, true) + ctx.Data["Processes"], ctx.Data["ProcessCount"] = process.GetManager().Processes(false, true) ctx.Data["Entries"] = cron.ListTasks() ctx.Data["Queues"] = queue.GetManager().ManagedQueues() - ctx.Data["Processes"], ctx.Data["ProcessCount"] = processes, processCount ctx.Data["Profiles"] = pprof.Profiles() From 2dbab58a3fd4cf14ed4a266959f27916ddc7f1f8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 5 Feb 2023 20:17:55 +0000 Subject: [PATCH 25/32] as per delvh Signed-off-by: Andrew Thornton --- cmd/manager.go | 128 ++++++++++++------------------------- routers/web/admin/admin.go | 1 - 2 files changed, 40 insertions(+), 89 deletions(-) diff --git a/cmd/manager.go b/cmd/manager.go index 53c77dac735cd..9b6b449cab793 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -4,10 +4,12 @@ package cmd import ( + "context" "fmt" "io" "net/http" "os" + "reflect" "time" "code.gitea.io/gitea/modules/private" @@ -201,11 +203,17 @@ var ( } ) -func runShutdown(c *cli.Context) error { +func setupManager(c *cli.Context) (context.Context, context.CancelFunc) { ctx, cancel := installSignals() - defer cancel() setup("manager", c.Bool("debug")) + return ctx, cancel +} + +func runShutdown(c *cli.Context) error { + ctx, cancel := setupManager(c) + defer cancel() + statusCode, msg := private.Shutdown(ctx) switch statusCode { case http.StatusInternalServerError: @@ -217,10 +225,9 @@ func runShutdown(c *cli.Context) error { } func runRestart(c *cli.Context) error { - ctx, cancel := installSignals() + ctx, cancel := setupManager(c) defer cancel() - setup("manager", c.Bool("debug")) statusCode, msg := private.Restart(ctx) switch statusCode { case http.StatusInternalServerError: @@ -232,10 +239,9 @@ func runRestart(c *cli.Context) error { } func runFlushQueues(c *cli.Context) error { - ctx, cancel := installSignals() + ctx, cancel := setupManager(c) defer cancel() - setup("manager", c.Bool("debug")) statusCode, msg := private.FlushQueues(ctx, c.Duration("timeout"), c.Bool("non-blocking")) switch statusCode { case http.StatusInternalServerError: @@ -263,18 +269,32 @@ func determineOutput(c *cli.Context, defaultFilename string) (io.WriteCloser, er return out, nil } -func runProcesses(c *cli.Context) error { - ctx, cancel := installSignals() +// runManagerPrivateFunc will requires that a provided fn has an interface: +// +// func(context.Context, io.Writer, ...argsTypes) (int, string) { +// +// but this cann't quite easily be expressed as a generic type +func runManagerPrivateFunc(c *cli.Context, defaultOutput string, fn interface{}, args ...any) error { + ctx, cancel := setupManager(c) defer cancel() - setup("manager", c.Bool("debug")) - out, err := determineOutput(c, "-") + out, err := determineOutput(c, defaultOutput) if err != nil { return err } defer out.Close() - statusCode, msg := private.Processes(ctx, out, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) + valFn := reflect.ValueOf(fn) + callArgs := []reflect.Value{ + reflect.ValueOf(ctx), + reflect.ValueOf(out), + } + for _, arg := range args { + callArgs = append(callArgs, reflect.ValueOf(arg)) + } + outArgs := valFn.Call(callArgs) + + statusCode, msg := outArgs[0].Interface().(int), outArgs[1].Interface().(string) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) @@ -283,94 +303,26 @@ func runProcesses(c *cli.Context) error { return nil } -func runCPUProfile(c *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - setup("manager", c.Bool("debug")) - - out, err := determineOutput(c, "cpu-profile") - if err != nil { - return err - } - defer out.Close() +func runProcesses(c *cli.Context) error { + return runManagerPrivateFunc(c, "-", private.Processes, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) +} - statusCode, msg := private.CPUProfile(ctx, out, c.Duration("duration")) - switch statusCode { - case http.StatusInternalServerError: - return fail("InternalServerError", msg) - } - return nil +func runCPUProfile(c *cli.Context) error { + return runManagerPrivateFunc(c, "cpu-profile", private.CPUProfile, c.Duration("duration")) } func runFGProfile(c *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - setup("manager", c.Bool("debug")) - out, err := determineOutput(c, "fg-profile") - if err != nil { - return err - } - defer out.Close() - - statusCode, msg := private.FGProfile(ctx, out, c.Duration("duration"), c.String("format")) - switch statusCode { - case http.StatusInternalServerError: - return fail("InternalServerError", msg) - } - return nil + return runManagerPrivateFunc(c, "fg-profile", private.FGProfile, c.Duration("duration"), c.String("format")) } func runNamedProfile(c *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - setup("manager", c.Bool("debug")) - out, err := determineOutput(c, c.String("name")+"-profile") - if err != nil { - return err - } - defer out.Close() - statusCode, msg := private.NamedProfile(ctx, out, c.String("name"), c.Int("debug-level")) - switch statusCode { - case http.StatusInternalServerError: - return fail("InternalServerError", msg) - } - return nil + return runManagerPrivateFunc(c, c.String("name")+"-profile", private.NamedProfile, c.String("name"), c.Int("debug-level")) } func runListNamedProfile(c *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - setup("manager", c.Bool("debug")) - - out, err := determineOutput(c, "-") - if err != nil { - return err - } - defer out.Close() - - statusCode, msg := private.ListNamedProfiles(ctx, out, c.Bool("json")) - switch statusCode { - case http.StatusInternalServerError: - return fail("InternalServerError", msg) - } - return nil + return runManagerPrivateFunc(c, "-", private.ListNamedProfiles, c.Bool("json")) } func runTrace(c *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - setup("manager", c.Bool("debug")) - - out, err := determineOutput(c, "trace") - if err != nil { - return err - } - defer out.Close() - - statusCode, msg := private.Trace(ctx, out, c.Duration("duration")) - switch statusCode { - case http.StatusInternalServerError: - return fail("InternalServerError", msg) - } - return nil + return runManagerPrivateFunc(c, "trace", private.Trace, c.Duration("duration")) } diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 08e1b263338ec..d195a43b79277 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -165,7 +165,6 @@ func Monitor(ctx *context.Context) { ctx.Data["Entries"] = cron.ListTasks() ctx.Data["Queues"] = queue.GetManager().ManagedQueues() - ctx.Data["Profiles"] = pprof.Profiles() ctx.HTML(http.StatusOK, tplMonitor) From 2b523f14f84442bd0248b13a247f63562a8bfad6 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 5 Feb 2023 20:27:58 +0000 Subject: [PATCH 26/32] include indent in WriteProcess Signed-off-by: Andrew Thornton --- modules/process/stacktraces_processlist.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/process/stacktraces_processlist.go b/modules/process/stacktraces_processlist.go index 33a451dc7bc1a..917c6a88d57cc 100644 --- a/modules/process/stacktraces_processlist.go +++ b/modules/process/stacktraces_processlist.go @@ -20,7 +20,7 @@ func WriteProcesses(out io.Writer, processes []*Process, processCount int, gorou return err } if len(processes) > 0 { - if err := WriteProcess(out, processes[0], " ", flat); err != nil { + if err := WriteProcess(out, processes[0], indent+" ", flat); err != nil { return err } } @@ -29,7 +29,7 @@ func WriteProcesses(out io.Writer, processes []*Process, processCount int, gorou if _, err := fmt.Fprintf(out, "%s | \n", indent); err != nil { return err } - if err := WriteProcess(out, process, " ", flat); err != nil { + if err := WriteProcess(out, process, indent+" ", flat); err != nil { return err } } From 5925f475115196877b9a18225e5e5caa5eb6062d Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 5 Feb 2023 21:01:47 +0000 Subject: [PATCH 27/32] as per delvh Signed-off-by: Andrew Thornton --- options/locale/locale_en-US.ini | 6 ++++-- templates/admin/pprof.tmpl | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ca07797db832a..cab747003d8ed 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2972,8 +2972,10 @@ monitor.pprof.fgprof.format = Format monitor.pprof.named_profiles = Named Profiles monitor.pprof.named_profiles.name = Name -monitor.pprof.named_profiles.debug = Debug level -monitor.pprof.named_profiles.debug_placeholder = e.g. 0 +monitor.pprof.named_profiles.debug = Format +monitor.pprof.named_profiles.format_pprof = pprof +monitor.pprof.named_profiles.format_text = text +monitor.pprof.named_profiles.format_goroutine = custom/goroutine monitor.pprof.stacktrace.flat = Do not nest processes under their parents monitor.pprof.stacktrace.no-system = Do not include go-routines associated with system processes diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 7e80adaaf0b7b..8589295622873 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -77,7 +77,11 @@
- +
From 91bc97afd0819b7b7a182a4a8d68a7deed293b12 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 5 Feb 2023 21:04:31 +0000 Subject: [PATCH 28/32] remove space Signed-off-by: Andrew Thornton --- modules/process/stacktraces_processlist.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/process/stacktraces_processlist.go b/modules/process/stacktraces_processlist.go index 917c6a88d57cc..7a31d38b974ce 100644 --- a/modules/process/stacktraces_processlist.go +++ b/modules/process/stacktraces_processlist.go @@ -26,7 +26,7 @@ func WriteProcesses(out io.Writer, processes []*Process, processCount int, gorou } if len(processes) > 1 { for _, process := range processes[1:] { - if _, err := fmt.Fprintf(out, "%s | \n", indent); err != nil { + if _, err := fmt.Fprintf(out, "%s |\n", indent); err != nil { return err } if err := WriteProcess(out, process, indent+" ", flat); err != nil { From 61755a7d2c37a78b0253b23f3010862059e6b7c5 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 5 Feb 2023 21:11:04 +0000 Subject: [PATCH 29/32] as per delvh Signed-off-by: Andrew Thornton --- routers/web/admin/pprof.go | 1 - 1 file changed, 1 deletion(-) diff --git a/routers/web/admin/pprof.go b/routers/web/admin/pprof.go index b0b1d1e44eaef..fcf188c993907 100644 --- a/routers/web/admin/pprof.go +++ b/routers/web/admin/pprof.go @@ -53,7 +53,6 @@ func PProfProcessStacktrace(ctx *context.Context) { if err := process.WriteProcesses(ctx.Resp, processStacks, processCount, goroutineCount, "", flat); err != nil { ctx.ServerError("WriteProcesses", err) - return } } From 1400ab2af51c84e3f7ac25f3dbf93b7b41eb88b2 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 5 Feb 2023 21:33:38 +0000 Subject: [PATCH 30/32] as per delvh Signed-off-by: Andrew Thornton --- cmd/manager.go | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/cmd/manager.go b/cmd/manager.go index 9b6b449cab793..a608eb07c5ac3 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -9,7 +9,6 @@ import ( "io" "net/http" "os" - "reflect" "time" "code.gitea.io/gitea/modules/private" @@ -269,12 +268,7 @@ func determineOutput(c *cli.Context, defaultFilename string) (io.WriteCloser, er return out, nil } -// runManagerPrivateFunc will requires that a provided fn has an interface: -// -// func(context.Context, io.Writer, ...argsTypes) (int, string) { -// -// but this cann't quite easily be expressed as a generic type -func runManagerPrivateFunc(c *cli.Context, defaultOutput string, fn interface{}, args ...any) error { +func wrapManagerPrivateFunc(c *cli.Context, defaultOutput string, fn func(ctx context.Context, out io.Writer) (int, string)) error { ctx, cancel := setupManager(c) defer cancel() @@ -284,17 +278,7 @@ func runManagerPrivateFunc(c *cli.Context, defaultOutput string, fn interface{}, } defer out.Close() - valFn := reflect.ValueOf(fn) - callArgs := []reflect.Value{ - reflect.ValueOf(ctx), - reflect.ValueOf(out), - } - for _, arg := range args { - callArgs = append(callArgs, reflect.ValueOf(arg)) - } - outArgs := valFn.Call(callArgs) - - statusCode, msg := outArgs[0].Interface().(int), outArgs[1].Interface().(string) + statusCode, msg := fn(ctx, out) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) @@ -304,25 +288,37 @@ func runManagerPrivateFunc(c *cli.Context, defaultOutput string, fn interface{}, } func runProcesses(c *cli.Context) error { - return runManagerPrivateFunc(c, "-", private.Processes, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) + return wrapManagerPrivateFunc(c, "-", func(ctx context.Context, out io.Writer) (int, string) { + return private.Processes(ctx, out, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) + }) } func runCPUProfile(c *cli.Context) error { - return runManagerPrivateFunc(c, "cpu-profile", private.CPUProfile, c.Duration("duration")) + return wrapManagerPrivateFunc(c, "cpu-profile", func(ctx context.Context, out io.Writer) (int, string) { + return private.CPUProfile(ctx, out, c.Duration("duration")) + }) } func runFGProfile(c *cli.Context) error { - return runManagerPrivateFunc(c, "fg-profile", private.FGProfile, c.Duration("duration"), c.String("format")) + return wrapManagerPrivateFunc(c, "fg-profile", func(ctx context.Context, out io.Writer) (int, string) { + return private.FGProfile(ctx, out, c.Duration("duration"), c.String("format")) + }) } func runNamedProfile(c *cli.Context) error { - return runManagerPrivateFunc(c, c.String("name")+"-profile", private.NamedProfile, c.String("name"), c.Int("debug-level")) + return wrapManagerPrivateFunc(c, c.String("name")+"-profile", func(ctx context.Context, out io.Writer) (int, string) { + return private.NamedProfile(ctx, out, c.String("name"), c.Int("debug-level")) + }) } func runListNamedProfile(c *cli.Context) error { - return runManagerPrivateFunc(c, "-", private.ListNamedProfiles, c.Bool("json")) + return wrapManagerPrivateFunc(c, "-", func(ctx context.Context, out io.Writer) (int, string) { + return private.ListNamedProfiles(ctx, out, c.Bool("json")) + }) } func runTrace(c *cli.Context) error { - return runManagerPrivateFunc(c, "trace", private.Trace, c.Duration("duration")) + return wrapManagerPrivateFunc(c, "trace", func(ctx context.Context, out io.Writer) (int, string) { + return private.Trace(ctx, out, c.Duration("duration")) + }) } From edebd86ac3e3ec4019b08841e3b081964b219214 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 5 Feb 2023 22:08:43 +0000 Subject: [PATCH 31/32] add some comments Signed-off-by: Andrew Thornton --- options/locale/locale_en-US.ini | 8 ++++++++ templates/admin/pprof.tmpl | 27 +++++++++++++++++---------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index cab747003d8ed..36ca2d9ac44cc 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2960,28 +2960,36 @@ monitor.process.cancel_notices = Cancel: %s? monitor.process.children = Children monitor.pprof = PProf Profiles +monitor.pprof.description = PProf profiles provide runtime profiling data. +monitor.pprof.description_2 = Text formats can be read directly but most data is in the format expected by the golang pprof visualization tool + monitor.pprof.download = Download monitor.pprof.duration = Duration monitor.pprof.duration_placeholder = e.g. 30s monitor.pprof.duration_invalid = Invalid duration - duration must be a golang duration string monitor.pprof.cpuprofile = CPU Profile +monitor.pprof.cpuprofile.description = CPU profile determines where Gitea spends its time while actively consuming CPU cycles monitor.pprof.fgprof = Full Go Profile +monitor.pprof.fgprof.description = Full Go profile provides wall clock profiling combining the CPU Profile with IO time monitor.pprof.fgprof.format = Format monitor.pprof.named_profiles = Named Profiles +monitor.pprof.named_profiles.description = Go provides a number of named profiles for other profiling monitor.pprof.named_profiles.name = Name monitor.pprof.named_profiles.debug = Format monitor.pprof.named_profiles.format_pprof = pprof monitor.pprof.named_profiles.format_text = text monitor.pprof.named_profiles.format_goroutine = custom/goroutine +monitor.pprof.stacktrace.description = Stacktraces provides stacktraces for all current goroutines mapped with Gitea's internal processes monitor.pprof.stacktrace.flat = Do not nest processes under their parents monitor.pprof.stacktrace.no-system = Do not include go-routines associated with system processes monitor.pprof.stacktrace.format = Format monitor.pprof.trace = Trace +monitor.pprof.trace.description = Trace provides tracing that can be used by the go tracing tool monitor.queues = Queues monitor.queue = Queue: %s diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 8589295622873..33899674d47f6 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -2,9 +2,12 @@ {{.locale.Tr "admin.monitor.pprof"}}
+

{{.locale.Tr "admin.monitor.pprof.description"}}

+

{{.locale.Tr "admin.monitor.pprof.description_2"}}

{{.locale.Tr "admin.monitor.pprof.cpuprofile"}}

+
{{.locale.Tr "admin.monitor.pprof.cpuprofile.description"}}
@@ -15,6 +18,7 @@

{{.locale.Tr "admin.monitor.pprof.fgprof"}}

+
{{.locale.Tr "admin.monitor.pprof.fgprof.description"}}
@@ -32,6 +36,7 @@

{{.locale.Tr "admin.monitor.stacktrace"}}

+
{{.locale.Tr "admin.monitor.pprof.stacktrace.description"}}
@@ -54,19 +59,10 @@
-

{{.locale.Tr "admin.monitor.pprof.trace"}}

-
-
-
- - -
- -
-

{{.locale.Tr "admin.monitor.pprof.named_profiles"}}

+
{{.locale.Tr "admin.monitor.pprof.named_profiles.description"}}
+

{{.locale.Tr "admin.monitor.pprof.trace"}}

+
+
{{.locale.Tr "admin.monitor.pprof.trace.description"}}
+
+
+ + +
+ +
+
From 92ac2951a1becdc9340592b83b994f353d5419da Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 19 Feb 2023 12:41:24 +0000 Subject: [PATCH 32/32] move trace help in to the ui form Signed-off-by: Andrew Thornton --- templates/admin/pprof.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin/pprof.tmpl b/templates/admin/pprof.tmpl index 33899674d47f6..c1c1b55f48224 100644 --- a/templates/admin/pprof.tmpl +++ b/templates/admin/pprof.tmpl @@ -84,8 +84,8 @@

{{.locale.Tr "admin.monitor.pprof.trace"}}

-
{{.locale.Tr "admin.monitor.pprof.trace.description"}}
+
{{.locale.Tr "admin.monitor.pprof.trace.description"}}