runtime: schedule cleanups across multiple goroutines

This change splits the finalizer and cleanup queues and implements a new
lock-free blocking queue for cleanups. The basic design is as follows:

The cleanup queue is organized in fixed-sized blocks. Individual cleanup
functions are queued, but only whole blocks are dequeued.

Enqueuing cleanups places them in P-local cleanup blocks. These are
flushed to the full list as they get full. Cleanups can only be enqueued
by an active sweeper.

Dequeuing cleanups always dequeues entire blocks from the full list.
Cleanup blocks can be dequeued and executed at any time.

The very last active sweeper in the sweep phase is responsible for
flushing all local cleanup blocks to the full list. It can do this
without any synchronization because the next GC can't start yet, so we
can be very certain that nobody else will be accessing the local blocks.

Cleanup blocks are stored off-heap because the need to be allocated by
the sweeper, which is called from heap allocation paths. As a result,
the GC treats cleanup blocks as roots, just like finalizer blocks.

Flushes to the full list signal to the scheduler that cleanup goroutines
should be awoken. Every time the scheduler goes to wake up a cleanup
goroutine and there were more signals than goroutines to wake, it then
forwards this signal to runtime.AddCleanup, so that it creates another
goroutine the next time it is called, up to gomaxprocs goroutines.

The signals here are a little convoluted, but exist because the sweeper
and the scheduler cannot safely create new goroutines.

For #71772.
For #71825.

Change-Id: Ie839fde2b67e1b79ac1426be0ea29a8d923a62cc
Reviewed-on: https://go-review.googlesource.com/c/go/+/650697
Reviewed-by: Michael Pratt <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
Auto-Submit: Michael Knyszek <[email protected]>

Author: mknyszek

src/cmd/internal/objabi/funcid.go

@@ -10,27 +10,28 @@ import (
 )
 
 var funcIDs = map[string]abi.FuncID{
-	"abort":                    abi.FuncID_abort,
-	"asmcgocall":               abi.FuncID_asmcgocall,
-	"asyncPreempt":             abi.FuncID_asyncPreempt,
-	"cgocallback":              abi.FuncID_cgocallback,
-	"corostart":                abi.FuncID_corostart,
-	"debugCallV2":              abi.FuncID_debugCallV2,
-	"gcBgMarkWorker":           abi.FuncID_gcBgMarkWorker,
-	"rt0_go":                   abi.FuncID_rt0_go,
-	"goexit":                   abi.FuncID_goexit,
-	"gogo":                     abi.FuncID_gogo,
-	"gopanic":                  abi.FuncID_gopanic,
-	"handleAsyncEvent":         abi.FuncID_handleAsyncEvent,
-	"main":                     abi.FuncID_runtime_main,
-	"mcall":                    abi.FuncID_mcall,
-	"morestack":                abi.FuncID_morestack,
-	"mstart":                   abi.FuncID_mstart,
-	"panicwrap":                abi.FuncID_panicwrap,
-	"runFinalizersAndCleanups": abi.FuncID_runFinalizersAndCleanups,
-	"sigpanic":                 abi.FuncID_sigpanic,
-	"systemstack_switch":       abi.FuncID_systemstack_switch,
-	"systemstack":              abi.FuncID_systemstack,
+	"abort":              abi.FuncID_abort,
+	"asmcgocall":         abi.FuncID_asmcgocall,
+	"asyncPreempt":       abi.FuncID_asyncPreempt,
+	"cgocallback":        abi.FuncID_cgocallback,
+	"corostart":          abi.FuncID_corostart,
+	"debugCallV2":        abi.FuncID_debugCallV2,
+	"gcBgMarkWorker":     abi.FuncID_gcBgMarkWorker,
+	"rt0_go":             abi.FuncID_rt0_go,
+	"goexit":             abi.FuncID_goexit,
+	"gogo":               abi.FuncID_gogo,
+	"gopanic":            abi.FuncID_gopanic,
+	"handleAsyncEvent":   abi.FuncID_handleAsyncEvent,
+	"main":               abi.FuncID_runtime_main,
+	"mcall":              abi.FuncID_mcall,
+	"morestack":          abi.FuncID_morestack,
+	"mstart":             abi.FuncID_mstart,
+	"panicwrap":          abi.FuncID_panicwrap,
+	"runFinalizers":      abi.FuncID_runFinalizers,
+	"runCleanups":        abi.FuncID_runCleanups,
+	"sigpanic":           abi.FuncID_sigpanic,
+	"systemstack_switch": abi.FuncID_systemstack_switch,
+	"systemstack":        abi.FuncID_systemstack,
 
 	// Don't show in call stack but otherwise not special.
 	"deferreturn": abi.FuncIDWrapper,

src/internal/abi/symtab.go

@@ -56,8 +56,9 @@ const (
 	FuncID_mstart
 	FuncID_panicwrap
 	FuncID_rt0_go
-	FuncID_runFinalizersAndCleanups
 	FuncID_runtime_main
+	FuncID_runFinalizers
+	FuncID_runCleanups
 	FuncID_sigpanic
 	FuncID_systemstack
 	FuncID_systemstack_switch

src/runtime/abi_test.go

@@ -66,6 +66,9 @@ func TestFinalizerRegisterABI(t *testing.T) {
 	runtime.GC()
 	runtime.GC()
 
+	// Make sure the finalizer goroutine is running.
+	runtime.SetFinalizer(new(TintPointer), func(_ *TintPointer) {})
+
 	// fing will only pick the new IntRegArgs up if it's currently
 	// sleeping and wakes up, so wait for it to go to sleep.
 	success := false

src/runtime/crash_test.go

@@ -1102,79 +1102,85 @@ func TestNetpollWaiters(t *testing.T) {
 	}
 }
 
-// The runtime.runFinalizersAndCleanups frame should appear in panics, even if
-// runtime frames are normally hidden (GOTRACEBACK=all).
-func TestFinalizerDeadlockPanic(t *testing.T) {
+func TestFinalizerOrCleanupDeadlock(t *testing.T) {
 	t.Parallel()
-	output := runTestProg(t, "testprog", "FinalizerDeadlock", "GOTRACEBACK=all", "GO_TEST_FINALIZER_DEADLOCK=panic")
 
-	want := "runtime.runFinalizersAndCleanups()"
-	if !strings.Contains(output, want) {
-		t.Errorf("output does not contain %q:\n%s", want, output)
-	}
-}
-
-// The runtime.runFinalizersAndCleanups frame should appear in runtime.Stack,
-// even though runtime frames are normally hidden.
-func TestFinalizerDeadlockStack(t *testing.T) {
-	t.Parallel()
-	output := runTestProg(t, "testprog", "FinalizerDeadlock", "GO_TEST_FINALIZER_DEADLOCK=stack")
-
-	want := "runtime.runFinalizersAndCleanups()"
-	if !strings.Contains(output, want) {
-		t.Errorf("output does not contain %q:\n%s", want, output)
-	}
-}
-
-// The runtime.runFinalizersAndCleanups frame should appear in goroutine
-// profiles.
-func TestFinalizerDeadlockPprofProto(t *testing.T) {
-	t.Parallel()
-	output := runTestProg(t, "testprog", "FinalizerDeadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_proto")
+	for _, useCleanup := range []bool{false, true} {
+		progName := "Finalizer"
+		want := "runtime.runFinalizers"
+		if useCleanup {
+			progName = "Cleanup"
+			want = "runtime.runCleanups"
+		}
 
-	p, err := profile.Parse(strings.NewReader(output))
-	if err != nil {
-		// Logging the binary proto data is not very nice, but it might
-		// be a text error message instead.
-		t.Logf("Output: %s", output)
-		t.Fatalf("Error parsing proto output: %v", err)
-	}
-
-	want := "runtime.runFinalizersAndCleanups"
-	for _, s := range p.Sample {
-		for _, loc := range s.Location {
-			for _, line := range loc.Line {
-				if line.Function.Name == want {
-					// Done!
-					return
+		// The runtime.runFinalizers/runtime.runCleanups frame should appear in panics, even if
+		// runtime frames are normally hidden (GOTRACEBACK=all).
+		t.Run("Panic", func(t *testing.T) {
+			t.Parallel()
+			output := runTestProg(t, "testprog", progName+"Deadlock", "GOTRACEBACK=all", "GO_TEST_FINALIZER_DEADLOCK=panic")
+			want := want + "()"
+			if !strings.Contains(output, want) {
+				t.Errorf("output does not contain %q:\n%s", want, output)
+			}
+		})
+
+		// The runtime.runFinalizers/runtime.Cleanups frame should appear in runtime.Stack,
+		// even though runtime frames are normally hidden.
+		t.Run("Stack", func(t *testing.T) {
+			t.Parallel()
+			output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=stack")
+			want := want + "()"
+			if !strings.Contains(output, want) {
+				t.Errorf("output does not contain %q:\n%s", want, output)
+			}
+		})
+
+		// The runtime.runFinalizers/runtime.Cleanups frame should appear in goroutine
+		// profiles.
+		t.Run("PprofProto", func(t *testing.T) {
+			t.Parallel()
+			output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_proto")
+
+			p, err := profile.Parse(strings.NewReader(output))
+			if err != nil {
+				// Logging the binary proto data is not very nice, but it might
+				// be a text error message instead.
+				t.Logf("Output: %s", output)
+				t.Fatalf("Error parsing proto output: %v", err)
+			}
+			for _, s := range p.Sample {
+				for _, loc := range s.Location {
+					for _, line := range loc.Line {
+						if line.Function.Name == want {
+							// Done!
+							return
+						}
+					}
 				}
 			}
-		}
-	}
-
-	t.Errorf("Profile does not contain %q:\n%s", want, p)
-}
-
-// The runtime.runFinalizersAndCleanups frame should appear in goroutine
-// profiles (debug=1).
-func TestFinalizerDeadlockPprofDebug1(t *testing.T) {
-	t.Parallel()
-	output := runTestProg(t, "testprog", "FinalizerDeadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug1")
-
-	want := "runtime.runFinalizersAndCleanups+"
-	if !strings.Contains(output, want) {
-		t.Errorf("output does not contain %q:\n%s", want, output)
-	}
-}
-
-// The runtime.runFinalizersAndCleanups frame should appear in goroutine
-// profiles (debug=2).
-func TestFinalizerDeadlockPprofDebug2(t *testing.T) {
-	t.Parallel()
-	output := runTestProg(t, "testprog", "FinalizerDeadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug2")
-
-	want := "runtime.runFinalizersAndCleanups()"
-	if !strings.Contains(output, want) {
-		t.Errorf("output does not contain %q:\n%s", want, output)
+			t.Errorf("Profile does not contain %q:\n%s", want, p)
+		})
+
+		// The runtime.runFinalizers/runtime.runCleanups frame should appear in goroutine
+		// profiles (debug=1).
+		t.Run("PprofDebug1", func(t *testing.T) {
+			t.Parallel()
+			output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug1")
+			want := want + "+"
+			if !strings.Contains(output, want) {
+				t.Errorf("output does not contain %q:\n%s", want, output)
+			}
+		})
+
+		// The runtime.runFinalizers/runtime.runCleanups frame should appear in goroutine
+		// profiles (debug=2).
+		t.Run("PprofDebug2", func(t *testing.T) {
+			t.Parallel()
+			output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug2")
+			want := want + "()"
+			if !strings.Contains(output, want) {
+				t.Errorf("output does not contain %q:\n%s", want, output)
+			}
+		})
 	}
 }

src/runtime/export_test.go

@@ -1798,6 +1798,10 @@ func BlockUntilEmptyFinalizerQueue(timeout int64) bool {
 	return blockUntilEmptyFinalizerQueue(timeout)
 }
 
+func BlockUntilEmptyCleanupQueue(timeout int64) bool {
+	return gcCleanups.blockUntilEmpty(timeout)
+}
+
 func FrameStartLine(f *Frame) int {
 	return f.startLine
 }