runtime: remove the allocheaders GOEXPERIMENT

This change removes the allocheaders, deleting all the old code and
merging mbitmap_allocheaders.go back into mbitmap.go.

This change also deletes the SetType benchmarks which were already
broken in the new GOEXPERIMENT (it's harder to set up than before). We
weren't really watching these benchmarks at all, and they don't provide
additional test coverage.

Change-Id: I135497201c3259087c5cd3722ed3fbe24791d25d
Reviewed-on: https://go-review.googlesource.com/c/go/+/567200
Reviewed-by: Keith Randall <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Cherry Mui <[email protected]>
Reviewed-by: Keith Randall <[email protected]>
Auto-Submit: Michael Knyszek <[email protected]>

Author: mknyszek

src/cmd/compile/internal/test/inl_test.go

@@ -72,7 +72,6 @@ func TestIntendedInlining(t *testing.T) {
 			"cgoInRange",
 			"gclinkptr.ptr",
 			"guintptr.ptr",
-			"writeHeapBitsForAddr",
 			"heapBitsSlice",
 			"markBits.isMarked",
 			"muintptr.ptr",
@@ -243,8 +242,6 @@ func TestIntendedInlining(t *testing.T) {
 		// On loong64, mips64x and riscv64, TrailingZeros64 is not intrinsified and causes nextFreeFast
 		// too expensive to inline (Issue 22239).
 		want["runtime"] = append(want["runtime"], "nextFreeFast")
-		// Same behavior for heapBits.nextFast.
-		want["runtime"] = append(want["runtime"], "heapBits.nextFast")
 	}
 	if runtime.GOARCH != "386" {
 		// As explained above, TrailingZeros64 and TrailingZeros32 are not Go code on 386.

src/internal/buildcfg/exp.go

@@ -73,7 +73,6 @@ func ParseGOEXPERIMENT(goos, goarch, goexp string) (*ExperimentFlags, error) {
 		RegabiWrappers:   regabiSupported,
 		RegabiArgs:       regabiSupported,
 		CoverageRedesign: true,
-		AllocHeaders:     true,
 		ExecTracer2:      true,
 	}
 

src/internal/goexperiment/exp_allocheaders_off.go

@@ -120,10 +120,6 @@ type Flags struct {
 	// Range enables range over int and func.
 	Range bool
 
-	// AllocHeaders enables a different, more efficient way for the GC to
-	// manage heap metadata.
-	AllocHeaders bool
-
 	// ExecTracer2 controls whether to use the new execution trace
 	// implementation.
 	ExecTracer2 bool

src/internal/goexperiment/exp_allocheaders_on.go

@@ -85,9 +85,9 @@ package runtime
 import (
 	"internal/abi"
 	"internal/goarch"
-	"internal/goexperiment"
 	"internal/runtime/atomic"
 	"runtime/internal/math"
+	"runtime/internal/sys"
 	"unsafe"
 )
 
@@ -224,14 +224,11 @@ func init() {
 // userArenaChunkReserveBytes returns the amount of additional bytes to reserve for
 // heap metadata.
 func userArenaChunkReserveBytes() uintptr {
-	if goexperiment.AllocHeaders {
-		// In the allocation headers experiment, we reserve the end of the chunk for
-		// a pointer/scalar bitmap. We also reserve space for a dummy _type that
-		// refers to the bitmap. The PtrBytes field of the dummy _type indicates how
-		// many of those bits are valid.
-		return userArenaChunkBytes/goarch.PtrSize/8 + unsafe.Sizeof(_type{})
-	}
-	return 0
+	// In the allocation headers experiment, we reserve the end of the chunk for
+	// a pointer/scalar bitmap. We also reserve space for a dummy _type that
+	// refers to the bitmap. The PtrBytes field of the dummy _type indicates how
+	// many of those bits are valid.
+	return userArenaChunkBytes/goarch.PtrSize/8 + unsafe.Sizeof(_type{})
 }
 
 type userArena struct {
@@ -549,6 +546,202 @@ func userArenaHeapBitsSetSliceType(typ *_type, n int, ptr unsafe.Pointer, s *msp
 	}
 }
 
+// userArenaHeapBitsSetType is the equivalent of heapSetType but for
+// non-slice-backing-store Go values allocated in a user arena chunk. It
+// sets up the type metadata for the value with type typ allocated at address ptr.
+// base is the base address of the arena chunk.
+func userArenaHeapBitsSetType(typ *_type, ptr unsafe.Pointer, s *mspan) {
+	base := s.base()
+	h := s.writeUserArenaHeapBits(uintptr(ptr))
+
+	p := typ.GCData // start of 1-bit pointer mask (or GC program)
+	var gcProgBits uintptr
+	if typ.Kind_&abi.KindGCProg != 0 {
+		// Expand gc program, using the object itself for storage.
+		gcProgBits = runGCProg(addb(p, 4), (*byte)(ptr))
+		p = (*byte)(ptr)
+	}
+	nb := typ.PtrBytes / goarch.PtrSize
+
+	for i := uintptr(0); i < nb; i += ptrBits {
+		k := nb - i
+		if k > ptrBits {
+			k = ptrBits
+		}
+		// N.B. On big endian platforms we byte swap the data that we
+		// read from GCData, which is always stored in little-endian order
+		// by the compiler. writeUserArenaHeapBits handles data in
+		// a platform-ordered way for efficiency, but stores back the
+		// data in little endian order, since we expose the bitmap through
+		// a dummy type.
+		h = h.write(s, readUintptr(addb(p, i/8)), k)
+	}
+	// Note: we call pad here to ensure we emit explicit 0 bits
+	// for the pointerless tail of the object. This ensures that
+	// there's only a single noMorePtrs mark for the next object
+	// to clear. We don't need to do this to clear stale noMorePtrs
+	// markers from previous uses because arena chunk pointer bitmaps
+	// are always fully cleared when reused.
+	h = h.pad(s, typ.Size_-typ.PtrBytes)
+	h.flush(s, uintptr(ptr), typ.Size_)
+
+	if typ.Kind_&abi.KindGCProg != 0 {
+		// Zero out temporary ptrmask buffer inside object.
+		memclrNoHeapPointers(ptr, (gcProgBits+7)/8)
+	}
+
+	// Update the PtrBytes value in the type information. After this
+	// point, the GC will observe the new bitmap.
+	s.largeType.PtrBytes = uintptr(ptr) - base + typ.PtrBytes
+
+	// Double-check that the bitmap was written out correctly.
+	const doubleCheck = false
+	if doubleCheck {
+		doubleCheckHeapPointersInterior(uintptr(ptr), uintptr(ptr), typ.Size_, typ.Size_, typ, &s.largeType, s)
+	}
+}
+
+type writeUserArenaHeapBits struct {
+	offset uintptr // offset in span that the low bit of mask represents the pointer state of.
+	mask   uintptr // some pointer bits starting at the address addr.
+	valid  uintptr // number of bits in buf that are valid (including low)
+	low    uintptr // number of low-order bits to not overwrite
+}
+
+func (s *mspan) writeUserArenaHeapBits(addr uintptr) (h writeUserArenaHeapBits) {
+	offset := addr - s.base()
+
+	// We start writing bits maybe in the middle of a heap bitmap word.
+	// Remember how many bits into the word we started, so we can be sure
+	// not to overwrite the previous bits.
+	h.low = offset / goarch.PtrSize % ptrBits
+
+	// round down to heap word that starts the bitmap word.
+	h.offset = offset - h.low*goarch.PtrSize
+
+	// We don't have any bits yet.
+	h.mask = 0
+	h.valid = h.low
+
+	return
+}
+
+// write appends the pointerness of the next valid pointer slots
+// using the low valid bits of bits. 1=pointer, 0=scalar.
+func (h writeUserArenaHeapBits) write(s *mspan, bits, valid uintptr) writeUserArenaHeapBits {
+	if h.valid+valid <= ptrBits {
+		// Fast path - just accumulate the bits.
+		h.mask |= bits << h.valid
+		h.valid += valid
+		return h
+	}
+	// Too many bits to fit in this word. Write the current word
+	// out and move on to the next word.
+
+	data := h.mask | bits<<h.valid       // mask for this word
+	h.mask = bits >> (ptrBits - h.valid) // leftover for next word
+	h.valid += valid - ptrBits           // have h.valid+valid bits, writing ptrBits of them
+
+	// Flush mask to the memory bitmap.
+	idx := h.offset / (ptrBits * goarch.PtrSize)
+	m := uintptr(1)<<h.low - 1
+	bitmap := s.heapBits()
+	bitmap[idx] = bswapIfBigEndian(bswapIfBigEndian(bitmap[idx])&m | data)
+	// Note: no synchronization required for this write because
+	// the allocator has exclusive access to the page, and the bitmap
+	// entries are all for a single page. Also, visibility of these
+	// writes is guaranteed by the publication barrier in mallocgc.
+
+	// Move to next word of bitmap.
+	h.offset += ptrBits * goarch.PtrSize
+	h.low = 0
+	return h
+}
+
+// Add padding of size bytes.
+func (h writeUserArenaHeapBits) pad(s *mspan, size uintptr) writeUserArenaHeapBits {
+	if size == 0 {
+		return h
+	}
+	words := size / goarch.PtrSize
+	for words > ptrBits {
+		h = h.write(s, 0, ptrBits)
+		words -= ptrBits
+	}
+	return h.write(s, 0, words)
+}
+
+// Flush the bits that have been written, and add zeros as needed
+// to cover the full object [addr, addr+size).
+func (h writeUserArenaHeapBits) flush(s *mspan, addr, size uintptr) {
+	offset := addr - s.base()
+
+	// zeros counts the number of bits needed to represent the object minus the
+	// number of bits we've already written. This is the number of 0 bits
+	// that need to be added.
+	zeros := (offset+size-h.offset)/goarch.PtrSize - h.valid
+
+	// Add zero bits up to the bitmap word boundary
+	if zeros > 0 {
+		z := ptrBits - h.valid
+		if z > zeros {
+			z = zeros
+		}
+		h.valid += z
+		zeros -= z
+	}
+
+	// Find word in bitmap that we're going to write.
+	bitmap := s.heapBits()
+	idx := h.offset / (ptrBits * goarch.PtrSize)
+
+	// Write remaining bits.
+	if h.valid != h.low {
+		m := uintptr(1)<<h.low - 1      // don't clear existing bits below "low"
+		m |= ^(uintptr(1)<<h.valid - 1) // don't clear existing bits above "valid"
+		bitmap[idx] = bswapIfBigEndian(bswapIfBigEndian(bitmap[idx])&m | h.mask)
+	}
+	if zeros == 0 {
+		return
+	}
+
+	// Advance to next bitmap word.
+	h.offset += ptrBits * goarch.PtrSize
+
+	// Continue on writing zeros for the rest of the object.
+	// For standard use of the ptr bits this is not required, as
+	// the bits are read from the beginning of the object. Some uses,
+	// like noscan spans, oblets, bulk write barriers, and cgocheck, might
+	// start mid-object, so these writes are still required.
+	for {
+		// Write zero bits.
+		idx := h.offset / (ptrBits * goarch.PtrSize)
+		if zeros < ptrBits {
+			bitmap[idx] = bswapIfBigEndian(bswapIfBigEndian(bitmap[idx]) &^ (uintptr(1)<<zeros - 1))
+			break
+		} else if zeros == ptrBits {
+			bitmap[idx] = 0
+			break
+		} else {
+			bitmap[idx] = 0
+			zeros -= ptrBits
+		}
+		h.offset += ptrBits * goarch.PtrSize
+	}
+}
+
+// bswapIfBigEndian swaps the byte order of the uintptr on goarch.BigEndian platforms,
+// and leaves it alone elsewhere.
+func bswapIfBigEndian(x uintptr) uintptr {
+	if goarch.BigEndian {
+		if goarch.PtrSize == 8 {
+			return uintptr(sys.Bswap64(uint64(x)))
+		}
+		return uintptr(sys.Bswap32(uint32(x)))
+	}
+	return x
+}
+
 // newUserArenaChunk allocates a user arena chunk, which maps to a single
 // heap arena and single span. Returns a pointer to the base of the chunk
 // (this is really important: we need to keep the chunk alive) and the span.
@@ -607,9 +800,7 @@ func newUserArenaChunk() (unsafe.Pointer, *mspan) {
 		// TODO(mknyszek): Track individual objects.
 		rzSize := computeRZlog(span.elemsize)
 		span.elemsize -= rzSize
-		if goexperiment.AllocHeaders {
-			span.largeType.Size_ = span.elemsize
-		}
+		span.largeType.Size_ = span.elemsize
 		rzStart := span.base() + span.elemsize
 		span.userArenaChunkFree = makeAddrRange(span.base(), rzStart)
 		asanpoison(unsafe.Pointer(rzStart), span.limit-rzStart)
@@ -924,13 +1115,12 @@ func (h *mheap) allocUserArenaChunk() *mspan {
 	// visible to the background sweeper.
 	h.central[spc].mcentral.fullSwept(h.sweepgen).push(s)
 
-	if goexperiment.AllocHeaders {
-		// Set up an allocation header. Avoid write barriers here because this type
-		// is not a real type, and it exists in an invalid location.
-		*(*uintptr)(unsafe.Pointer(&s.largeType)) = uintptr(unsafe.Pointer(s.limit))
-		*(*uintptr)(unsafe.Pointer(&s.largeType.GCData)) = s.limit + unsafe.Sizeof(_type{})
-		s.largeType.PtrBytes = 0
-		s.largeType.Size_ = s.elemsize
-	}
+	// Set up an allocation header. Avoid write barriers here because this type
+	// is not a real type, and it exists in an invalid location.
+	*(*uintptr)(unsafe.Pointer(&s.largeType)) = uintptr(unsafe.Pointer(s.limit))
+	*(*uintptr)(unsafe.Pointer(&s.largeType.GCData)) = s.limit + unsafe.Sizeof(_type{})
+	s.largeType.PtrBytes = 0
+	s.largeType.Size_ = s.elemsize
+
 	return s
 }

src/internal/goexperiment/exp_exectracer2_off.go

@@ -671,30 +671,15 @@ func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) {
 		if base == 0 {
 			return
 		}
-		if goexperiment.AllocHeaders {
-			tp := span.typePointersOfUnchecked(base)
-			for {
-				var addr uintptr
-				if tp, addr = tp.next(base + span.elemsize); addr == 0 {
-					break
-				}
-				pp := *(*unsafe.Pointer)(unsafe.Pointer(addr))
-				if cgoIsGoPointer(pp) && !isPinned(pp) {
-					panic(errorString(msg))
-				}
+		tp := span.typePointersOfUnchecked(base)
+		for {
+			var addr uintptr
+			if tp, addr = tp.next(base + span.elemsize); addr == 0 {
+				break
 			}
-		} else {
-			n := span.elemsize
-			hbits := heapBitsForAddr(base, n)
-			for {
-				var addr uintptr
-				if hbits, addr = hbits.next(); addr == 0 {
-					break
-				}
-				pp := *(*unsafe.Pointer)(unsafe.Pointer(addr))
-				if cgoIsGoPointer(pp) && !isPinned(pp) {
-					panic(errorString(msg))
-				}
+			pp := *(*unsafe.Pointer)(unsafe.Pointer(addr))
+			if cgoIsGoPointer(pp) && !isPinned(pp) {
+				panic(errorString(msg))
 			}
 		}
 		return