runtime: avoid zeroing scavenged memory

On Linux, memory returned to the kernel via MADV_DONTNEED is guaranteed
to be zero-filled on its next use.

This commit leverages this kernel behavior to avoid a redundant software
zeroing pass in the runtime, improving performance.

Signed-off-by: Lance Yang <[email protected]>

Author: ioworker0

src/runtime/mem.go

@@ -70,6 +70,12 @@ func sysUnused(v unsafe.Pointer, n uintptr) {
 	sysUnusedOS(v, n)
 }
 
+// sysNeedZeroAfterUnused reports whether memory returned by sysUnused must be
+// zeroed for use.
+func sysNeedZeroAfterUnused() bool {
+	return sysNeedZeroAfterUnusedOS()
+}
+
 // sysUsed transitions a memory region from Prepared to Ready. It notifies the
 // operating system that the memory region is needed and ensures that the region
 // may be safely accessed. This is typically a no-op on systems that don't have

src/runtime/mem_aix.go

@@ -79,3 +79,7 @@ func sysMapOS(v unsafe.Pointer, n uintptr, _ string) {
 		throw("runtime: cannot map pages in arena address space")
 	}
 }
+
+func sysNeedZeroAfterUnusedOS() bool {
+	return true
+}

src/runtime/mem_bsd.go

@@ -85,3 +85,7 @@ func sysMapOS(v unsafe.Pointer, n uintptr, _ string) {
 		throw("runtime: cannot map pages in arena address space")
 	}
 }
+
+func sysNeedZeroAfterUnusedOS() bool {
+	return true
+}

src/runtime/mem_darwin.go

@@ -74,3 +74,7 @@ func sysMapOS(v unsafe.Pointer, n uintptr, _ string) {
 		throw("runtime: cannot map pages in arena address space")
 	}
 }
+
+func sysNeedZeroAfterUnusedOS() bool {
+	return true
+}

src/runtime/mem_linux.go

@@ -188,3 +188,7 @@ func sysMapOS(v unsafe.Pointer, n uintptr, vmaName string) {
 		sysNoHugePageOS(v, n)
 	}
 }
+
+func sysNeedZeroAfterUnusedOS() bool {
+	return debug.madvdontneed == 0
+}

src/runtime/mem_sbrk.go

@@ -296,3 +296,7 @@ func sysReserveAlignedSbrk(size, align uintptr) (unsafe.Pointer, uintptr) {
 	})
 	return unsafe.Pointer(p), size
 }
+
+func sysNeedZeroAfterUnusedOS() bool {
+	return true
+}

src/runtime/mem_windows.go

@@ -132,3 +132,7 @@ func sysReserveOS(v unsafe.Pointer, n uintptr, _ string) unsafe.Pointer {
 
 func sysMapOS(v unsafe.Pointer, n uintptr, _ string) {
 }
+
+func sysNeedZeroAfterUnusedOS() bool {
+	return true
+}

src/runtime/mheap.go

@@ -1074,6 +1074,13 @@ func (h *mheap) setSpans(base, npage uintptr, s *mspan) {
 //
 // There are no locking constraints on this method.
 func (h *mheap) allocNeedsZero(base, npage uintptr) (needZero bool) {
+	// If these pages were scavenged (returned to the OS), the kernel guarantees
+	// they will be zero-filled on next use (fault-in), so we can treat them as
+	// already zeroed and skip explicit clearing.
+	if !sysNeedZeroAfterUnused() && h.pages.isScavenged(base, npage) {
+		return false
+	}
+
 	for npage > 0 {
 		ai := arenaIndex(base)
 		ha := h.arenas[ai.l1()][ai.l2()]

src/runtime/mpagealloc.go

@@ -481,6 +481,39 @@ func (p *pageAlloc) enableChunkHugePages() {
 	}
 }
 
+// isScavenged returns true if the page range [base, base+npages*pageSize) is
+// fully scavenged.
+//
+// p.mheapLock must be held.
+func (p *pageAlloc) isScavenged(base, npages uintptr) bool {
+	limit := base + npages*pageSize - 1
+	sc, ec := chunkIndex(base), chunkIndex(limit)
+	si, ei := chunkPageIndex(base), chunkPageIndex(limit)
+
+	if sc == ec {
+		// The range doesn't cross any chunk boundaries.
+		chunk := p.chunkOf(sc)
+		return chunk.scavenged.isRangeSet(si, ei+1-si)
+	}
+	// The range crosses at least one chunk boundary.
+	chunk := p.chunkOf(sc)
+	if !chunk.scavenged.isRangeSet(si, pallocChunkPages-si) {
+		return false
+	}
+	for c := sc + 1; c < ec; c++ {
+		chunk := p.chunkOf(c)
+		if !chunk.scavenged.isRangeSet(0, pallocChunkPages) {
+			return false
+		}
+	}
+	chunk = p.chunkOf(ec)
+	if !chunk.scavenged.isRangeSet(0, ei+1) {
+		return false
+	}
+	return true
+}
+
+
 // update updates heap metadata. It must be called each time the bitmap
 // is updated.
 //

src/runtime/mpallocbits.go

@@ -57,6 +57,42 @@ func (b *pageBits) setAll() {
 	}
 }
 
+// isRangeSet returns true if all bits in the range [i, i+n) are set.
+func (b *pageBits) isRangeSet(i, n uint) bool {
+	if n == 0 {
+		return true
+	}
+	// Check bits [i, j].
+	j := i + n - 1
+	if i/64 == j/64 {
+		// Fast path: the range fits in a single uint64.
+		mask := ((uint64(1) << n) - 1) << (i % 64)
+		return b[i/64]&mask == mask
+	}
+
+	// Slow path: the range spans multiple uint64s.
+
+	// Check leading bits.
+	if b[i/64]>>(i%64) != ^uint64(0)>>(i%64) {
+		return false
+	}
+
+	// Check full uint64s in the middle.
+	for k := i/64 + 1; k < j/64; k++ {
+		if b[k] != ^uint64(0) {
+			return false
+		}
+	}
+
+	// Check trailing bits.
+	mask := (uint64(1) << (j%64 + 1)) - 1
+	if b[j/64]&mask != mask {
+		return false
+	}
+
+	return true
+}
+
 // setBlock64 sets the 64-bit aligned block of bits containing the i'th bit that
 // are set in v.
 func (b *pageBits) setBlock64(i uint, v uint64) {