runtime: lock M for the entire fast path In size-specialized malloc, the fast path assumes that GC marking is not occurring. Thus once we decide to take the fast path, we must guarantee that the GC cannot start. No such guarantee exists today because the forceSlowPath check occurs outside of any locking. This may not have any impact on normal builds because we don't async preempt the runtime, and I'm not sure there are any preemptible calls before acquirem. But when inlining is disabled, there are preemptible calls which could thus allow a GC cycle to start. Fix this by calling acquirem prior to the forceSlowPath check and holding it through inlinedStub. Note that the slow path takes acquirem a bit later because we must not acquirem before deductAssistCredit, as that function won't run any assists when locked. For #79667. For #79699. Change-Id: Ief7220435b6a5fd70d721f58cbeac86e6a6a6964 Reviewed-on: https://go-review.googlesource.com/c/go/+/794121 Auto-Submit: Michael Pratt <mpratt@google.com> Reviewed-by: Michael Matloob <matloob@google.com> Reviewed-by: Michael Matloob <matloob@golang.org> LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/src/runtime/malloc_generated.go b/src/runtime/malloc_generated.go index 6fe8ad5..9500072 100644 --- a/src/runtime/malloc_generated.go +++ b/src/runtime/malloc_generated.go
@@ -17,9 +17,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(1<<1) | spanClass(0) const elemsize = uintptr(8) @@ -32,7 +37,6 @@ const sizeclass = 1 const elemsize = 8 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallScanNoHeader(size, typ, mp) @@ -166,9 +170,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(2<<1) | spanClass(0) const elemsize = uintptr(16) @@ -181,7 +190,6 @@ const sizeclass = 2 const elemsize = 16 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallScanNoHeader(size, typ, mp) @@ -315,9 +323,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(3<<1) | spanClass(0) const elemsize = uintptr(24) @@ -330,7 +343,6 @@ const sizeclass = 3 const elemsize = 24 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallScanNoHeader(size, typ, mp) @@ -464,9 +476,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(4<<1) | spanClass(0) const elemsize = uintptr(32) @@ -479,7 +496,6 @@ const sizeclass = 4 const elemsize = 32 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallScanNoHeader(size, typ, mp) @@ -613,9 +629,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(5<<1) | spanClass(0) const elemsize = uintptr(48) @@ -628,7 +649,6 @@ const sizeclass = 5 const elemsize = 48 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallScanNoHeader(size, typ, mp) @@ -762,9 +782,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(6<<1) | spanClass(0) const elemsize = uintptr(64) @@ -777,7 +802,6 @@ const sizeclass = 6 const elemsize = 64 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallScanNoHeader(size, typ, mp) @@ -911,9 +935,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(7<<1) | spanClass(0) const elemsize = uintptr(80) @@ -926,7 +955,6 @@ const sizeclass = 7 const elemsize = 80 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallScanNoHeader(size, typ, mp) @@ -1060,9 +1088,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) return mallocgcTinySlowPath(size, typ, needzero) @@ -1072,7 +1105,6 @@ const elemsize = 16 - mp := acquirem() if doubleCheckMalloc { doubleCheckTiny(size, typ, mp) } @@ -1163,9 +1195,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(2<<1) | spanClass(1) const elemsize = uintptr(16) @@ -1178,7 +1215,6 @@ const sizeclass = 2 const elemsize = 16 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallNoScan(typ, mp) @@ -1258,9 +1294,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(3<<1) | spanClass(1) const elemsize = uintptr(24) @@ -1273,7 +1314,6 @@ const sizeclass = 3 const elemsize = 24 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallNoScan(typ, mp) @@ -1353,9 +1393,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(4<<1) | spanClass(1) const elemsize = uintptr(32) @@ -1368,7 +1413,6 @@ const sizeclass = 4 const elemsize = 32 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallNoScan(typ, mp) @@ -1448,9 +1492,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(5<<1) | spanClass(1) const elemsize = uintptr(48) @@ -1463,7 +1512,6 @@ const sizeclass = 5 const elemsize = 48 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallNoScan(typ, mp) @@ -1543,9 +1591,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(6<<1) | spanClass(1) const elemsize = uintptr(64) @@ -1558,7 +1611,6 @@ const sizeclass = 6 const elemsize = 64 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallNoScan(typ, mp) @@ -1638,9 +1690,14 @@ } } + var mp *m + + mp = acquirem() + forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) const spc = spanClass(7<<1) | spanClass(1) const elemsize = uintptr(80) @@ -1653,7 +1710,6 @@ const sizeclass = 7 const elemsize = 80 - mp := acquirem() if doubleCheckMalloc { doubleCheckSmallNoScan(typ, mp) @@ -1733,6 +1789,8 @@ } } + var mp *m + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) @@ -1758,9 +1816,10 @@ } } + mp = acquirem() + const elemsize = 16 - mp := acquirem() if doubleCheckMalloc { doubleCheckTiny(size, typ, mp) } @@ -1876,6 +1935,8 @@ } } + var mp *m + lockRankMayQueueFinalizer() if debug.malloc { @@ -1896,7 +1957,8 @@ } } - mp := acquirem() + mp = acquirem() + if doubleCheckMalloc { doubleCheckSmallScanNoHeader(size, typ, mp) @@ -2050,6 +2112,8 @@ } } + var mp *m + lockRankMayQueueFinalizer() if debug.malloc { @@ -2070,7 +2134,8 @@ } } - mp := acquirem() + mp = acquirem() + if doubleCheckMalloc { doubleCheckSmallNoScan(typ, mp)
diff --git a/src/runtime/malloc_stubs.go b/src/runtime/malloc_stubs.go index e3ac77f..337055c 100644 --- a/src/runtime/malloc_stubs.go +++ b/src/runtime/malloc_stubs.go
@@ -69,11 +69,20 @@ } } + var mp *m if !isSlowPath_ { - // Fast path. Do we need to fall back to slow path? + // Fast path. + + // The fast path assumes that GC marking is not running. We + // must acquirem to ensure the GC does not start after we + // check. + mp = acquirem() + + // Do we need to fall back to slow path? forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { + releasem(mp) // Slow path will reacquire. if isTiny_ { return mallocgcTinySlowPath(size, typ, needzero) } else { @@ -120,10 +129,12 @@ if gcBlackenEnabled != 0 { deductAssistCredit(size) } + + mp = acquirem() } // Actually do the allocation. - return inlinedMalloc(size, typ, needzero) + return inlinedMalloc(mp, size, typ, needzero) } func postMallocgc(x unsafe.Pointer, typ *_type, size uintptr, elemsize uintptr) { @@ -172,7 +183,10 @@ // will be replaced with the inlined body of smallStub or tinyStub when generating the // size-specialized malloc function. See the comment at the top of this file for more // information. -func inlinedMalloc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { +// +// The caller must acquirem prior to calling inlinedMalloc, which will releasem +// before returning. +func inlinedMalloc(mp *m, size uintptr, typ *_type, needzero bool) unsafe.Pointer { return unsafe.Pointer(uintptr(0)) } @@ -191,12 +205,13 @@ } } -func smallStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { +// The caller must acquirem prior to calling smallStub, which will releasem +// before returning. +func smallStub(mp *m, size uintptr, typ *_type, needzero bool) unsafe.Pointer { const sizeclass = sizeclass_ const elemsize = elemsize_ // Set mp.mallocing to keep from being preempted by GC. - mp := acquirem() if doubleCheckMalloc { if isNoScan_ { doubleCheckSmallNoScan(typ, mp) @@ -340,11 +355,12 @@ } } -func tinyStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { +// The caller must acquirem prior to calling tinyStub, which will releasem +// before returning. +func tinyStub(mp *m, size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = elemsize_ // Set mp.mallocing to keep from being preempted by GC. - mp := acquirem() if doubleCheckMalloc { doubleCheckTiny(size, typ, mp) }