runtime: unify mallocStub fast/slow blocks Inside mallocStub, combine all of the fast path (!isSlowPath_) logic into a single block, and all of the slow path logic into a single block. This is primarily a refactor with no semantic change. The intention is to make it easier to read which part are fast/slow, and to make it easier for a follow up to change to acquirem early in the fast path. The only semantic change is to move doubleCheckMalloc earlier. If the fast path decides to fall back to the slow path, that case will execute twice, but it is a trivial check and it's even compiled in by default. This arrangement does require duplicating lockRankMayQueueFinalizer in both blocks. We can't move it earlier because the fast path falling back to the slow path would call this twice, which would trigger a lock rank violation. We could potentially move it below (just before inlineMalloc). I don't believe it needs to come before deductAssistCredit, but I'd rather do that in a separate change. For #79667. For #79699. Change-Id: I1101cb112e0c48280c0317d986607ba66a6a6964 Reviewed-on: https://go-review.googlesource.com/c/go/+/794120 Reviewed-by: Michael Matloob <matloob@google.com> Reviewed-by: Michael Matloob <matloob@golang.org> Auto-Submit: Michael Pratt <mpratt@google.com> LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/src/runtime/malloc_generated.go b/src/runtime/malloc_generated.go index c694aad..6fe8ad5 100644 --- a/src/runtime/malloc_generated.go +++ b/src/runtime/malloc_generated.go
@@ -11,6 +11,11 @@ ) func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -22,12 +27,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 1 @@ -161,6 +160,11 @@ } func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -172,12 +176,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 2 @@ -311,6 +309,11 @@ } func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -322,12 +325,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 3 @@ -461,6 +458,11 @@ } func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -472,12 +474,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 4 @@ -611,6 +607,11 @@ } func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -622,12 +623,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 5 @@ -761,6 +756,11 @@ } func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -772,12 +772,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 6 @@ -911,6 +905,11 @@ } func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -922,12 +921,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 7 @@ -1061,6 +1054,11 @@ } func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -1070,12 +1068,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const elemsize = 16 @@ -1165,6 +1157,11 @@ } func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -1176,12 +1173,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 2 @@ -1261,6 +1252,11 @@ } func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -1272,12 +1268,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 3 @@ -1357,6 +1347,11 @@ } func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -1368,12 +1363,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 4 @@ -1453,6 +1442,11 @@ } func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -1464,12 +1458,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 5 @@ -1549,6 +1537,11 @@ } func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -1560,12 +1553,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 6 @@ -1645,6 +1632,11 @@ } func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) @@ -1656,12 +1648,6 @@ } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") - } - } - lockRankMayQueueFinalizer() const sizeclass = 7 @@ -1741,18 +1727,17 @@ } func mallocgcTinySlowPath(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } - if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") } } + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { + return mallocgcSmallNoScanSC2(size, typ, needzero) + } + lockRankMayQueueFinalizer() if debug.malloc {
diff --git a/src/runtime/malloc_stubs.go b/src/runtime/malloc_stubs.go index 177d694..e3ac77f 100644 --- a/src/runtime/malloc_stubs.go +++ b/src/runtime/malloc_stubs.go
@@ -63,19 +63,14 @@ // WARNING: mallocStub does not do any work for sanitizers so callers need // to steer out of this codepath early if sanitizers are enabled. func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - if isSlowPath_ && isTiny_ { - // secret code, need to avoid the tiny allocator since it might keep - // co-located values alive longer and prevent timely zero-ing - // - // Call directly into the NoScan allocator. - // See go.dev/issue/76356 - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") } } if !isSlowPath_ { + // Fast path. Do we need to fall back to slow path? forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) if forceSlowPath { @@ -87,30 +82,44 @@ return mallocgcSlowPathStub(size, typ, needzero, spc, elemsize) } } - } - if doubleCheckMalloc { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") + // It's possible for any malloc to trigger sweeping, which may + // in turn queue finalizers. Record this dynamic lock edge. + // N.B. Compiled away if lockrank experiment is not enabled. + lockRankMayQueueFinalizer() + } else { + // Slow path. + if isTiny_ { + // secret code, need to avoid the tiny allocator since + // it might keep co-located values alive longer and + // prevent timely zero-ing. + // + // Call directly into the NoScan allocator. + // See go.dev/issue/76356 + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { + return mallocgcSmallNoScanSC2(size, typ, needzero) + } } - } - // It's possible for any malloc to trigger sweeping, which may in - // turn queue finalizers. Record this dynamic lock edge. - // N.B. Compiled away if lockrank experiment is not enabled. - lockRankMayQueueFinalizer() + // It's possible for any malloc to trigger sweeping, which may + // in turn queue finalizers. Record this dynamic lock edge. + // N.B. Compiled away if lockrank experiment is not enabled. + lockRankMayQueueFinalizer() - // Pre-malloc debug hooks. - if isSlowPath_ && debug.malloc { - if x := preMallocgcDebug(size, typ); x != nil { - return x + // Pre-malloc debug hooks. + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } } - } - // Assist the GC if needed. (On the reuse path, we currently compensate for this; - // changes here might require changes there.) - if isSlowPath_ && gcBlackenEnabled != 0 { - deductAssistCredit(size) + // Assist the GC if needed. (On the reuse path, we currently + // compensate for this; changes here might require changes + // there.) + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } } // Actually do the allocation.