runtime: dynamically allocate allp This makes it possible to eliminate the hard cap on GOMAXPROCS. Updates #15131. Change-Id: I4c422b340791621584c118a6be1b38e8a44f8b70 Reviewed-on: https://go-review.googlesource.com/45573 Run-TryBot: Austin Clements <austin@google.com> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Rick Hudson <rlh@golang.org>
diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 48ccfe8..23dc79d 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go
@@ -465,7 +465,7 @@ } // Clear per-P state - for _, p := range &allp { + for _, p := range allp { if p == nil { break } @@ -1662,7 +1662,7 @@ func gcBgMarkStartWorkers() { // Background marking is performed by per-P G's. Ensure that // each P has a background GC G. - for _, p := range &allp { + for _, p := range allp { if p == nil || p.status == _Pdead { break }
diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 9029d19..efc1a04 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go
@@ -1356,7 +1356,7 @@ // // The world must be stopped. func gcMarkTinyAllocs() { - for _, p := range &allp { + for _, p := range allp { if p == nil || p.status == _Pdead { break }
diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go index 1cb44a1..8538bad 100644 --- a/src/runtime/mstats.go +++ b/src/runtime/mstats.go
@@ -589,9 +589,13 @@ memstats.heap_objects = memstats.nmalloc - memstats.nfree } +// cachestats flushes all mcache stats. +// +// The world must be stopped. +// //go:nowritebarrier func cachestats() { - for _, p := range &allp { + for _, p := range allp { if p == nil { break }
diff --git a/src/runtime/proc.go b/src/runtime/proc.go index c58e806..188c897 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go
@@ -3231,7 +3231,7 @@ func gcount() int32 { n := int32(allglen) - sched.ngfree - int32(atomic.Load(&sched.ngsys)) - for _, _p_ := range &allp { + for _, _p_ := range allp { if _p_ == nil { break } @@ -3543,6 +3543,23 @@ } sched.procresizetime = now + // Grow allp if necessary. + if nprocs > int32(len(allp)) { + // Synchronize with retake, which could be running + // concurrently since it doesn't run on a P. + lock(&allpLock) + if nprocs <= int32(cap(allp)) { + allp = allp[:nprocs] + } else { + nallp := make([]*p, nprocs) + // Copy everything up to allp's cap so we + // never lose old allocated Ps. + copy(nallp, allp[:cap(allp)]) + allp = nallp + } + unlock(&allpLock) + } + // initialize new P's for i := int32(0); i < nprocs; i++ { pp := allp[i] @@ -3631,6 +3648,13 @@ // can't free P itself because it can be referenced by an M in syscall } + // Trim allp. + if int32(len(allp)) != nprocs { + lock(&allpLock) + allp = allp[:nprocs] + unlock(&allpLock) + } + _g_ := getg() if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs { // continue to use the current P @@ -3956,7 +3980,10 @@ func retake(now int64) uint32 { n := 0 - for i := int32(0); i < gomaxprocs; i++ { + // Prevent allp slice changes. This lock will be completely + // uncontended unless we're already stopping the world. + lock(&allpLock) + for i := 0; i < len(allp); i++ { _p_ := allp[i] if _p_ == nil { continue @@ -3977,6 +4004,8 @@ if runqempty(_p_) && atomic.Load(&sched.nmspinning)+atomic.Load(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now { continue } + // Drop allpLock so we can take sched.lock. + unlock(&allpLock) // Need to decrement number of idle locked M's // (pretending that one more is running) before the CAS. // Otherwise the M from which we retake can exit the syscall, @@ -3992,6 +4021,7 @@ handoffp(_p_) } incidlelocked(1) + lock(&allpLock) } else if s == _Prunning { // Preempt G if it's running for too long. t := int64(_p_.schedtick) @@ -4006,6 +4036,7 @@ preemptone(_p_) } } + unlock(&allpLock) return uint32(n) }
diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 27b1e37..269c5b1 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go
@@ -722,7 +722,8 @@ var ( allglen uintptr allm *m - allp [_MaxGomaxprocs + 1]*p + allp []*p // len(allp) == gomaxprocs; may change at safe points, otherwise immutable + allpLock mutex // Protects P-less reads of allp and all writes gomaxprocs int32 ncpu int32 forcegc forcegcstate
diff --git a/src/runtime/trace.go b/src/runtime/trace.go index 50e4c73..398d044 100644 --- a/src/runtime/trace.go +++ b/src/runtime/trace.go
@@ -277,7 +277,9 @@ traceGoSched() - for _, p := range &allp { + // Loop over all allocated Ps because dead Ps may still have + // trace buffers. + for _, p := range allp[:cap(allp)] { if p == nil { break } @@ -320,7 +322,7 @@ // The lock protects us from races with StartTrace/StopTrace because they do stop-the-world. lock(&trace.lock) - for _, p := range &allp { + for _, p := range allp[:cap(allp)] { if p == nil { break }