| // Copyright 2013 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package runtime |
| |
| import ( |
| "internal/cpu" |
| "runtime/internal/atomic" |
| "runtime/internal/sys" |
| "unsafe" |
| ) |
| |
| /* |
| Stack layout parameters. |
| Included both by runtime (compiled via 6c) and linkers (compiled via gcc). |
| |
| The per-goroutine g->stackguard is set to point StackGuard bytes |
| above the bottom of the stack. Each function compares its stack |
| pointer against g->stackguard to check for overflow. To cut one |
| instruction from the check sequence for functions with tiny frames, |
| the stack is allowed to protrude StackSmall bytes below the stack |
| guard. Functions with large frames don't bother with the check and |
| always call morestack. The sequences are (for amd64, others are |
| similar): |
| |
| guard = g->stackguard |
| frame = function's stack frame size |
| argsize = size of function arguments (call + return) |
| |
| stack frame size <= StackSmall: |
| CMPQ guard, SP |
| JHI 3(PC) |
| MOVQ m->morearg, $(argsize << 32) |
| CALL morestack(SB) |
| |
| stack frame size > StackSmall but < StackBig |
| LEAQ (frame-StackSmall)(SP), R0 |
| CMPQ guard, R0 |
| JHI 3(PC) |
| MOVQ m->morearg, $(argsize << 32) |
| CALL morestack(SB) |
| |
| stack frame size >= StackBig: |
| MOVQ m->morearg, $((argsize << 32) | frame) |
| CALL morestack(SB) |
| |
| The bottom StackGuard - StackSmall bytes are important: there has |
| to be enough room to execute functions that refuse to check for |
| stack overflow, either because they need to be adjacent to the |
| actual caller's frame (deferproc) or because they handle the imminent |
| stack overflow (morestack). |
| |
| For example, deferproc might call malloc, which does one of the |
| above checks (without allocating a full frame), which might trigger |
| a call to morestack. This sequence needs to fit in the bottom |
| section of the stack. On amd64, morestack's frame is 40 bytes, and |
| deferproc's frame is 56 bytes. That fits well within the |
| StackGuard - StackSmall bytes at the bottom. |
| The linkers explore all possible call traces involving non-splitting |
| functions to make sure that this limit cannot be violated. |
| */ |
| |
| const ( |
| // StackSystem is a number of additional bytes to add |
| // to each stack below the usual guard area for OS-specific |
| // purposes like signal handling. Used on Windows, Plan 9, |
| // and iOS because they do not use a separate stack. |
| _StackSystem = sys.GoosWindows*512*sys.PtrSize + sys.GoosPlan9*512 + sys.GoosIos*sys.GoarchArm64*1024 |
| |
| // The minimum size of stack used by Go code |
| _StackMin = 2048 |
| |
| // The minimum stack size to allocate. |
| // The hackery here rounds FixedStack0 up to a power of 2. |
| _FixedStack0 = _StackMin + _StackSystem |
| _FixedStack1 = _FixedStack0 - 1 |
| _FixedStack2 = _FixedStack1 | (_FixedStack1 >> 1) |
| _FixedStack3 = _FixedStack2 | (_FixedStack2 >> 2) |
| _FixedStack4 = _FixedStack3 | (_FixedStack3 >> 4) |
| _FixedStack5 = _FixedStack4 | (_FixedStack4 >> 8) |
| _FixedStack6 = _FixedStack5 | (_FixedStack5 >> 16) |
| _FixedStack = _FixedStack6 + 1 |
| |
| // Functions that need frames bigger than this use an extra |
| // instruction to do the stack split check, to avoid overflow |
| // in case SP - framesize wraps below zero. |
| // This value can be no bigger than the size of the unmapped |
| // space at zero. |
| _StackBig = 4096 |
| |
| // The stack guard is a pointer this many bytes above the |
| // bottom of the stack. |
| _StackGuard = 928*sys.StackGuardMultiplier + _StackSystem |
| |
| // After a stack split check the SP is allowed to be this |
| // many bytes below the stack guard. This saves an instruction |
| // in the checking sequence for tiny frames. |
| _StackSmall = 128 |
| |
| // The maximum number of bytes that a chain of NOSPLIT |
| // functions can use. |
| _StackLimit = _StackGuard - _StackSystem - _StackSmall |
| ) |
| |
| const ( |
| // stackDebug == 0: no logging |
| // == 1: logging of per-stack operations |
| // == 2: logging of per-frame operations |
| // == 3: logging of per-word updates |
| // == 4: logging of per-word reads |
| stackDebug = 0 |
| stackFromSystem = 0 // allocate stacks from system memory instead of the heap |
| stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free |
| stackPoisonCopy = 1 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy |
| stackNoCache = 0 // disable per-P small stack caches |
| |
| // check the BP links during traceback. |
| debugCheckBP = false |
| ) |
| |
| const ( |
| uintptrMask = 1<<(8*sys.PtrSize) - 1 |
| |
| // Goroutine preemption request. |
| // Stored into g->stackguard0 to cause split stack check failure. |
| // Must be greater than any real sp. |
| // 0xfffffade in hex. |
| stackPreempt = uintptrMask & -1314 |
| |
| // Thread is forking. |
| // Stored into g->stackguard0 to cause split stack check failure. |
| // Must be greater than any real sp. |
| stackFork = uintptrMask & -1234 |
| ) |
| |
| // Global pool of spans that have free stacks. |
| // Stacks are assigned an order according to size. |
| // order = log_2(size/FixedStack) |
| // There is a free list for each order. |
| var stackpool [_NumStackOrders]struct { |
| item stackpoolItem |
| _ [cpu.CacheLinePadSize - unsafe.Sizeof(stackpoolItem{})%cpu.CacheLinePadSize]byte |
| } |
| |
| //go:notinheap |
| type stackpoolItem struct { |
| mu mutex |
| span mSpanList |
| } |
| |
| // Global pool of large stack spans. |
| var stackLarge struct { |
| lock mutex |
| free [heapAddrBits - pageShift]mSpanList // free lists by log_2(s.npages) |
| } |
| |
| func stackinit() { |
| if _StackCacheSize&_PageMask != 0 { |
| throw("cache size must be a multiple of page size") |
| } |
| for i := range stackpool { |
| stackpool[i].item.span.init() |
| lockInit(&stackpool[i].item.mu, lockRankStackpool) |
| } |
| for i := range stackLarge.free { |
| stackLarge.free[i].init() |
| lockInit(&stackLarge.lock, lockRankStackLarge) |
| } |
| } |
| |
| // stacklog2 returns ⌊log_2(n)⌋. |
| func stacklog2(n uintptr) int { |
| log2 := 0 |
| for n > 1 { |
| n >>= 1 |
| log2++ |
| } |
| return log2 |
| } |
| |
| // Allocates a stack from the free pool. Must be called with |
| // stackpool[order].item.mu held. |
| func stackpoolalloc(order uint8) gclinkptr { |
| list := &stackpool[order].item.span |
| s := list.first |
| lockWithRankMayAcquire(&mheap_.lock, lockRankMheap) |
| if s == nil { |
| // no free stacks. Allocate another span worth. |
| s = mheap_.allocManual(_StackCacheSize>>_PageShift, spanAllocStack) |
| if s == nil { |
| throw("out of memory") |
| } |
| if s.allocCount != 0 { |
| throw("bad allocCount") |
| } |
| if s.manualFreeList.ptr() != nil { |
| throw("bad manualFreeList") |
| } |
| osStackAlloc(s) |
| s.elemsize = _FixedStack << order |
| for i := uintptr(0); i < _StackCacheSize; i += s.elemsize { |
| x := gclinkptr(s.base() + i) |
| x.ptr().next = s.manualFreeList |
| s.manualFreeList = x |
| } |
| list.insert(s) |
| } |
| x := s.manualFreeList |
| if x.ptr() == nil { |
| throw("span has no free stacks") |
| } |
| s.manualFreeList = x.ptr().next |
| s.allocCount++ |
| if s.manualFreeList.ptr() == nil { |
| // all stacks in s are allocated. |
| list.remove(s) |
| } |
| return x |
| } |
| |
| // Adds stack x to the free pool. Must be called with stackpool[order].item.mu held. |
| func stackpoolfree(x gclinkptr, order uint8) { |
| s := spanOfUnchecked(uintptr(x)) |
| if s.state.get() != mSpanManual { |
| throw("freeing stack not in a stack span") |
| } |
| if s.manualFreeList.ptr() == nil { |
| // s will now have a free stack |
| stackpool[order].item.span.insert(s) |
| } |
| x.ptr().next = s.manualFreeList |
| s.manualFreeList = x |
| s.allocCount-- |
| if gcphase == _GCoff && s.allocCount == 0 { |
| // Span is completely free. Return it to the heap |
| // immediately if we're sweeping. |
| // |
| // If GC is active, we delay the free until the end of |
| // GC to avoid the following type of situation: |
| // |
| // 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer |
| // 2) The stack that pointer points to is copied |
| // 3) The old stack is freed |
| // 4) The containing span is marked free |
| // 5) GC attempts to mark the SudoG.elem pointer. The |
| // marking fails because the pointer looks like a |
| // pointer into a free span. |
| // |
| // By not freeing, we prevent step #4 until GC is done. |
| stackpool[order].item.span.remove(s) |
| s.manualFreeList = 0 |
| osStackFree(s) |
| mheap_.freeManual(s, spanAllocStack) |
| } |
| } |
| |
| // stackcacherefill/stackcacherelease implement a global pool of stack segments. |
| // The pool is required to prevent unlimited growth of per-thread caches. |
| // |
| //go:systemstack |
| func stackcacherefill(c *mcache, order uint8) { |
| if stackDebug >= 1 { |
| print("stackcacherefill order=", order, "\n") |
| } |
| |
| // Grab some stacks from the global cache. |
| // Grab half of the allowed capacity (to prevent thrashing). |
| var list gclinkptr |
| var size uintptr |
| lock(&stackpool[order].item.mu) |
| for size < _StackCacheSize/2 { |
| x := stackpoolalloc(order) |
| x.ptr().next = list |
| list = x |
| size += _FixedStack << order |
| } |
| unlock(&stackpool[order].item.mu) |
| c.stackcache[order].list = list |
| c.stackcache[order].size = size |
| } |
| |
| //go:systemstack |
| func stackcacherelease(c *mcache, order uint8) { |
| if stackDebug >= 1 { |
| print("stackcacherelease order=", order, "\n") |
| } |
| x := c.stackcache[order].list |
| size := c.stackcache[order].size |
| lock(&stackpool[order].item.mu) |
| for size > _StackCacheSize/2 { |
| y := x.ptr().next |
| stackpoolfree(x, order) |
| x = y |
| size -= _FixedStack << order |
| } |
| unlock(&stackpool[order].item.mu) |
| c.stackcache[order].list = x |
| c.stackcache[order].size = size |
| } |
| |
| //go:systemstack |
| func stackcache_clear(c *mcache) { |
| if stackDebug >= 1 { |
| print("stackcache clear\n") |
| } |
| for order := uint8(0); order < _NumStackOrders; order++ { |
| lock(&stackpool[order].item.mu) |
| x := c.stackcache[order].list |
| for x.ptr() != nil { |
| y := x.ptr().next |
| stackpoolfree(x, order) |
| x = y |
| } |
| c.stackcache[order].list = 0 |
| c.stackcache[order].size = 0 |
| unlock(&stackpool[order].item.mu) |
| } |
| } |
| |
| // stackalloc allocates an n byte stack. |
| // |
| // stackalloc must run on the system stack because it uses per-P |
| // resources and must not split the stack. |
| // |
| //go:systemstack |
| func stackalloc(n uint32) stack { |
| // Stackalloc must be called on scheduler stack, so that we |
| // never try to grow the stack during the code that stackalloc runs. |
| // Doing so would cause a deadlock (issue 1547). |
| thisg := getg() |
| if thisg != thisg.m.g0 { |
| throw("stackalloc not on scheduler stack") |
| } |
| if n&(n-1) != 0 { |
| throw("stack size not a power of 2") |
| } |
| if stackDebug >= 1 { |
| print("stackalloc ", n, "\n") |
| } |
| |
| if debug.efence != 0 || stackFromSystem != 0 { |
| n = uint32(alignUp(uintptr(n), physPageSize)) |
| v := sysAlloc(uintptr(n), &memstats.stacks_sys) |
| if v == nil { |
| throw("out of memory (stackalloc)") |
| } |
| return stack{uintptr(v), uintptr(v) + uintptr(n)} |
| } |
| |
| // Small stacks are allocated with a fixed-size free-list allocator. |
| // If we need a stack of a bigger size, we fall back on allocating |
| // a dedicated span. |
| var v unsafe.Pointer |
| if n < _FixedStack<<_NumStackOrders && n < _StackCacheSize { |
| order := uint8(0) |
| n2 := n |
| for n2 > _FixedStack { |
| order++ |
| n2 >>= 1 |
| } |
| var x gclinkptr |
| if stackNoCache != 0 || thisg.m.p == 0 || thisg.m.preemptoff != "" { |
| // thisg.m.p == 0 can happen in the guts of exitsyscall |
| // or procresize. Just get a stack from the global pool. |
| // Also don't touch stackcache during gc |
| // as it's flushed concurrently. |
| lock(&stackpool[order].item.mu) |
| x = stackpoolalloc(order) |
| unlock(&stackpool[order].item.mu) |
| } else { |
| c := thisg.m.p.ptr().mcache |
| x = c.stackcache[order].list |
| if x.ptr() == nil { |
| stackcacherefill(c, order) |
| x = c.stackcache[order].list |
| } |
| c.stackcache[order].list = x.ptr().next |
| c.stackcache[order].size -= uintptr(n) |
| } |
| v = unsafe.Pointer(x) |
| } else { |
| var s *mspan |
| npage := uintptr(n) >> _PageShift |
| log2npage := stacklog2(npage) |
| |
| // Try to get a stack from the large stack cache. |
| lock(&stackLarge.lock) |
| if !stackLarge.free[log2npage].isEmpty() { |
| s = stackLarge.free[log2npage].first |
| stackLarge.free[log2npage].remove(s) |
| } |
| unlock(&stackLarge.lock) |
| |
| lockWithRankMayAcquire(&mheap_.lock, lockRankMheap) |
| |
| if s == nil { |
| // Allocate a new stack from the heap. |
| s = mheap_.allocManual(npage, spanAllocStack) |
| if s == nil { |
| throw("out of memory") |
| } |
| osStackAlloc(s) |
| s.elemsize = uintptr(n) |
| } |
| v = unsafe.Pointer(s.base()) |
| } |
| |
| if raceenabled { |
| racemalloc(v, uintptr(n)) |
| } |
| if msanenabled { |
| msanmalloc(v, uintptr(n)) |
| } |
| if stackDebug >= 1 { |
| print(" allocated ", v, "\n") |
| } |
| return stack{uintptr(v), uintptr(v) + uintptr(n)} |
| } |
| |
| // stackfree frees an n byte stack allocation at stk. |
| // |
| // stackfree must run on the system stack because it uses per-P |
| // resources and must not split the stack. |
| // |
| //go:systemstack |
| func stackfree(stk stack) { |
| gp := getg() |
| v := unsafe.Pointer(stk.lo) |
| n := stk.hi - stk.lo |
| if n&(n-1) != 0 { |
| throw("stack not a power of 2") |
| } |
| if stk.lo+n < stk.hi { |
| throw("bad stack size") |
| } |
| if stackDebug >= 1 { |
| println("stackfree", v, n) |
| memclrNoHeapPointers(v, n) // for testing, clobber stack data |
| } |
| if debug.efence != 0 || stackFromSystem != 0 { |
| if debug.efence != 0 || stackFaultOnFree != 0 { |
| sysFault(v, n) |
| } else { |
| sysFree(v, n, &memstats.stacks_sys) |
| } |
| return |
| } |
| if msanenabled { |
| msanfree(v, n) |
| } |
| if n < _FixedStack<<_NumStackOrders && n < _StackCacheSize { |
| order := uint8(0) |
| n2 := n |
| for n2 > _FixedStack { |
| order++ |
| n2 >>= 1 |
| } |
| x := gclinkptr(v) |
| if stackNoCache != 0 || gp.m.p == 0 || gp.m.preemptoff != "" { |
| lock(&stackpool[order].item.mu) |
| stackpoolfree(x, order) |
| unlock(&stackpool[order].item.mu) |
| } else { |
| c := gp.m.p.ptr().mcache |
| if c.stackcache[order].size >= _StackCacheSize { |
| stackcacherelease(c, order) |
| } |
| x.ptr().next = c.stackcache[order].list |
| c.stackcache[order].list = x |
| c.stackcache[order].size += n |
| } |
| } else { |
| s := spanOfUnchecked(uintptr(v)) |
| if s.state.get() != mSpanManual { |
| println(hex(s.base()), v) |
| throw("bad span state") |
| } |
| if gcphase == _GCoff { |
| // Free the stack immediately if we're |
| // sweeping. |
| osStackFree(s) |
| mheap_.freeManual(s, spanAllocStack) |
| } else { |
| // If the GC is running, we can't return a |
| // stack span to the heap because it could be |
| // reused as a heap span, and this state |
| // change would race with GC. Add it to the |
| // large stack cache instead. |
| log2npage := stacklog2(s.npages) |
| lock(&stackLarge.lock) |
| stackLarge.free[log2npage].insert(s) |
| unlock(&stackLarge.lock) |
| } |
| } |
| } |
| |
| var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real |
| |
| var maxstackceiling = maxstacksize |
| |
| var ptrnames = []string{ |
| 0: "scalar", |
| 1: "ptr", |
| } |
| |
| // Stack frame layout |
| // |
| // (x86) |
| // +------------------+ |
| // | args from caller | |
| // +------------------+ <- frame->argp |
| // | return address | |
| // +------------------+ |
| // | caller's BP (*) | (*) if framepointer_enabled && varp < sp |
| // +------------------+ <- frame->varp |
| // | locals | |
| // +------------------+ |
| // | args to callee | |
| // +------------------+ <- frame->sp |
| // |
| // (arm) |
| // +------------------+ |
| // | args from caller | |
| // +------------------+ <- frame->argp |
| // | caller's retaddr | |
| // +------------------+ <- frame->varp |
| // | locals | |
| // +------------------+ |
| // | args to callee | |
| // +------------------+ |
| // | return address | |
| // +------------------+ <- frame->sp |
| |
| type adjustinfo struct { |
| old stack |
| delta uintptr // ptr distance from old to new stack (newbase - oldbase) |
| cache pcvalueCache |
| |
| // sghi is the highest sudog.elem on the stack. |
| sghi uintptr |
| } |
| |
| // Adjustpointer checks whether *vpp is in the old stack described by adjinfo. |
| // If so, it rewrites *vpp to point into the new stack. |
| func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) { |
| pp := (*uintptr)(vpp) |
| p := *pp |
| if stackDebug >= 4 { |
| print(" ", pp, ":", hex(p), "\n") |
| } |
| if adjinfo.old.lo <= p && p < adjinfo.old.hi { |
| *pp = p + adjinfo.delta |
| if stackDebug >= 3 { |
| print(" adjust ptr ", pp, ":", hex(p), " -> ", hex(*pp), "\n") |
| } |
| } |
| } |
| |
| // Information from the compiler about the layout of stack frames. |
| // Note: this type must agree with reflect.bitVector. |
| type bitvector struct { |
| n int32 // # of bits |
| bytedata *uint8 |
| } |
| |
| // ptrbit returns the i'th bit in bv. |
| // ptrbit is less efficient than iterating directly over bitvector bits, |
| // and should only be used in non-performance-critical code. |
| // See adjustpointers for an example of a high-efficiency walk of a bitvector. |
| func (bv *bitvector) ptrbit(i uintptr) uint8 { |
| b := *(addb(bv.bytedata, i/8)) |
| return (b >> (i % 8)) & 1 |
| } |
| |
| // bv describes the memory starting at address scanp. |
| // Adjust any pointers contained therein. |
| func adjustpointers(scanp unsafe.Pointer, bv *bitvector, adjinfo *adjustinfo, f funcInfo) { |
| minp := adjinfo.old.lo |
| maxp := adjinfo.old.hi |
| delta := adjinfo.delta |
| num := uintptr(bv.n) |
| // If this frame might contain channel receive slots, use CAS |
| // to adjust pointers. If the slot hasn't been received into |
| // yet, it may contain stack pointers and a concurrent send |
| // could race with adjusting those pointers. (The sent value |
| // itself can never contain stack pointers.) |
| useCAS := uintptr(scanp) < adjinfo.sghi |
| for i := uintptr(0); i < num; i += 8 { |
| if stackDebug >= 4 { |
| for j := uintptr(0); j < 8; j++ { |
| print(" ", add(scanp, (i+j)*sys.PtrSize), ":", ptrnames[bv.ptrbit(i+j)], ":", hex(*(*uintptr)(add(scanp, (i+j)*sys.PtrSize))), " # ", i, " ", *addb(bv.bytedata, i/8), "\n") |
| } |
| } |
| b := *(addb(bv.bytedata, i/8)) |
| for b != 0 { |
| j := uintptr(sys.Ctz8(b)) |
| b &= b - 1 |
| pp := (*uintptr)(add(scanp, (i+j)*sys.PtrSize)) |
| retry: |
| p := *pp |
| if f.valid() && 0 < p && p < minLegalPointer && debug.invalidptr != 0 { |
| // Looks like a junk value in a pointer slot. |
| // Live analysis wrong? |
| getg().m.traceback = 2 |
| print("runtime: bad pointer in frame ", funcname(f), " at ", pp, ": ", hex(p), "\n") |
| throw("invalid pointer found on stack") |
| } |
| if minp <= p && p < maxp { |
| if stackDebug >= 3 { |
| print("adjust ptr ", hex(p), " ", funcname(f), "\n") |
| } |
| if useCAS { |
| ppu := (*unsafe.Pointer)(unsafe.Pointer(pp)) |
| if !atomic.Casp1(ppu, unsafe.Pointer(p), unsafe.Pointer(p+delta)) { |
| goto retry |
| } |
| } else { |
| *pp = p + delta |
| } |
| } |
| } |
| } |
| } |
| |
| // Note: the argument/return area is adjusted by the callee. |
| func adjustframe(frame *stkframe, arg unsafe.Pointer) bool { |
| adjinfo := (*adjustinfo)(arg) |
| if frame.continpc == 0 { |
| // Frame is dead. |
| return true |
| } |
| f := frame.fn |
| if stackDebug >= 2 { |
| print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n") |
| } |
| if f.funcID == funcID_systemstack_switch { |
| // A special routine at the bottom of stack of a goroutine that does a systemstack call. |
| // We will allow it to be copied even though we don't |
| // have full GC info for it (because it is written in asm). |
| return true |
| } |
| |
| locals, args, objs := getStackMap(frame, &adjinfo.cache, true) |
| |
| // Adjust local variables if stack frame has been allocated. |
| if locals.n > 0 { |
| size := uintptr(locals.n) * sys.PtrSize |
| adjustpointers(unsafe.Pointer(frame.varp-size), &locals, adjinfo, f) |
| } |
| |
| // Adjust saved base pointer if there is one. |
| // TODO what about arm64 frame pointer adjustment? |
| if sys.ArchFamily == sys.AMD64 && frame.argp-frame.varp == 2*sys.PtrSize { |
| if stackDebug >= 3 { |
| print(" saved bp\n") |
| } |
| if debugCheckBP { |
| // Frame pointers should always point to the next higher frame on |
| // the Go stack (or be nil, for the top frame on the stack). |
| bp := *(*uintptr)(unsafe.Pointer(frame.varp)) |
| if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) { |
| println("runtime: found invalid frame pointer") |
| print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n") |
| throw("bad frame pointer") |
| } |
| } |
| adjustpointer(adjinfo, unsafe.Pointer(frame.varp)) |
| } |
| |
| // Adjust arguments. |
| if args.n > 0 { |
| if stackDebug >= 3 { |
| print(" args\n") |
| } |
| adjustpointers(unsafe.Pointer(frame.argp), &args, adjinfo, funcInfo{}) |
| } |
| |
| // Adjust pointers in all stack objects (whether they are live or not). |
| // See comments in mgcmark.go:scanframeworker. |
| if frame.varp != 0 { |
| for _, obj := range objs { |
| off := obj.off |
| base := frame.varp // locals base pointer |
| if off >= 0 { |
| base = frame.argp // arguments and return values base pointer |
| } |
| p := base + uintptr(off) |
| if p < frame.sp { |
| // Object hasn't been allocated in the frame yet. |
| // (Happens when the stack bounds check fails and |
| // we call into morestack.) |
| continue |
| } |
| t := obj.typ |
| gcdata := t.gcdata |
| var s *mspan |
| if t.kind&kindGCProg != 0 { |
| // See comments in mgcmark.go:scanstack |
| s = materializeGCProg(t.ptrdata, gcdata) |
| gcdata = (*byte)(unsafe.Pointer(s.startAddr)) |
| } |
| for i := uintptr(0); i < t.ptrdata; i += sys.PtrSize { |
| if *addb(gcdata, i/(8*sys.PtrSize))>>(i/sys.PtrSize&7)&1 != 0 { |
| adjustpointer(adjinfo, unsafe.Pointer(p+i)) |
| } |
| } |
| if s != nil { |
| dematerializeGCProg(s) |
| } |
| } |
| } |
| |
| return true |
| } |
| |
| func adjustctxt(gp *g, adjinfo *adjustinfo) { |
| adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt)) |
| if !framepointer_enabled { |
| return |
| } |
| if debugCheckBP { |
| bp := gp.sched.bp |
| if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) { |
| println("runtime: found invalid top frame pointer") |
| print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n") |
| throw("bad top frame pointer") |
| } |
| } |
| adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.bp)) |
| } |
| |
| func adjustdefers(gp *g, adjinfo *adjustinfo) { |
| // Adjust pointers in the Defer structs. |
| // We need to do this first because we need to adjust the |
| // defer.link fields so we always work on the new stack. |
| adjustpointer(adjinfo, unsafe.Pointer(&gp._defer)) |
| for d := gp._defer; d != nil; d = d.link { |
| adjustpointer(adjinfo, unsafe.Pointer(&d.fn)) |
| adjustpointer(adjinfo, unsafe.Pointer(&d.sp)) |
| adjustpointer(adjinfo, unsafe.Pointer(&d._panic)) |
| adjustpointer(adjinfo, unsafe.Pointer(&d.link)) |
| adjustpointer(adjinfo, unsafe.Pointer(&d.varp)) |
| adjustpointer(adjinfo, unsafe.Pointer(&d.fd)) |
| } |
| |
| // Adjust defer argument blocks the same way we adjust active stack frames. |
| // Note: this code is after the loop above, so that if a defer record is |
| // stack allocated, we work on the copy in the new stack. |
| tracebackdefers(gp, adjustframe, noescape(unsafe.Pointer(adjinfo))) |
| } |
| |
| func adjustpanics(gp *g, adjinfo *adjustinfo) { |
| // Panics are on stack and already adjusted. |
| // Update pointer to head of list in G. |
| adjustpointer(adjinfo, unsafe.Pointer(&gp._panic)) |
| } |
| |
| func adjustsudogs(gp *g, adjinfo *adjustinfo) { |
| // the data elements pointed to by a SudoG structure |
| // might be in the stack. |
| for s := gp.waiting; s != nil; s = s.waitlink { |
| adjustpointer(adjinfo, unsafe.Pointer(&s.elem)) |
| } |
| } |
| |
| func fillstack(stk stack, b byte) { |
| for p := stk.lo; p < stk.hi; p++ { |
| *(*byte)(unsafe.Pointer(p)) = b |
| } |
| } |
| |
| func findsghi(gp *g, stk stack) uintptr { |
| var sghi uintptr |
| for sg := gp.waiting; sg != nil; sg = sg.waitlink { |
| p := uintptr(sg.elem) + uintptr(sg.c.elemsize) |
| if stk.lo <= p && p < stk.hi && p > sghi { |
| sghi = p |
| } |
| } |
| return sghi |
| } |
| |
| // syncadjustsudogs adjusts gp's sudogs and copies the part of gp's |
| // stack they refer to while synchronizing with concurrent channel |
| // operations. It returns the number of bytes of stack copied. |
| func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr { |
| if gp.waiting == nil { |
| return 0 |
| } |
| |
| // Lock channels to prevent concurrent send/receive. |
| var lastc *hchan |
| for sg := gp.waiting; sg != nil; sg = sg.waitlink { |
| if sg.c != lastc { |
| // There is a ranking cycle here between gscan bit and |
| // hchan locks. Normally, we only allow acquiring hchan |
| // locks and then getting a gscan bit. In this case, we |
| // already have the gscan bit. We allow acquiring hchan |
| // locks here as a special case, since a deadlock can't |
| // happen because the G involved must already be |
| // suspended. So, we get a special hchan lock rank here |
| // that is lower than gscan, but doesn't allow acquiring |
| // any other locks other than hchan. |
| lockWithRank(&sg.c.lock, lockRankHchanLeaf) |
| } |
| lastc = sg.c |
| } |
| |
| // Adjust sudogs. |
| adjustsudogs(gp, adjinfo) |
| |
| // Copy the part of the stack the sudogs point in to |
| // while holding the lock to prevent races on |
| // send/receive slots. |
| var sgsize uintptr |
| if adjinfo.sghi != 0 { |
| oldBot := adjinfo.old.hi - used |
| newBot := oldBot + adjinfo.delta |
| sgsize = adjinfo.sghi - oldBot |
| memmove(unsafe.Pointer(newBot), unsafe.Pointer(oldBot), sgsize) |
| } |
| |
| // Unlock channels. |
| lastc = nil |
| for sg := gp.waiting; sg != nil; sg = sg.waitlink { |
| if sg.c != lastc { |
| unlock(&sg.c.lock) |
| } |
| lastc = sg.c |
| } |
| |
| return sgsize |
| } |
| |
| // Copies gp's stack to a new stack of a different size. |
| // Caller must have changed gp status to Gcopystack. |
| func copystack(gp *g, newsize uintptr) { |
| if gp.syscallsp != 0 { |
| throw("stack growth not allowed in system call") |
| } |
| old := gp.stack |
| if old.lo == 0 { |
| throw("nil stackbase") |
| } |
| used := old.hi - gp.sched.sp |
| |
| // allocate new stack |
| new := stackalloc(uint32(newsize)) |
| if stackPoisonCopy != 0 { |
| fillstack(new, 0xfd) |
| } |
| if stackDebug >= 1 { |
| print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]", " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n") |
| } |
| |
| // Compute adjustment. |
| var adjinfo adjustinfo |
| adjinfo.old = old |
| adjinfo.delta = new.hi - old.hi |
| |
| // Adjust sudogs, synchronizing with channel ops if necessary. |
| ncopy := used |
| if !gp.activeStackChans { |
| if newsize < old.hi-old.lo && atomic.Load8(&gp.parkingOnChan) != 0 { |
| // It's not safe for someone to shrink this stack while we're actively |
| // parking on a channel, but it is safe to grow since we do that |
| // ourselves and explicitly don't want to synchronize with channels |
| // since we could self-deadlock. |
| throw("racy sudog adjustment due to parking on channel") |
| } |
| adjustsudogs(gp, &adjinfo) |
| } else { |
| // sudogs may be pointing in to the stack and gp has |
| // released channel locks, so other goroutines could |
| // be writing to gp's stack. Find the highest such |
| // pointer so we can handle everything there and below |
| // carefully. (This shouldn't be far from the bottom |
| // of the stack, so there's little cost in handling |
| // everything below it carefully.) |
| adjinfo.sghi = findsghi(gp, old) |
| |
| // Synchronize with channel ops and copy the part of |
| // the stack they may interact with. |
| ncopy -= syncadjustsudogs(gp, used, &adjinfo) |
| } |
| |
| // Copy the stack (or the rest of it) to the new location |
| memmove(unsafe.Pointer(new.hi-ncopy), unsafe.Pointer(old.hi-ncopy), ncopy) |
| |
| // Adjust remaining structures that have pointers into stacks. |
| // We have to do most of these before we traceback the new |
| // stack because gentraceback uses them. |
| adjustctxt(gp, &adjinfo) |
| adjustdefers(gp, &adjinfo) |
| adjustpanics(gp, &adjinfo) |
| if adjinfo.sghi != 0 { |
| adjinfo.sghi += adjinfo.delta |
| } |
| |
| // Swap out old stack for new one |
| gp.stack = new |
| gp.stackguard0 = new.lo + _StackGuard // NOTE: might clobber a preempt request |
| gp.sched.sp = new.hi - used |
| gp.stktopsp += adjinfo.delta |
| |
| // Adjust pointers in the new stack. |
| gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, adjustframe, noescape(unsafe.Pointer(&adjinfo)), 0) |
| |
| // free old stack |
| if stackPoisonCopy != 0 { |
| fillstack(old, 0xfc) |
| } |
| stackfree(old) |
| } |
| |
| // round x up to a power of 2. |
| func round2(x int32) int32 { |
| s := uint(0) |
| for 1<<s < x { |
| s++ |
| } |
| return 1 << s |
| } |
| |
| // Called from runtime·morestack when more stack is needed. |
| // Allocate larger stack and relocate to new stack. |
| // Stack growth is multiplicative, for constant amortized cost. |
| // |
| // g->atomicstatus will be Grunning or Gscanrunning upon entry. |
| // If the scheduler is trying to stop this g, then it will set preemptStop. |
| // |
| // This must be nowritebarrierrec because it can be called as part of |
| // stack growth from other nowritebarrierrec functions, but the |
| // compiler doesn't check this. |
| // |
| //go:nowritebarrierrec |
| func newstack() { |
| thisg := getg() |
| // TODO: double check all gp. shouldn't be getg(). |
| if thisg.m.morebuf.g.ptr().stackguard0 == stackFork { |
| throw("stack growth after fork") |
| } |
| if thisg.m.morebuf.g.ptr() != thisg.m.curg { |
| print("runtime: newstack called from g=", hex(thisg.m.morebuf.g), "\n"+"\tm=", thisg.m, " m->curg=", thisg.m.curg, " m->g0=", thisg.m.g0, " m->gsignal=", thisg.m.gsignal, "\n") |
| morebuf := thisg.m.morebuf |
| traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g.ptr()) |
| throw("runtime: wrong goroutine in newstack") |
| } |
| |
| gp := thisg.m.curg |
| |
| if thisg.m.curg.throwsplit { |
| // Update syscallsp, syscallpc in case traceback uses them. |
| morebuf := thisg.m.morebuf |
| gp.syscallsp = morebuf.sp |
| gp.syscallpc = morebuf.pc |
| pcname, pcoff := "(unknown)", uintptr(0) |
| f := findfunc(gp.sched.pc) |
| if f.valid() { |
| pcname = funcname(f) |
| pcoff = gp.sched.pc - f.entry |
| } |
| print("runtime: newstack at ", pcname, "+", hex(pcoff), |
| " sp=", hex(gp.sched.sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n", |
| "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n", |
| "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n") |
| |
| thisg.m.traceback = 2 // Include runtime frames |
| traceback(morebuf.pc, morebuf.sp, morebuf.lr, gp) |
| throw("runtime: stack split at bad time") |
| } |
| |
| morebuf := thisg.m.morebuf |
| thisg.m.morebuf.pc = 0 |
| thisg.m.morebuf.lr = 0 |
| thisg.m.morebuf.sp = 0 |
| thisg.m.morebuf.g = 0 |
| |
| // NOTE: stackguard0 may change underfoot, if another thread |
| // is about to try to preempt gp. Read it just once and use that same |
| // value now and below. |
| preempt := atomic.Loaduintptr(&gp.stackguard0) == stackPreempt |
| |
| // Be conservative about where we preempt. |
| // We are interested in preempting user Go code, not runtime code. |
| // If we're holding locks, mallocing, or preemption is disabled, don't |
| // preempt. |
| // This check is very early in newstack so that even the status change |
| // from Grunning to Gwaiting and back doesn't happen in this case. |
| // That status change by itself can be viewed as a small preemption, |
| // because the GC might change Gwaiting to Gscanwaiting, and then |
| // this goroutine has to wait for the GC to finish before continuing. |
| // If the GC is in some way dependent on this goroutine (for example, |
| // it needs a lock held by the goroutine), that small preemption turns |
| // into a real deadlock. |
| if preempt { |
| if !canPreemptM(thisg.m) { |
| // Let the goroutine keep running for now. |
| // gp->preempt is set, so it will be preempted next time. |
| gp.stackguard0 = gp.stack.lo + _StackGuard |
| gogo(&gp.sched) // never return |
| } |
| } |
| |
| if gp.stack.lo == 0 { |
| throw("missing stack in newstack") |
| } |
| sp := gp.sched.sp |
| if sys.ArchFamily == sys.AMD64 || sys.ArchFamily == sys.I386 || sys.ArchFamily == sys.WASM { |
| // The call to morestack cost a word. |
| sp -= sys.PtrSize |
| } |
| if stackDebug >= 1 || sp < gp.stack.lo { |
| print("runtime: newstack sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n", |
| "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n", |
| "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n") |
| } |
| if sp < gp.stack.lo { |
| print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->status=", hex(readgstatus(gp)), "\n ") |
| print("runtime: split stack overflow: ", hex(sp), " < ", hex(gp.stack.lo), "\n") |
| throw("runtime: split stack overflow") |
| } |
| |
| if preempt { |
| if gp == thisg.m.g0 { |
| throw("runtime: preempt g0") |
| } |
| if thisg.m.p == 0 && thisg.m.locks == 0 { |
| throw("runtime: g is running but p is not") |
| } |
| |
| if gp.preemptShrink { |
| // We're at a synchronous safe point now, so |
| // do the pending stack shrink. |
| gp.preemptShrink = false |
| shrinkstack(gp) |
| } |
| |
| if gp.preemptStop { |
| preemptPark(gp) // never returns |
| } |
| |
| // Act like goroutine called runtime.Gosched. |
| gopreempt_m(gp) // never return |
| } |
| |
| // Allocate a bigger segment and move the stack. |
| oldsize := gp.stack.hi - gp.stack.lo |
| newsize := oldsize * 2 |
| |
| // Make sure we grow at least as much as needed to fit the new frame. |
| // (This is just an optimization - the caller of morestack will |
| // recheck the bounds on return.) |
| if f := findfunc(gp.sched.pc); f.valid() { |
| max := uintptr(funcMaxSPDelta(f)) |
| for newsize-oldsize < max+_StackGuard { |
| newsize *= 2 |
| } |
| } |
| |
| if newsize > maxstacksize || newsize > maxstackceiling { |
| if maxstacksize < maxstackceiling { |
| print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n") |
| } else { |
| print("runtime: goroutine stack exceeds ", maxstackceiling, "-byte limit\n") |
| } |
| print("runtime: sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n") |
| throw("stack overflow") |
| } |
| |
| // The goroutine must be executing in order to call newstack, |
| // so it must be Grunning (or Gscanrunning). |
| casgstatus(gp, _Grunning, _Gcopystack) |
| |
| // The concurrent GC will not scan the stack while we are doing the copy since |
| // the gp is in a Gcopystack status. |
| copystack(gp, newsize) |
| if stackDebug >= 1 { |
| print("stack grow done\n") |
| } |
| casgstatus(gp, _Gcopystack, _Grunning) |
| gogo(&gp.sched) |
| } |
| |
| //go:nosplit |
| func nilfunc() { |
| *(*uint8)(nil) = 0 |
| } |
| |
| // adjust Gobuf as if it executed a call to fn |
| // and then stopped before the first instruction in fn. |
| func gostartcallfn(gobuf *gobuf, fv *funcval) { |
| var fn unsafe.Pointer |
| if fv != nil { |
| fn = unsafe.Pointer(fv.fn) |
| } else { |
| fn = unsafe.Pointer(funcPC(nilfunc)) |
| } |
| gostartcall(gobuf, fn, unsafe.Pointer(fv)) |
| } |
| |
| // isShrinkStackSafe returns whether it's safe to attempt to shrink |
| // gp's stack. Shrinking the stack is only safe when we have precise |
| // pointer maps for all frames on the stack. |
| func isShrinkStackSafe(gp *g) bool { |
| // We can't copy the stack if we're in a syscall. |
| // The syscall might have pointers into the stack and |
| // often we don't have precise pointer maps for the innermost |
| // frames. |
| // |
| // We also can't copy the stack if we're at an asynchronous |
| // safe-point because we don't have precise pointer maps for |
| // all frames. |
| // |
| // We also can't *shrink* the stack in the window between the |
| // goroutine calling gopark to park on a channel and |
| // gp.activeStackChans being set. |
| return gp.syscallsp == 0 && !gp.asyncSafePoint && atomic.Load8(&gp.parkingOnChan) == 0 |
| } |
| |
| // Maybe shrink the stack being used by gp. |
| // |
| // gp must be stopped and we must own its stack. It may be in |
| // _Grunning, but only if this is our own user G. |
| func shrinkstack(gp *g) { |
| if gp.stack.lo == 0 { |
| throw("missing stack in shrinkstack") |
| } |
| if s := readgstatus(gp); s&_Gscan == 0 { |
| // We don't own the stack via _Gscan. We could still |
| // own it if this is our own user G and we're on the |
| // system stack. |
| if !(gp == getg().m.curg && getg() != getg().m.curg && s == _Grunning) { |
| // We don't own the stack. |
| throw("bad status in shrinkstack") |
| } |
| } |
| if !isShrinkStackSafe(gp) { |
| throw("shrinkstack at bad time") |
| } |
| // Check for self-shrinks while in a libcall. These may have |
| // pointers into the stack disguised as uintptrs, but these |
| // code paths should all be nosplit. |
| if gp == getg().m.curg && gp.m.libcallsp != 0 { |
| throw("shrinking stack in libcall") |
| } |
| |
| if debug.gcshrinkstackoff > 0 { |
| return |
| } |
| f := findfunc(gp.startpc) |
| if f.valid() && f.funcID == funcID_gcBgMarkWorker { |
| // We're not allowed to shrink the gcBgMarkWorker |
| // stack (see gcBgMarkWorker for explanation). |
| return |
| } |
| |
| oldsize := gp.stack.hi - gp.stack.lo |
| newsize := oldsize / 2 |
| // Don't shrink the allocation below the minimum-sized stack |
| // allocation. |
| if newsize < _FixedStack { |
| return |
| } |
| // Compute how much of the stack is currently in use and only |
| // shrink the stack if gp is using less than a quarter of its |
| // current stack. The currently used stack includes everything |
| // down to the SP plus the stack guard space that ensures |
| // there's room for nosplit functions. |
| avail := gp.stack.hi - gp.stack.lo |
| if used := gp.stack.hi - gp.sched.sp + _StackLimit; used >= avail/4 { |
| return |
| } |
| |
| if stackDebug > 0 { |
| print("shrinking stack ", oldsize, "->", newsize, "\n") |
| } |
| |
| copystack(gp, newsize) |
| } |
| |
| // freeStackSpans frees unused stack spans at the end of GC. |
| func freeStackSpans() { |
| |
| // Scan stack pools for empty stack spans. |
| for order := range stackpool { |
| lock(&stackpool[order].item.mu) |
| list := &stackpool[order].item.span |
| for s := list.first; s != nil; { |
| next := s.next |
| if s.allocCount == 0 { |
| list.remove(s) |
| s.manualFreeList = 0 |
| osStackFree(s) |
| mheap_.freeManual(s, spanAllocStack) |
| } |
| s = next |
| } |
| unlock(&stackpool[order].item.mu) |
| } |
| |
| // Free large stack spans. |
| lock(&stackLarge.lock) |
| for i := range stackLarge.free { |
| for s := stackLarge.free[i].first; s != nil; { |
| next := s.next |
| stackLarge.free[i].remove(s) |
| osStackFree(s) |
| mheap_.freeManual(s, spanAllocStack) |
| s = next |
| } |
| } |
| unlock(&stackLarge.lock) |
| } |
| |
| // getStackMap returns the locals and arguments live pointer maps, and |
| // stack object list for frame. |
| func getStackMap(frame *stkframe, cache *pcvalueCache, debug bool) (locals, args bitvector, objs []stackObjectRecord) { |
| targetpc := frame.continpc |
| if targetpc == 0 { |
| // Frame is dead. Return empty bitvectors. |
| return |
| } |
| |
| f := frame.fn |
| pcdata := int32(-1) |
| if targetpc != f.entry { |
| // Back up to the CALL. If we're at the function entry |
| // point, we want to use the entry map (-1), even if |
| // the first instruction of the function changes the |
| // stack map. |
| targetpc-- |
| pcdata = pcdatavalue(f, _PCDATA_StackMapIndex, targetpc, cache) |
| } |
| if pcdata == -1 { |
| // We do not have a valid pcdata value but there might be a |
| // stackmap for this function. It is likely that we are looking |
| // at the function prologue, assume so and hope for the best. |
| pcdata = 0 |
| } |
| |
| // Local variables. |
| size := frame.varp - frame.sp |
| var minsize uintptr |
| switch sys.ArchFamily { |
| case sys.ARM64: |
| minsize = sys.StackAlign |
| default: |
| minsize = sys.MinFrameSize |
| } |
| if size > minsize { |
| stackid := pcdata |
| stkmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps)) |
| if stkmap == nil || stkmap.n <= 0 { |
| print("runtime: frame ", funcname(f), " untyped locals ", hex(frame.varp-size), "+", hex(size), "\n") |
| throw("missing stackmap") |
| } |
| // If nbit == 0, there's no work to do. |
| if stkmap.nbit > 0 { |
| if stackid < 0 || stackid >= stkmap.n { |
| // don't know where we are |
| print("runtime: pcdata is ", stackid, " and ", stkmap.n, " locals stack map entries for ", funcname(f), " (targetpc=", hex(targetpc), ")\n") |
| throw("bad symbol table") |
| } |
| locals = stackmapdata(stkmap, stackid) |
| if stackDebug >= 3 && debug { |
| print(" locals ", stackid, "/", stkmap.n, " ", locals.n, " words ", locals.bytedata, "\n") |
| } |
| } else if stackDebug >= 3 && debug { |
| print(" no locals to adjust\n") |
| } |
| } |
| |
| // Arguments. |
| if frame.arglen > 0 { |
| if frame.argmap != nil { |
| // argmap is set when the function is reflect.makeFuncStub or reflect.methodValueCall. |
| // In this case, arglen specifies how much of the args section is actually live. |
| // (It could be either all the args + results, or just the args.) |
| args = *frame.argmap |
| n := int32(frame.arglen / sys.PtrSize) |
| if n < args.n { |
| args.n = n // Don't use more of the arguments than arglen. |
| } |
| } else { |
| stackmap := (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps)) |
| if stackmap == nil || stackmap.n <= 0 { |
| print("runtime: frame ", funcname(f), " untyped args ", hex(frame.argp), "+", hex(frame.arglen), "\n") |
| throw("missing stackmap") |
| } |
| if pcdata < 0 || pcdata >= stackmap.n { |
| // don't know where we are |
| print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " args stack map entries for ", funcname(f), " (targetpc=", hex(targetpc), ")\n") |
| throw("bad symbol table") |
| } |
| if stackmap.nbit > 0 { |
| args = stackmapdata(stackmap, pcdata) |
| } |
| } |
| } |
| |
| // stack objects. |
| p := funcdata(f, _FUNCDATA_StackObjects) |
| if p != nil { |
| n := *(*uintptr)(p) |
| p = add(p, sys.PtrSize) |
| *(*slice)(unsafe.Pointer(&objs)) = slice{array: noescape(p), len: int(n), cap: int(n)} |
| // Note: the noescape above is needed to keep |
| // getStackMap from "leaking param content: |
| // frame". That leak propagates up to getgcmask, then |
| // GCMask, then verifyGCInfo, which converts the stack |
| // gcinfo tests into heap gcinfo tests :( |
| } |
| |
| return |
| } |
| |
| // A stackObjectRecord is generated by the compiler for each stack object in a stack frame. |
| // This record must match the generator code in cmd/compile/internal/gc/ssa.go:emitStackObjects. |
| type stackObjectRecord struct { |
| // offset in frame |
| // if negative, offset from varp |
| // if non-negative, offset from argp |
| off int |
| typ *_type |
| } |
| |
| // This is exported as ABI0 via linkname so obj can call it. |
| // |
| //go:nosplit |
| //go:linkname morestackc |
| func morestackc() { |
| throw("attempt to execute system stack code on user stack") |
| } |