| // Copyright 2013 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| #include "runtime.h" |
| #include "arch_GOARCH.h" |
| #include "malloc.h" |
| #include "stack.h" |
| #include "funcdata.h" |
| #include "typekind.h" |
| #include "type.h" |
| |
| enum |
| { |
| // StackDebug == 0: no logging |
| // == 1: logging of per-stack operations |
| // == 2: logging of per-frame operations |
| // == 3: logging of per-word updates |
| // == 4: logging of per-word reads |
| StackDebug = 0, |
| StackFromSystem = 0, // allocate stacks from system memory instead of the heap |
| StackFaultOnFree = 0, // old stacks are mapped noaccess to detect use after free |
| }; |
| |
| typedef struct StackCacheNode StackCacheNode; |
| struct StackCacheNode |
| { |
| StackCacheNode *next; |
| void* batch[StackCacheBatch-1]; |
| }; |
| |
| static StackCacheNode *stackcache; |
| static Lock stackcachemu; |
| |
| // stackcacherefill/stackcacherelease implement a global cache of stack segments. |
| // The cache is required to prevent unlimited growth of per-thread caches. |
| static void |
| stackcacherefill(void) |
| { |
| StackCacheNode *n; |
| int32 i, pos; |
| |
| runtime·lock(&stackcachemu); |
| n = stackcache; |
| if(n) |
| stackcache = n->next; |
| runtime·unlock(&stackcachemu); |
| if(n == nil) { |
| n = (StackCacheNode*)runtime·SysAlloc(FixedStack*StackCacheBatch, &mstats.stacks_sys); |
| if(n == nil) |
| runtime·throw("out of memory (stackcacherefill)"); |
| for(i = 0; i < StackCacheBatch-1; i++) |
| n->batch[i] = (byte*)n + (i+1)*FixedStack; |
| } |
| pos = m->stackcachepos; |
| for(i = 0; i < StackCacheBatch-1; i++) { |
| m->stackcache[pos] = n->batch[i]; |
| pos = (pos + 1) % StackCacheSize; |
| } |
| m->stackcache[pos] = n; |
| pos = (pos + 1) % StackCacheSize; |
| m->stackcachepos = pos; |
| m->stackcachecnt += StackCacheBatch; |
| } |
| |
| static void |
| stackcacherelease(void) |
| { |
| StackCacheNode *n; |
| uint32 i, pos; |
| |
| pos = (m->stackcachepos - m->stackcachecnt) % StackCacheSize; |
| n = (StackCacheNode*)m->stackcache[pos]; |
| pos = (pos + 1) % StackCacheSize; |
| for(i = 0; i < StackCacheBatch-1; i++) { |
| n->batch[i] = m->stackcache[pos]; |
| pos = (pos + 1) % StackCacheSize; |
| } |
| m->stackcachecnt -= StackCacheBatch; |
| runtime·lock(&stackcachemu); |
| n->next = stackcache; |
| stackcache = n; |
| runtime·unlock(&stackcachemu); |
| } |
| |
| void* |
| runtime·stackalloc(G *gp, uint32 n) |
| { |
| uint32 pos; |
| void *v; |
| bool malloced; |
| Stktop *top; |
| |
| // Stackalloc must be called on scheduler stack, so that we |
| // never try to grow the stack during the code that stackalloc runs. |
| // Doing so would cause a deadlock (issue 1547). |
| if(g != m->g0) |
| runtime·throw("stackalloc not on scheduler stack"); |
| if((n & (n-1)) != 0) |
| runtime·throw("stack size not a power of 2"); |
| if(StackDebug >= 1) |
| runtime·printf("stackalloc %d\n", n); |
| |
| gp->stacksize += n; |
| if(runtime·debug.efence || StackFromSystem) { |
| v = runtime·SysAlloc(ROUND(n, PageSize), &mstats.stacks_sys); |
| if(v == nil) |
| runtime·throw("out of memory (stackalloc)"); |
| return v; |
| } |
| |
| // Minimum-sized stacks are allocated with a fixed-size free-list allocator, |
| // but if we need a stack of a bigger size, we fall back on malloc |
| // (assuming that inside malloc all the stack frames are small, |
| // so that we do not deadlock). |
| malloced = true; |
| if(n == FixedStack || m->mallocing) { |
| if(n != FixedStack) { |
| runtime·printf("stackalloc: in malloc, size=%d want %d\n", FixedStack, n); |
| runtime·throw("stackalloc"); |
| } |
| if(m->stackcachecnt == 0) |
| stackcacherefill(); |
| pos = m->stackcachepos; |
| pos = (pos - 1) % StackCacheSize; |
| v = m->stackcache[pos]; |
| m->stackcachepos = pos; |
| m->stackcachecnt--; |
| m->stackinuse++; |
| malloced = false; |
| } else |
| v = runtime·mallocgc(n, 0, FlagNoProfiling|FlagNoGC|FlagNoZero|FlagNoInvokeGC); |
| |
| top = (Stktop*)((byte*)v+n-sizeof(Stktop)); |
| runtime·memclr((byte*)top, sizeof(*top)); |
| top->malloced = malloced; |
| return v; |
| } |
| |
| void |
| runtime·stackfree(G *gp, void *v, Stktop *top) |
| { |
| uint32 pos; |
| uintptr n; |
| |
| n = (uintptr)(top+1) - (uintptr)v; |
| if(StackDebug >= 1) |
| runtime·printf("stackfree %p %d\n", v, (int32)n); |
| gp->stacksize -= n; |
| if(runtime·debug.efence || StackFromSystem) { |
| if(runtime·debug.efence || StackFaultOnFree) |
| runtime·SysFault(v, n); |
| else |
| runtime·SysFree(v, n, &mstats.stacks_sys); |
| return; |
| } |
| if(top->malloced) { |
| runtime·free(v); |
| return; |
| } |
| if(n != FixedStack) |
| runtime·throw("stackfree: bad fixed size"); |
| if(m->stackcachecnt == StackCacheSize) |
| stackcacherelease(); |
| pos = m->stackcachepos; |
| m->stackcache[pos] = v; |
| m->stackcachepos = (pos + 1) % StackCacheSize; |
| m->stackcachecnt++; |
| m->stackinuse--; |
| } |
| |
| // Called from runtime·lessstack when returning from a function which |
| // allocated a new stack segment. The function's return value is in |
| // m->cret. |
| void |
| runtime·oldstack(void) |
| { |
| Stktop *top; |
| uint32 argsize; |
| byte *sp, *old; |
| uintptr *src, *dst, *dstend; |
| G *gp; |
| int64 goid; |
| int32 oldstatus; |
| |
| gp = m->curg; |
| top = (Stktop*)gp->stackbase; |
| old = (byte*)gp->stackguard - StackGuard; |
| sp = (byte*)top; |
| argsize = top->argsize; |
| |
| if(StackDebug >= 1) { |
| runtime·printf("runtime: oldstack gobuf={pc:%p sp:%p lr:%p} cret=%p argsize=%p\n", |
| top->gobuf.pc, top->gobuf.sp, top->gobuf.lr, (uintptr)m->cret, (uintptr)argsize); |
| } |
| |
| // gp->status is usually Grunning, but it could be Gsyscall if a stack overflow |
| // happens during a function call inside entersyscall. |
| oldstatus = gp->status; |
| |
| gp->sched = top->gobuf; |
| gp->sched.ret = m->cret; |
| m->cret = 0; // drop reference |
| gp->status = Gwaiting; |
| gp->waitreason = "stack unsplit"; |
| |
| if(argsize > 0) { |
| sp -= argsize; |
| dst = (uintptr*)top->argp; |
| dstend = dst + argsize/sizeof(*dst); |
| src = (uintptr*)sp; |
| while(dst < dstend) |
| *dst++ = *src++; |
| } |
| goid = top->gobuf.g->goid; // fault if g is bad, before gogo |
| USED(goid); |
| |
| gp->stackbase = top->stackbase; |
| gp->stackguard = top->stackguard; |
| gp->stackguard0 = gp->stackguard; |
| gp->panicwrap = top->panicwrap; |
| runtime·stackfree(gp, old, top); |
| |
| gp->status = oldstatus; |
| runtime·gogo(&gp->sched); |
| } |
| |
| uintptr runtime·maxstacksize = 1<<20; // enough until runtime.main sets it for real |
| |
| static uint8* |
| mapnames[] = { |
| (uint8*)"---", |
| (uint8*)"scalar", |
| (uint8*)"ptr", |
| (uint8*)"multi", |
| }; |
| |
| // Stack frame layout |
| // |
| // (x86) |
| // +------------------+ |
| // | args from caller | |
| // +------------------+ <- frame->argp |
| // | return address | |
| // +------------------+ <- frame->varp |
| // | locals | |
| // +------------------+ |
| // | args to callee | |
| // +------------------+ <- frame->sp |
| // |
| // (arm: TODO) |
| |
| typedef struct CopyableInfo CopyableInfo; |
| struct CopyableInfo { |
| byte *stk; // bottom address of segment |
| byte *base; // top address of segment (including Stktop) |
| int32 frames; // count of copyable frames (-1 = not copyable) |
| }; |
| |
| void runtime·main(void); |
| |
| static bool |
| checkframecopy(Stkframe *frame, void *arg) |
| { |
| CopyableInfo *cinfo; |
| Func *f; |
| StackMap *stackmap; |
| |
| cinfo = arg; |
| f = frame->fn; |
| if(StackDebug >= 2) |
| runtime·printf(" checking %s frame=[%p,%p] stk=[%p,%p]\n", runtime·funcname(f), frame->sp, frame->fp, cinfo->stk, cinfo->base); |
| // if we're not in the segment any more, return immediately. |
| if(frame->varp < cinfo->stk || frame->varp >= cinfo->base) { |
| if(StackDebug >= 2) |
| runtime·printf(" <next segment>\n"); |
| return false; // stop traceback |
| } |
| if(f->entry == (uintptr)runtime·main) { |
| // A special routine at the TOS of the main routine. |
| // We will allow it to be copied even though we don't |
| // have full GC info for it (because it is written in C). |
| cinfo->frames++; |
| return false; // stop traceback |
| } |
| if(frame->varp != (byte*)frame->sp) { // not in prologue (and has at least one local or outarg) |
| stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); |
| if(stackmap == nil) { |
| cinfo->frames = -1; |
| if(StackDebug >= 1) |
| runtime·printf("copystack: no locals info for %s\n", runtime·funcname(f)); |
| return false; |
| } |
| if(stackmap->n <= 0) { |
| cinfo->frames = -1; |
| if(StackDebug >= 1) |
| runtime·printf("copystack: locals size info only for %s\n", runtime·funcname(f)); |
| return false; |
| } |
| } |
| if(frame->arglen != 0) { |
| stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); |
| if(stackmap == nil) { |
| cinfo->frames = -1; |
| if(StackDebug >= 1) |
| runtime·printf("copystack: no arg info for %s\n", runtime·funcname(f)); |
| return false; |
| } |
| } |
| cinfo->frames++; |
| return true; // this frame is ok; keep going |
| } |
| |
| // If the top segment of the stack contains an uncopyable |
| // frame, return -1. Otherwise return the number of frames |
| // in the top segment, all of which are copyable. |
| static int32 |
| copyabletopsegment(G *gp) |
| { |
| CopyableInfo cinfo; |
| Defer *d; |
| Func *f; |
| FuncVal *fn; |
| StackMap *stackmap; |
| |
| cinfo.stk = (byte*)gp->stackguard - StackGuard; |
| cinfo.base = (byte*)gp->stackbase + sizeof(Stktop); |
| cinfo.frames = 0; |
| |
| // Check that each frame is copyable. As a side effect, |
| // count the frames. |
| runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, checkframecopy, &cinfo, false); |
| if(StackDebug >= 1 && cinfo.frames != -1) |
| runtime·printf("copystack: %d copyable frames\n", cinfo.frames); |
| |
| // Check to make sure all Defers are copyable |
| for(d = gp->defer; d != nil; d = d->link) { |
| if(cinfo.stk <= (byte*)d && (byte*)d < cinfo.base) { |
| // Defer is on the stack. Its copyableness has |
| // been established during stack walking. |
| // For now, this only happens with the Defer in runtime.main. |
| continue; |
| } |
| if(d->argp < cinfo.stk || cinfo.base <= d->argp) |
| break; // a defer for the next segment |
| fn = d->fn; |
| f = runtime·findfunc((uintptr)fn->fn); |
| if(f == nil) |
| return -1; |
| |
| // Check to make sure we have an args pointer map for the defer's args. |
| // We only need the args map, but we check |
| // for the locals map also, because when the locals map |
| // isn't provided it means the ptr map came from C and |
| // C (particularly, cgo) lies to us. See issue 7695. |
| stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); |
| if(stackmap == nil || stackmap->n <= 0) |
| return -1; |
| stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); |
| if(stackmap == nil || stackmap->n <= 0) |
| return -1; |
| |
| if(cinfo.stk <= (byte*)fn && (byte*)fn < cinfo.base) { |
| // FuncVal is on the stack. Again, its copyableness |
| // was established during stack walking. |
| continue; |
| } |
| // The FuncVal may have pointers in it, but fortunately for us |
| // the compiler won't put pointers into the stack in a |
| // heap-allocated FuncVal. |
| // One day if we do need to check this, we'll need maps of the |
| // pointerness of the closure args. The only place we have that map |
| // right now is in the gc program for the FuncVal. Ugh. |
| } |
| |
| return cinfo.frames; |
| } |
| |
| typedef struct AdjustInfo AdjustInfo; |
| struct AdjustInfo { |
| byte *oldstk; // bottom address of segment |
| byte *oldbase; // top address of segment (after Stktop) |
| uintptr delta; // ptr distance from old to new stack (newbase - oldbase) |
| }; |
| |
| // bv describes the memory starting at address scanp. |
| // Adjust any pointers contained therein. |
| static void |
| adjustpointers(byte **scanp, BitVector *bv, AdjustInfo *adjinfo, Func *f) |
| { |
| uintptr delta; |
| int32 num, i; |
| byte *p, *minp, *maxp; |
| Type *t; |
| Itab *tab; |
| |
| minp = adjinfo->oldstk; |
| maxp = adjinfo->oldbase; |
| delta = adjinfo->delta; |
| num = bv->n / BitsPerPointer; |
| for(i = 0; i < num; i++) { |
| if(StackDebug >= 4) |
| runtime·printf(" %p:%s:%p\n", &scanp[i], mapnames[bv->data[i / (32 / BitsPerPointer)] >> (i * BitsPerPointer & 31) & 3], scanp[i]); |
| switch(bv->data[i / (32 / BitsPerPointer)] >> (i * BitsPerPointer & 31) & 3) { |
| case BitsDead: |
| if(runtime·debug.gcdead) |
| scanp[i] = (byte*)PoisonStack; |
| break; |
| case BitsScalar: |
| break; |
| case BitsPointer: |
| p = scanp[i]; |
| if(f != nil && (byte*)0 < p && (p < (byte*)PageSize || (uintptr)p == PoisonGC || (uintptr)p == PoisonStack)) { |
| // Looks like a junk value in a pointer slot. |
| // Live analysis wrong? |
| m->traceback = 2; |
| runtime·printf("runtime: bad pointer in frame %s at %p: %p\n", runtime·funcname(f), &scanp[i], p); |
| runtime·throw("bad pointer!"); |
| } |
| if(minp <= p && p < maxp) { |
| if(StackDebug >= 3) |
| runtime·printf("adjust ptr %p %s\n", p, runtime·funcname(f)); |
| scanp[i] = p + delta; |
| } |
| break; |
| case BitsMultiWord: |
| switch(bv->data[(i+1) / (32 / BitsPerPointer)] >> ((i+1) * BitsPerPointer & 31) & 3) { |
| case BitsString: |
| // string referents are never on the stack, never need to be adjusted |
| i++; // skip len |
| break; |
| case BitsSlice: |
| p = scanp[i]; |
| if(minp <= p && p < maxp) { |
| if(StackDebug >= 3) |
| runtime·printf("adjust slice %p\n", p); |
| scanp[i] = p + delta; |
| } |
| i += 2; // skip len, cap |
| break; |
| case BitsEface: |
| t = (Type*)scanp[i]; |
| if(t != nil && (t->size > PtrSize || (t->kind & KindNoPointers) == 0)) { |
| p = scanp[i+1]; |
| if(minp <= p && p < maxp) { |
| if(StackDebug >= 3) |
| runtime·printf("adjust eface %p\n", p); |
| if(t->size > PtrSize) // currently we always allocate such objects on the heap |
| runtime·throw("large interface value found on stack"); |
| scanp[i+1] = p + delta; |
| } |
| } |
| i++; |
| break; |
| case BitsIface: |
| tab = (Itab*)scanp[i]; |
| if(tab != nil) { |
| t = tab->type; |
| //runtime·printf(" type=%p\n", t); |
| if(t->size > PtrSize || (t->kind & KindNoPointers) == 0) { |
| p = scanp[i+1]; |
| if(minp <= p && p < maxp) { |
| if(StackDebug >= 3) |
| runtime·printf("adjust iface %p\n", p); |
| if(t->size > PtrSize) // currently we always allocate such objects on the heap |
| runtime·throw("large interface value found on stack"); |
| scanp[i+1] = p + delta; |
| } |
| } |
| } |
| i++; |
| break; |
| } |
| break; |
| } |
| } |
| } |
| |
| // Note: the argument/return area is adjusted by the callee. |
| static bool |
| adjustframe(Stkframe *frame, void *arg) |
| { |
| AdjustInfo *adjinfo; |
| Func *f; |
| StackMap *stackmap; |
| int32 pcdata; |
| BitVector bv; |
| uintptr targetpc; |
| |
| adjinfo = arg; |
| f = frame->fn; |
| if(StackDebug >= 2) |
| runtime·printf(" adjusting %s frame=[%p,%p] pc=%p\n", runtime·funcname(f), frame->sp, frame->fp, frame->pc); |
| if(f->entry == (uintptr)runtime·main) |
| return true; |
| targetpc = frame->pc; |
| if(targetpc != f->entry) |
| targetpc--; |
| pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc); |
| if(pcdata == -1) |
| pcdata = 0; // in prologue |
| |
| // adjust local pointers |
| if(frame->varp != (byte*)frame->sp) { |
| stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); |
| if(stackmap == nil) |
| runtime·throw("no locals info"); |
| if(stackmap->n <= 0) |
| runtime·throw("locals size info only"); |
| bv = runtime·stackmapdata(stackmap, pcdata); |
| if(StackDebug >= 3) |
| runtime·printf(" locals\n"); |
| adjustpointers((byte**)frame->varp - bv.n / BitsPerPointer, &bv, adjinfo, f); |
| } |
| // adjust inargs and outargs |
| if(frame->arglen != 0) { |
| stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); |
| if(stackmap == nil) |
| runtime·throw("no arg info"); |
| bv = runtime·stackmapdata(stackmap, pcdata); |
| if(StackDebug >= 3) |
| runtime·printf(" args\n"); |
| adjustpointers((byte**)frame->argp, &bv, adjinfo, nil); |
| } |
| return true; |
| } |
| |
| static void |
| adjustctxt(G *gp, AdjustInfo *adjinfo) |
| { |
| if(adjinfo->oldstk <= (byte*)gp->sched.ctxt && (byte*)gp->sched.ctxt < adjinfo->oldbase) |
| gp->sched.ctxt = (byte*)gp->sched.ctxt + adjinfo->delta; |
| } |
| |
| static void |
| adjustdefers(G *gp, AdjustInfo *adjinfo) |
| { |
| Defer *d, **dp; |
| Func *f; |
| FuncVal *fn; |
| StackMap *stackmap; |
| BitVector bv; |
| |
| for(dp = &gp->defer, d = *dp; d != nil; dp = &d->link, d = *dp) { |
| if(adjinfo->oldstk <= (byte*)d && (byte*)d < adjinfo->oldbase) { |
| // The Defer record is on the stack. Its fields will |
| // get adjusted appropriately. |
| // This only happens for runtime.main now, but a compiler |
| // optimization could do more of this. |
| *dp = (Defer*)((byte*)d + adjinfo->delta); |
| continue; |
| } |
| if(d->argp < adjinfo->oldstk || adjinfo->oldbase <= d->argp) |
| break; // a defer for the next segment |
| f = runtime·findfunc((uintptr)d->fn->fn); |
| if(f == nil) |
| runtime·throw("can't adjust unknown defer"); |
| if(StackDebug >= 4) |
| runtime·printf(" checking defer %s\n", runtime·funcname(f)); |
| // Defer's FuncVal might be on the stack |
| fn = d->fn; |
| if(adjinfo->oldstk <= (byte*)fn && (byte*)fn < adjinfo->oldbase) { |
| if(StackDebug >= 3) |
| runtime·printf(" adjust defer fn %s\n", runtime·funcname(f)); |
| d->fn = (FuncVal*)((byte*)fn + adjinfo->delta); |
| } else { |
| // deferred function's args might point into the stack. |
| if(StackDebug >= 3) |
| runtime·printf(" adjust deferred args for %s\n", runtime·funcname(f)); |
| stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); |
| if(stackmap == nil) |
| runtime·throw("runtime: deferred function has no arg ptr map"); |
| bv = runtime·stackmapdata(stackmap, 0); |
| adjustpointers(d->args, &bv, adjinfo, f); |
| } |
| d->argp += adjinfo->delta; |
| } |
| } |
| |
| // Copies the top stack segment of gp to a new stack segment of a |
| // different size. The top segment must contain nframes frames. |
| static void |
| copystack(G *gp, uintptr nframes, uintptr newsize) |
| { |
| byte *oldstk, *oldbase, *newstk, *newbase; |
| uintptr oldsize, used; |
| AdjustInfo adjinfo; |
| Stktop *oldtop, *newtop; |
| bool malloced; |
| |
| if(gp->syscallstack != 0) |
| runtime·throw("can't handle stack copy in syscall yet"); |
| oldstk = (byte*)gp->stackguard - StackGuard; |
| oldbase = (byte*)gp->stackbase + sizeof(Stktop); |
| oldsize = oldbase - oldstk; |
| used = oldbase - (byte*)gp->sched.sp; |
| oldtop = (Stktop*)gp->stackbase; |
| |
| // allocate new stack |
| newstk = runtime·stackalloc(gp, newsize); |
| newbase = newstk + newsize; |
| newtop = (Stktop*)(newbase - sizeof(Stktop)); |
| malloced = newtop->malloced; |
| |
| if(StackDebug >= 1) |
| runtime·printf("copystack [%p %p]/%d -> [%p %p]/%d\n", oldstk, oldbase, (int32)oldsize, newstk, newbase, (int32)newsize); |
| USED(oldsize); |
| |
| // adjust pointers in the to-be-copied frames |
| adjinfo.oldstk = oldstk; |
| adjinfo.oldbase = oldbase; |
| adjinfo.delta = newbase - oldbase; |
| runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, nframes, adjustframe, &adjinfo, false); |
| |
| // adjust other miscellaneous things that have pointers into stacks. |
| adjustctxt(gp, &adjinfo); |
| adjustdefers(gp, &adjinfo); |
| |
| // copy the stack (including Stktop) to the new location |
| runtime·memmove(newbase - used, oldbase - used, used); |
| newtop->malloced = malloced; |
| |
| // Swap out old stack for new one |
| gp->stackbase = (uintptr)newtop; |
| gp->stackguard = (uintptr)newstk + StackGuard; |
| gp->stackguard0 = (uintptr)newstk + StackGuard; // NOTE: might clobber a preempt request |
| if(gp->stack0 == (uintptr)oldstk) |
| gp->stack0 = (uintptr)newstk; |
| gp->sched.sp = (uintptr)(newbase - used); |
| |
| // free old stack |
| runtime·stackfree(gp, oldstk, oldtop); |
| } |
| |
| // round x up to a power of 2. |
| int32 |
| runtime·round2(int32 x) |
| { |
| int32 s; |
| |
| s = 0; |
| while((1 << s) < x) |
| s++; |
| return 1 << s; |
| } |
| |
| // Called from runtime·newstackcall or from runtime·morestack when a new |
| // stack segment is needed. Allocate a new stack big enough for |
| // m->moreframesize bytes, copy m->moreargsize bytes to the new frame, |
| // and then act as though runtime·lessstack called the function at |
| // m->morepc. |
| void |
| runtime·newstack(void) |
| { |
| int32 framesize, argsize, oldstatus, oldsize, newsize, nframes; |
| Stktop *top, *oldtop; |
| byte *stk, *oldstk, *oldbase; |
| uintptr sp; |
| uintptr *src, *dst, *dstend; |
| G *gp; |
| Gobuf label, morebuf; |
| void *moreargp; |
| bool newstackcall; |
| |
| if(m->forkstackguard) |
| runtime·throw("split stack after fork"); |
| if(m->morebuf.g != m->curg) { |
| runtime·printf("runtime: newstack called from g=%p\n" |
| "\tm=%p m->curg=%p m->g0=%p m->gsignal=%p\n", |
| m->morebuf.g, m, m->curg, m->g0, m->gsignal); |
| runtime·throw("runtime: wrong goroutine in newstack"); |
| } |
| |
| // gp->status is usually Grunning, but it could be Gsyscall if a stack overflow |
| // happens during a function call inside entersyscall. |
| gp = m->curg; |
| oldstatus = gp->status; |
| |
| framesize = m->moreframesize; |
| argsize = m->moreargsize; |
| moreargp = m->moreargp; |
| m->moreargp = nil; |
| morebuf = m->morebuf; |
| m->morebuf.pc = (uintptr)nil; |
| m->morebuf.lr = (uintptr)nil; |
| m->morebuf.sp = (uintptr)nil; |
| gp->status = Gwaiting; |
| gp->waitreason = "stack growth"; |
| newstackcall = framesize==1; |
| if(newstackcall) |
| framesize = 0; |
| |
| // For newstackcall the context already points to beginning of runtime·newstackcall. |
| if(!newstackcall) |
| runtime·rewindmorestack(&gp->sched); |
| |
| sp = gp->sched.sp; |
| if(thechar == '6' || thechar == '8') { |
| // The call to morestack cost a word. |
| sp -= sizeof(uintptr); |
| } |
| if(StackDebug >= 1 || sp < gp->stackguard - StackGuard) { |
| runtime·printf("runtime: newstack framesize=%p argsize=%p sp=%p stack=[%p, %p]\n" |
| "\tmorebuf={pc:%p sp:%p lr:%p}\n" |
| "\tsched={pc:%p sp:%p lr:%p ctxt:%p}\n", |
| (uintptr)framesize, (uintptr)argsize, sp, gp->stackguard - StackGuard, gp->stackbase, |
| m->morebuf.pc, m->morebuf.sp, m->morebuf.lr, |
| gp->sched.pc, gp->sched.sp, gp->sched.lr, gp->sched.ctxt); |
| } |
| if(sp < gp->stackguard - StackGuard) { |
| runtime·printf("runtime: split stack overflow: %p < %p\n", sp, gp->stackguard - StackGuard); |
| runtime·throw("runtime: split stack overflow"); |
| } |
| |
| if(argsize % sizeof(uintptr) != 0) { |
| runtime·printf("runtime: stack growth with misaligned argsize %d\n", argsize); |
| runtime·throw("runtime: stack growth argsize"); |
| } |
| |
| if(gp->stackguard0 == (uintptr)StackPreempt) { |
| if(gp == m->g0) |
| runtime·throw("runtime: preempt g0"); |
| if(oldstatus == Grunning && m->p == nil && m->locks == 0) |
| runtime·throw("runtime: g is running but p is not"); |
| if(oldstatus == Gsyscall && m->locks == 0) |
| runtime·throw("runtime: stack growth during syscall"); |
| // Be conservative about where we preempt. |
| // We are interested in preempting user Go code, not runtime code. |
| if(oldstatus != Grunning || m->locks || m->mallocing || m->gcing || m->p->status != Prunning) { |
| // Let the goroutine keep running for now. |
| // gp->preempt is set, so it will be preempted next time. |
| gp->stackguard0 = gp->stackguard; |
| gp->status = oldstatus; |
| runtime·gogo(&gp->sched); // never return |
| } |
| // Act like goroutine called runtime.Gosched. |
| gp->status = oldstatus; |
| runtime·gosched0(gp); // never return |
| } |
| |
| // If every frame on the top segment is copyable, allocate a bigger segment |
| // and move the segment instead of allocating a new segment. |
| if(runtime·copystack) { |
| if(!runtime·precisestack) |
| runtime·throw("can't copy stacks without precise stacks"); |
| nframes = copyabletopsegment(gp); |
| if(nframes != -1) { |
| oldstk = (byte*)gp->stackguard - StackGuard; |
| oldbase = (byte*)gp->stackbase + sizeof(Stktop); |
| oldsize = oldbase - oldstk; |
| newsize = oldsize * 2; |
| copystack(gp, nframes, newsize); |
| if(StackDebug >= 1) |
| runtime·printf("stack grow done\n"); |
| if(gp->stacksize > runtime·maxstacksize) { |
| runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize); |
| runtime·throw("stack overflow"); |
| } |
| gp->status = oldstatus; |
| runtime·gogo(&gp->sched); |
| } |
| // TODO: if stack is uncopyable because we're in C code, patch return value at |
| // end of C code to trigger a copy as soon as C code exits. That way, we'll |
| // have stack available if we get this deep again. |
| } |
| |
| // allocate new segment. |
| framesize += argsize; |
| framesize += StackExtra; // room for more functions, Stktop. |
| if(framesize < StackMin) |
| framesize = StackMin; |
| framesize += StackSystem; |
| framesize = runtime·round2(framesize); |
| stk = runtime·stackalloc(gp, framesize); |
| if(gp->stacksize > runtime·maxstacksize) { |
| runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize); |
| runtime·throw("stack overflow"); |
| } |
| top = (Stktop*)(stk+framesize-sizeof(*top)); |
| |
| if(StackDebug >= 1) { |
| runtime·printf("\t-> new stack [%p, %p]\n", stk, top); |
| } |
| |
| top->stackbase = gp->stackbase; |
| top->stackguard = gp->stackguard; |
| top->gobuf = morebuf; |
| top->argp = moreargp; |
| top->argsize = argsize; |
| |
| // copy flag from panic |
| top->panic = gp->ispanic; |
| gp->ispanic = false; |
| |
| // if this isn't a panic, maybe we're splitting the stack for a panic. |
| // if we're splitting in the top frame, propagate the panic flag |
| // forward so that recover will know we're in a panic. |
| oldtop = (Stktop*)top->stackbase; |
| if(oldtop != nil && oldtop->panic && top->argp == (byte*)oldtop - oldtop->argsize - gp->panicwrap) |
| top->panic = true; |
| |
| top->panicwrap = gp->panicwrap; |
| gp->panicwrap = 0; |
| |
| gp->stackbase = (uintptr)top; |
| gp->stackguard = (uintptr)stk + StackGuard; |
| gp->stackguard0 = gp->stackguard; |
| |
| sp = (uintptr)top; |
| if(argsize > 0) { |
| sp -= argsize; |
| dst = (uintptr*)sp; |
| dstend = dst + argsize/sizeof(*dst); |
| src = (uintptr*)top->argp; |
| while(dst < dstend) |
| *dst++ = *src++; |
| } |
| if(thechar == '5') { |
| // caller would have saved its LR below args. |
| sp -= sizeof(void*); |
| *(void**)sp = nil; |
| } |
| |
| // Continue as if lessstack had just called m->morepc |
| // (the PC that decided to grow the stack). |
| runtime·memclr((byte*)&label, sizeof label); |
| label.sp = sp; |
| label.pc = (uintptr)runtime·lessstack; |
| label.g = m->curg; |
| if(newstackcall) |
| runtime·gostartcallfn(&label, (FuncVal*)m->cret); |
| else { |
| runtime·gostartcall(&label, (void(*)(void))gp->sched.pc, gp->sched.ctxt); |
| gp->sched.ctxt = nil; |
| } |
| gp->status = oldstatus; |
| runtime·gogo(&label); |
| |
| *(int32*)345 = 123; // never return |
| } |
| |
| // adjust Gobuf as if it executed a call to fn |
| // and then did an immediate gosave. |
| void |
| runtime·gostartcallfn(Gobuf *gobuf, FuncVal *fv) |
| { |
| runtime·gostartcall(gobuf, fv->fn, fv); |
| } |
| |
| // Maybe shrink the stack being used by gp. |
| // Called at garbage collection time. |
| void |
| runtime·shrinkstack(G *gp) |
| { |
| int32 nframes; |
| byte *oldstk, *oldbase; |
| uintptr used, oldsize, newsize; |
| MSpan *span; |
| |
| if(!runtime·copystack) |
| return; |
| oldstk = (byte*)gp->stackguard - StackGuard; |
| oldbase = (byte*)gp->stackbase + sizeof(Stktop); |
| oldsize = oldbase - oldstk; |
| newsize = oldsize / 2; |
| if(newsize < FixedStack) |
| return; // don't shrink below the minimum-sized stack |
| used = oldbase - (byte*)gp->sched.sp; |
| if(used >= oldsize / 4) |
| return; // still using at least 1/4 of the segment. |
| |
| // To shrink to less than 1/2 a page, we need to copy. |
| if(newsize < PageSize/2) { |
| if(gp->syscallstack != (uintptr)nil) // TODO: can we handle this case? |
| return; |
| #ifdef GOOS_windows |
| if(gp->m != nil && gp->m->libcallsp != 0) |
| return; |
| #endif |
| nframes = copyabletopsegment(gp); |
| if(nframes == -1) |
| return; |
| copystack(gp, nframes, newsize); |
| return; |
| } |
| |
| // To shrink a stack of one page size or more, we can shrink it |
| // without copying. Just deallocate the lower half. |
| span = runtime·MHeap_LookupMaybe(&runtime·mheap, oldstk); |
| if(span == nil) |
| return; // stack allocated outside heap. Can't shrink it. Can happen if stack is allocated while inside malloc. TODO: shrink by copying? |
| if(span->elemsize != oldsize) |
| runtime·throw("span element size doesn't match stack size"); |
| if((uintptr)oldstk != span->start << PageShift) |
| runtime·throw("stack not at start of span"); |
| |
| if(StackDebug) |
| runtime·printf("shrinking stack in place %p %X->%X\n", oldstk, oldsize, newsize); |
| |
| // new stack guard for smaller stack |
| gp->stackguard = (uintptr)oldstk + newsize + StackGuard; |
| gp->stackguard0 = (uintptr)oldstk + newsize + StackGuard; |
| if(gp->stack0 == (uintptr)oldstk) |
| gp->stack0 = (uintptr)oldstk + newsize; |
| gp->stacksize -= oldsize - newsize; |
| |
| // Free bottom half of the stack. |
| if(runtime·debug.efence || StackFromSystem) { |
| if(runtime·debug.efence || StackFaultOnFree) |
| runtime·SysFault(oldstk, newsize); |
| else |
| runtime·SysFree(oldstk, newsize, &mstats.stacks_sys); |
| return; |
| } |
| // First, we trick malloc into thinking |
| // we allocated the stack as two separate half-size allocs. Then the |
| // free() call does the rest of the work for us. |
| runtime·MSpan_EnsureSwept(span); |
| runtime·MHeap_SplitSpan(&runtime·mheap, span); |
| runtime·free(oldstk); |
| } |
