src/pkg/runtime/stack.c - go - Git at Google

 // Copyright 2013 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 #include "runtime.h"
 #include "arch_GOARCH.h"
 #include "malloc.h"
 #include "stack.h"

 typedef struct StackCacheNode StackCacheNode;
 struct StackCacheNode
 {
 	StackCacheNode *next;
 	void*	batch[StackCacheBatch-1];
 };

 static StackCacheNode *stackcache;
 static Lock stackcachemu;

 // stackcacherefill/stackcacherelease implement a global cache of stack segments.
 // The cache is required to prevent unlimited growth of per-thread caches.
 static void
 stackcacherefill(void)
 {
 	StackCacheNode *n;
 	int32 i, pos;

 	runtime·lock(&stackcachemu);
 	n = stackcache;
 	if(n)
 		stackcache = n->next;
 	runtime·unlock(&stackcachemu);
 	if(n == nil) {
 		n = (StackCacheNode*)runtime·SysAlloc(FixedStack*StackCacheBatch);
 		if(n == nil)
 			runtime·throw("out of memory (stackcacherefill)");
 		runtime·xadd64(&mstats.stacks_sys, FixedStack*StackCacheBatch);
 		for(i = 0; i < StackCacheBatch-1; i++)
 			n->batch[i] = (byte*)n + (i+1)*FixedStack;
 	}
 	pos = m->stackcachepos;
 	for(i = 0; i < StackCacheBatch-1; i++) {
 		m->stackcache[pos] = n->batch[i];
 		pos = (pos + 1) % StackCacheSize;
 	}
 	m->stackcache[pos] = n;
 	pos = (pos + 1) % StackCacheSize;
 	m->stackcachepos = pos;
 	m->stackcachecnt += StackCacheBatch;
 }

 static void
 stackcacherelease(void)
 {
 	StackCacheNode *n;
 	uint32 i, pos;

 	pos = (m->stackcachepos - m->stackcachecnt) % StackCacheSize;
 	n = (StackCacheNode*)m->stackcache[pos];
 	pos = (pos + 1) % StackCacheSize;
 	for(i = 0; i < StackCacheBatch-1; i++) {
 		n->batch[i] = m->stackcache[pos];
 		pos = (pos + 1) % StackCacheSize;
 	}
 	m->stackcachecnt -= StackCacheBatch;
 	runtime·lock(&stackcachemu);
 	n->next = stackcache;
 	stackcache = n;
 	runtime·unlock(&stackcachemu);
 }

 void*
 runtime·stackalloc(uint32 n)
 {
 	uint32 pos;
 	void *v;

 	// Stackalloc must be called on scheduler stack, so that we
 	// never try to grow the stack during the code that stackalloc runs.
 	// Doing so would cause a deadlock (issue 1547).
 	if(g != m->g0)
 		runtime·throw("stackalloc not on scheduler stack");

 	// Stack allocator uses malloc/free most of the time,
 	// but if we're in the middle of malloc and need stack,
 	// we have to do something else to avoid deadlock.
 	// In that case, we fall back on a fixed-size free-list
 	// allocator, assuming that inside malloc all the stack
 	// frames are small, so that all the stack allocations
 	// will be a single size, the minimum (right now, 5k).
 	if(n == FixedStack || m->mallocing || m->gcing) {
 		if(n != FixedStack) {
 			runtime·printf("stackalloc: in malloc, size=%d want %d\n", FixedStack, n);
 			runtime·throw("stackalloc");
 		}
 		if(m->stackcachecnt == 0)
 			stackcacherefill();
 		pos = m->stackcachepos;
 		pos = (pos - 1) % StackCacheSize;
 		v = m->stackcache[pos];
 		m->stackcachepos = pos;
 		m->stackcachecnt--;
 		m->stackinuse++;
 		return v;
 	}
 	return runtime·mallocgc(n, FlagNoProfiling|FlagNoGC, 0, 0);
 }

 void
 runtime·stackfree(void *v, uintptr n)
 {
 	uint32 pos;

 	if(n == FixedStack || m->mallocing || m->gcing) {
 		if(m->stackcachecnt == StackCacheSize)
 			stackcacherelease();
 		pos = m->stackcachepos;
 		m->stackcache[pos] = v;
 		m->stackcachepos = (pos + 1) % StackCacheSize;
 		m->stackcachecnt++;
 		m->stackinuse--;
 		return;
 	}
 	runtime·free(v);
 }

 // Called from runtime·lessstack when returning from a function which
 // allocated a new stack segment.  The function's return value is in
 // m->cret.
 void
 runtime·oldstack(void)
 {
 	Stktop *top;
 	Gobuf label;
 	uint32 argsize;
 	uintptr cret;
 	byte *sp, *old;
 	uintptr *src, *dst, *dstend;
 	G *gp;
 	int64 goid;

 //printf("oldstack m->cret=%p\n", m->cret);

 	gp = m->curg;
 	top = (Stktop*)gp->stackbase;
 	old = (byte*)gp->stackguard - StackGuard;
 	sp = (byte*)top;
 	argsize = top->argsize;
 	if(argsize > 0) {
 		sp -= argsize;
 		dst = (uintptr*)top->argp;
 		dstend = dst + argsize/sizeof(*dst);
 		src = (uintptr*)sp;
 		while(dst < dstend)
 			*dst++ = *src++;
 	}
 	goid = top->gobuf.g->goid;	// fault if g is bad, before gogo
 	USED(goid);

 	label = top->gobuf;
 	gp->stackbase = (uintptr)top->stackbase;
 	gp->stackguard = (uintptr)top->stackguard;
 	if(top->free != 0)
 		runtime·stackfree(old, top->free);

 	cret = m->cret;
 	m->cret = 0;  // drop reference
 	runtime·gogo(&label, cret);
 }

 // Called from reflect·call or from runtime·morestack when a new
 // stack segment is needed.  Allocate a new stack big enough for
 // m->moreframesize bytes, copy m->moreargsize bytes to the new frame,
 // and then act as though runtime·lessstack called the function at
 // m->morepc.
 void
 runtime·newstack(void)
 {
 	int32 framesize, minalloc, argsize;
 	Stktop *top;
 	byte *stk, *sp;
 	uintptr *src, *dst, *dstend;
 	G *gp;
 	Gobuf label;
 	bool reflectcall;
 	uintptr free;

 	framesize = m->moreframesize;
 	argsize = m->moreargsize;
 	gp = m->curg;

 	if(m->morebuf.sp < gp->stackguard - StackGuard) {
 		runtime·printf("runtime: split stack overflow: %p < %p\n", m->morebuf.sp, gp->stackguard - StackGuard);
 		runtime·throw("runtime: split stack overflow");
 	}
 	if(argsize % sizeof(uintptr) != 0) {
 		runtime·printf("runtime: stack split with misaligned argsize %d\n", argsize);
 		runtime·throw("runtime: stack split argsize");
 	}

 	minalloc = 0;
 	reflectcall = framesize==1;
 	if(reflectcall) {
 		framesize = 0;
 		// moreframesize_minalloc is only set in runtime·gc(),
 		// that calls newstack via reflect·call().
 		minalloc = m->moreframesize_minalloc;
 		m->moreframesize_minalloc = 0;
 		if(framesize < minalloc)
 			framesize = minalloc;
 	}

 	if(reflectcall && minalloc == 0 && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > gp->stackguard) {
 		// special case: called from reflect.call (framesize==1)
 		// to call code with an arbitrary argument size,
 		// and we have enough space on the current stack.
 		// the new Stktop* is necessary to unwind, but
 		// we don't need to create a new segment.
 		top = (Stktop*)(m->morebuf.sp - sizeof(*top));
 		stk = (byte*)gp->stackguard - StackGuard;
 		free = 0;
 	} else {
 		// allocate new segment.
 		framesize += argsize;
 		framesize += StackExtra;	// room for more functions, Stktop.
 		if(framesize < StackMin)
 			framesize = StackMin;
 		framesize += StackSystem;
 		stk = runtime·stackalloc(framesize);
 		top = (Stktop*)(stk+framesize-sizeof(*top));
 		free = framesize;
 	}

 	if(0) {
 		runtime·printf("newstack framesize=%d argsize=%d morepc=%p moreargp=%p gobuf=%p, %p top=%p old=%p\n",
 			framesize, argsize, m->morepc, m->moreargp, m->morebuf.pc, m->morebuf.sp, top, gp->stackbase);
 	}

 	top->stackbase = (byte*)gp->stackbase;
 	top->stackguard = (byte*)gp->stackguard;
 	top->gobuf = m->morebuf;
 	top->argp = m->moreargp;
 	top->argsize = argsize;
 	top->free = free;
 	m->moreargp = nil;
 	m->morebuf.pc = nil;
 	m->morebuf.sp = (uintptr)nil;

 	// copy flag from panic
 	top->panic = gp->ispanic;
 	gp->ispanic = false;

 	gp->stackbase = (uintptr)top;
 	gp->stackguard = (uintptr)stk + StackGuard;

 	sp = (byte*)top;
 	if(argsize > 0) {
 		sp -= argsize;
 		dst = (uintptr*)sp;
 		dstend = dst + argsize/sizeof(*dst);
 		src = (uintptr*)top->argp;
 		while(dst < dstend)
 			*dst++ = *src++;
 	}
 	if(thechar == '5') {
 		// caller would have saved its LR below args.
 		sp -= sizeof(void*);
 		*(void**)sp = nil;
 	}

 	// Continue as if lessstack had just called m->morepc
 	// (the PC that decided to grow the stack).
 	label.sp = (uintptr)sp;
 	label.pc = (byte*)runtime·lessstack;
 	label.g = m->curg;
 	if(reflectcall)
 		runtime·gogocallfn(&label, (FuncVal*)m->morepc);
 	else
 		runtime·gogocall(&label, m->morepc, m->cret);

 	*(int32*)345 = 123;	// never return
 }
	// Copyright 2013 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	#include "runtime.h"
	#include "arch_GOARCH.h"
	#include "malloc.h"
	#include "stack.h"

	typedef struct StackCacheNode StackCacheNode;
	struct StackCacheNode
	{
	StackCacheNode *next;
	void* batch[StackCacheBatch-1];
	};

	static StackCacheNode *stackcache;
	static Lock stackcachemu;

	// stackcacherefill/stackcacherelease implement a global cache of stack segments.
	// The cache is required to prevent unlimited growth of per-thread caches.
	static void
	stackcacherefill(void)
	{
	StackCacheNode *n;
	int32 i, pos;

	runtime·lock(&stackcachemu);
	n = stackcache;
	if(n)
	stackcache = n->next;
	runtime·unlock(&stackcachemu);
	if(n == nil) {
	n = (StackCacheNode)runtime·SysAlloc(FixedStackStackCacheBatch);
	if(n == nil)
	runtime·throw("out of memory (stackcacherefill)");
	runtime·xadd64(&mstats.stacks_sys, FixedStack*StackCacheBatch);
	for(i = 0; i < StackCacheBatch-1; i++)
	n->batch[i] = (byte)n + (i+1)FixedStack;
	}
	pos = m->stackcachepos;
	for(i = 0; i < StackCacheBatch-1; i++) {
	m->stackcache[pos] = n->batch[i];
	pos = (pos + 1) % StackCacheSize;
	}
	m->stackcache[pos] = n;
	pos = (pos + 1) % StackCacheSize;
	m->stackcachepos = pos;
	m->stackcachecnt += StackCacheBatch;
	}

	static void
	stackcacherelease(void)
	{
	StackCacheNode *n;
	uint32 i, pos;

	pos = (m->stackcachepos - m->stackcachecnt) % StackCacheSize;
	n = (StackCacheNode*)m->stackcache[pos];
	pos = (pos + 1) % StackCacheSize;
	for(i = 0; i < StackCacheBatch-1; i++) {
	n->batch[i] = m->stackcache[pos];
	pos = (pos + 1) % StackCacheSize;
	}
	m->stackcachecnt -= StackCacheBatch;
	runtime·lock(&stackcachemu);
	n->next = stackcache;
	stackcache = n;
	runtime·unlock(&stackcachemu);
	}

	void*
	runtime·stackalloc(uint32 n)
	{
	uint32 pos;
	void *v;

	// Stackalloc must be called on scheduler stack, so that we
	// never try to grow the stack during the code that stackalloc runs.
	// Doing so would cause a deadlock (issue 1547).
	if(g != m->g0)
	runtime·throw("stackalloc not on scheduler stack");

	// Stack allocator uses malloc/free most of the time,
	// but if we're in the middle of malloc and need stack,
	// we have to do something else to avoid deadlock.
	// In that case, we fall back on a fixed-size free-list
	// allocator, assuming that inside malloc all the stack
	// frames are small, so that all the stack allocations
	// will be a single size, the minimum (right now, 5k).
	if(n == FixedStack \|\| m->mallocing \|\| m->gcing) {
	if(n != FixedStack) {
	runtime·printf("stackalloc: in malloc, size=%d want %d\n", FixedStack, n);
	runtime·throw("stackalloc");
	}
	if(m->stackcachecnt == 0)
	stackcacherefill();
	pos = m->stackcachepos;
	pos = (pos - 1) % StackCacheSize;
	v = m->stackcache[pos];
	m->stackcachepos = pos;
	m->stackcachecnt--;
	m->stackinuse++;
	return v;
	}
	return runtime·mallocgc(n, FlagNoProfiling\|FlagNoGC, 0, 0);
	}

	void
	runtime·stackfree(void *v, uintptr n)
	{
	uint32 pos;

	if(n == FixedStack \|\| m->mallocing \|\| m->gcing) {
	if(m->stackcachecnt == StackCacheSize)
	stackcacherelease();
	pos = m->stackcachepos;
	m->stackcache[pos] = v;
	m->stackcachepos = (pos + 1) % StackCacheSize;
	m->stackcachecnt++;
	m->stackinuse--;
	return;
	}
	runtime·free(v);
	}

	// Called from runtime·lessstack when returning from a function which
	// allocated a new stack segment. The function's return value is in
	// m->cret.
	void
	runtime·oldstack(void)
	{
	Stktop *top;
	Gobuf label;
	uint32 argsize;
	uintptr cret;
	byte sp, old;
	uintptr src, dst, *dstend;
	G *gp;
	int64 goid;

	//printf("oldstack m->cret=%p\n", m->cret);

	gp = m->curg;
	top = (Stktop*)gp->stackbase;
	old = (byte*)gp->stackguard - StackGuard;
	sp = (byte*)top;
	argsize = top->argsize;
	if(argsize > 0) {
	sp -= argsize;
	dst = (uintptr*)top->argp;
	dstend = dst + argsize/sizeof(*dst);
	src = (uintptr*)sp;
	while(dst < dstend)
	dst++ = src++;
	}
	goid = top->gobuf.g->goid; // fault if g is bad, before gogo
	USED(goid);

	label = top->gobuf;
	gp->stackbase = (uintptr)top->stackbase;
	gp->stackguard = (uintptr)top->stackguard;
	if(top->free != 0)
	runtime·stackfree(old, top->free);

	cret = m->cret;
	m->cret = 0; // drop reference
	runtime·gogo(&label, cret);
	}

	// Called from reflect·call or from runtime·morestack when a new
	// stack segment is needed. Allocate a new stack big enough for
	// m->moreframesize bytes, copy m->moreargsize bytes to the new frame,
	// and then act as though runtime·lessstack called the function at
	// m->morepc.
	void
	runtime·newstack(void)
	{
	int32 framesize, minalloc, argsize;
	Stktop *top;
	byte stk, sp;
	uintptr src, dst, *dstend;
	G *gp;
	Gobuf label;
	bool reflectcall;
	uintptr free;

	framesize = m->moreframesize;
	argsize = m->moreargsize;
	gp = m->curg;

	if(m->morebuf.sp < gp->stackguard - StackGuard) {
	runtime·printf("runtime: split stack overflow: %p < %p\n", m->morebuf.sp, gp->stackguard - StackGuard);
	runtime·throw("runtime: split stack overflow");
	}
	if(argsize % sizeof(uintptr) != 0) {
	runtime·printf("runtime: stack split with misaligned argsize %d\n", argsize);
	runtime·throw("runtime: stack split argsize");
	}

	minalloc = 0;
	reflectcall = framesize==1;
	if(reflectcall) {
	framesize = 0;
	// moreframesize_minalloc is only set in runtime·gc(),
	// that calls newstack via reflect·call().
	minalloc = m->moreframesize_minalloc;
	m->moreframesize_minalloc = 0;
	if(framesize < minalloc)
	framesize = minalloc;
	}

	if(reflectcall && minalloc == 0 && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > gp->stackguard) {
	// special case: called from reflect.call (framesize==1)
	// to call code with an arbitrary argument size,
	// and we have enough space on the current stack.
	// the new Stktop* is necessary to unwind, but
	// we don't need to create a new segment.
	top = (Stktop)(m->morebuf.sp - sizeof(top));
	stk = (byte*)gp->stackguard - StackGuard;
	free = 0;
	} else {
	// allocate new segment.
	framesize += argsize;
	framesize += StackExtra; // room for more functions, Stktop.
	if(framesize < StackMin)
	framesize = StackMin;
	framesize += StackSystem;
	stk = runtime·stackalloc(framesize);
	top = (Stktop)(stk+framesize-sizeof(top));
	free = framesize;
	}

	if(0) {
	runtime·printf("newstack framesize=%d argsize=%d morepc=%p moreargp=%p gobuf=%p, %p top=%p old=%p\n",
	framesize, argsize, m->morepc, m->moreargp, m->morebuf.pc, m->morebuf.sp, top, gp->stackbase);
	}

	top->stackbase = (byte*)gp->stackbase;
	top->stackguard = (byte*)gp->stackguard;
	top->gobuf = m->morebuf;
	top->argp = m->moreargp;
	top->argsize = argsize;
	top->free = free;
	m->moreargp = nil;
	m->morebuf.pc = nil;
	m->morebuf.sp = (uintptr)nil;

	// copy flag from panic
	top->panic = gp->ispanic;
	gp->ispanic = false;

	gp->stackbase = (uintptr)top;
	gp->stackguard = (uintptr)stk + StackGuard;

	sp = (byte*)top;
	if(argsize > 0) {
	sp -= argsize;
	dst = (uintptr*)sp;
	dstend = dst + argsize/sizeof(*dst);
	src = (uintptr*)top->argp;
	while(dst < dstend)
	dst++ = src++;
	}
	if(thechar == '5') {
	// caller would have saved its LR below args.
	sp -= sizeof(void*);
	(void*)sp = nil;
	}

	// Continue as if lessstack had just called m->morepc
	// (the PC that decided to grow the stack).
	label.sp = (uintptr)sp;
	label.pc = (byte*)runtime·lessstack;
	label.g = m->curg;
	if(reflectcall)
	runtime·gogocallfn(&label, (FuncVal*)m->morepc);
	else
	runtime·gogocall(&label, m->morepc, m->cret);

	(int32)345 = 123; // never return
	}