src/pkg/runtime/malloc.cgo - go - Git at Google

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 // See malloc.h for overview.
 //
 // TODO(rsc): double-check stats.

 package runtime
 #include "runtime.h"
 #include "malloc.h"
 #include "defs.h"
 #include "type.h"

 MHeap mheap;
 MStats mstats;

 // Allocate an object of at least size bytes.
 // Small objects are allocated from the per-thread cache's free lists.
 // Large objects (> 32 kB) are allocated straight from the heap.
 void*
 mallocgc(uintptr size, uint32 refflag, int32 dogc)
 {
 	int32 sizeclass;
 	MCache *c;
 	uintptr npages;
 	MSpan *s;
 	void *v;
 	uint32 *ref;

 	if(gcwaiting && g != m->g0 && m->locks == 0)
 		gosched();
 	if(m->mallocing)
 		throw("malloc/free - deadlock");
 	m->mallocing = 1;
 	if(size == 0)
 		size = 1;

 	mstats.nmalloc++;
 	if(size <= MaxSmallSize) {
 		// Allocate from mcache free lists.
 		sizeclass = SizeToClass(size);
 		size = class_to_size[sizeclass];
 		c = m->mcache;
 		v = MCache_Alloc(c, sizeclass, size);
 		if(v == nil)
 			throw("out of memory");
 		mstats.alloc += size;
 	} else {
 		// TODO(rsc): Report tracebacks for very large allocations.

 		// Allocate directly from heap.
 		npages = size >> PageShift;
 		if((size & PageMask) != 0)
 			npages++;
 		s = MHeap_Alloc(&mheap, npages, 0);
 		if(s == nil)
 			throw("out of memory");
 		mstats.alloc += npages<<PageShift;
 		v = (void*)(s->start << PageShift);
 	}

 	// setup for mark sweep
 	if(!mlookup(v, nil, nil, &ref)) {
 		printf("malloc %D; mlookup failed\n", (uint64)size);
 		throw("malloc mlookup");
 	}
 	*ref = RefNone | refflag;

 	m->mallocing = 0;

 	if(dogc && mstats.inuse_pages > mstats.next_gc)
 		gc(0);
 	return v;
 }

 void*
 malloc(uintptr size)
 {
 	return mallocgc(size, 0, 0);
 }

 // Free the object whose base pointer is v.
 void
 free(void *v)
 {
 	int32 sizeclass, size;
 	uintptr page, tmp;
 	MSpan *s;
 	MCache *c;
 	uint32 *ref;

 	if(v == nil)
 		return;

 	if(m->mallocing)
 		throw("malloc/free - deadlock");
 	m->mallocing = 1;

 	if(!mlookup(v, nil, nil, &ref)) {
 		printf("free %p: not an allocated block\n", v);
 		throw("free mlookup");
 	}
 	*ref = RefFree;

 	// Find size class for v.
 	page = (uintptr)v >> PageShift;
 	sizeclass = MHeapMapCache_GET(&mheap.mapcache, page, tmp);
 	if(sizeclass == 0) {
 		// Missed in cache.
 		s = MHeap_Lookup(&mheap, page);
 		if(s == nil)
 			throw("free - invalid pointer");
 		sizeclass = s->sizeclass;
 		if(sizeclass == 0) {
 			// Large object.
 			mstats.alloc -= s->npages<<PageShift;
 			runtime_memclr(v, s->npages<<PageShift);
 			MHeap_Free(&mheap, s);
 			goto out;
 		}
 		MHeapMapCache_SET(&mheap.mapcache, page, sizeclass);
 	}

 	// Small object.
 	c = m->mcache;
 	size = class_to_size[sizeclass];
 	runtime_memclr(v, size);
 	mstats.alloc -= size;
 	MCache_Free(c, v, sizeclass, size);

 out:
 	m->mallocing = 0;
 }

 int32
 mlookup(void *v, byte **base, uintptr *size, uint32 **ref)
 {
 	uintptr n, nobj, i;
 	byte *p;
 	MSpan *s;

 	mstats.nlookup++;
 	s = MHeap_LookupMaybe(&mheap, (uintptr)v>>PageShift);
 	if(s == nil) {
 		if(base)
 			*base = nil;
 		if(size)
 			*size = 0;
 		if(ref)
 			*ref = 0;
 		return 0;
 	}

 	p = (byte*)((uintptr)s->start<<PageShift);
 	if(s->sizeclass == 0) {
 		// Large object.
 		if(base)
 			*base = p;
 		if(size)
 			*size = s->npages<<PageShift;
 		if(ref)
 			*ref = &s->gcref0;
 		return 1;
 	}

 	if((byte*)v >= (byte*)s->gcref) {
 		// pointers into the gc ref counts
 		// do not count as pointers.
 		return 0;
 	}

 	n = class_to_size[s->sizeclass];
 	i = ((byte*)v - p)/n;
 	if(base)
 		*base = p + i*n;
 	if(size)
 		*size = n;
 	nobj = (s->npages << PageShift) / (n + RefcountOverhead);
 	if((byte*)s->gcref < p || (byte*)(s->gcref+nobj) > p+(s->npages<<PageShift)) {
 		printf("odd span state=%d span=%p base=%p sizeclass=%d n=%D size=%D npages=%D\n",
 			s->state, s, p, s->sizeclass, (uint64)nobj, (uint64)n, (uint64)s->npages);
 		printf("s->base sizeclass %d v=%p base=%p gcref=%p blocksize=%D nobj=%D size=%D end=%p end=%p\n",
 			s->sizeclass, v, p, s->gcref, (uint64)s->npages<<PageShift,
 			(uint64)nobj, (uint64)n, s->gcref + nobj, p+(s->npages<<PageShift));
 		throw("bad gcref");
 	}
 	if(ref)
 		*ref = &s->gcref[i];

 	return 1;
 }

 MCache*
 allocmcache(void)
 {
 	return FixAlloc_Alloc(&mheap.cachealloc);
 }

 void
 mallocinit(void)
 {
 	InitSizes();
 	MHeap_Init(&mheap, SysAlloc);
 	m->mcache = allocmcache();

 	// See if it works.
 	free(malloc(1));
 }

 // Runtime stubs.

 void*
 mal(uint32 n)
 {
 	return mallocgc(n, 0, 1);
 }

 // Stack allocator uses malloc/free most of the time,
 // but if we're in the middle of malloc and need stack,
 // we have to do something else to avoid deadlock.
 // In that case, we fall back on a fixed-size free-list
 // allocator, assuming that inside malloc all the stack
 // frames are small, so that all the stack allocations
 // will be a single size, the minimum (right now, 5k).
 struct {
 	Lock;
 	FixAlloc;
 } stacks;

 void*
 stackalloc(uint32 n)
 {
 	void *v;
 	uint32 *ref;

 	if(m->mallocing || m->gcing) {
 		lock(&stacks);
 		if(stacks.size == 0)
 			FixAlloc_Init(&stacks, n, SysAlloc, nil, nil);
 		if(stacks.size != n) {
 			printf("stackalloc: in malloc, size=%D want %d", (uint64)stacks.size, n);
 			throw("stackalloc");
 		}
 		v = FixAlloc_Alloc(&stacks);
 		unlock(&stacks);
 		return v;
 	}
 	v = malloc(n);
 	if(!mlookup(v, nil, nil, &ref))
 		throw("stackalloc mlookup");
 	*ref = RefStack;
 	return v;
 }

 void
 stackfree(void *v)
 {
 	if(m->mallocing || m->gcing) {
 		lock(&stacks);
 		FixAlloc_Free(&stacks, v);
 		unlock(&stacks);
 		return;
 	}
 	free(v);
 }

 func Alloc(n uintptr) (p *byte) {
 	p = malloc(n);
 }

 func Free(p *byte) {
 	free(p);
 }

 func Lookup(p *byte) (base *byte, size uintptr) {
 	mlookup(p, &base, &size, nil);
 }

 func GC() {
 	gc(1);
 }

 func SetFinalizer(obj Eface, finalizer Eface) {
 	byte *base;
 	uintptr size;
 	FuncType *ft;

 	if(obj.type == nil) {
 		printf("runtime.SetFinalizer: first argument is nil interface\n");
 	throw:
 		throw("runtime.SetFinalizer");
 	}
 	if(obj.type->kind != KindPtr) {
 		printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.type->string);
 		goto throw;
 	}
 	if(!mlookup(obj.data, &base, &size, nil) || obj.data != base) {
 		printf("runtime.SetFinalizer: pointer not at beginning of allocated block\n");
 		goto throw;
 	}
 	if(finalizer.type != nil) {
 		if(finalizer.type->kind != KindFunc) {
 		badfunc:
 			printf("runtime.SetFinalizer: second argument is %S, not func(%S)\n", *finalizer.type->string, *obj.type->string);
 			goto throw;
 		}
 		ft = (FuncType*)finalizer.type;
 		if(ft->dotdotdot || ft->out.len != 0 || ft->in.len != 1 || *(Type**)ft->in.array != obj.type)
 			goto badfunc;
 		if(getfinalizer(obj.data, 0)) {
 			printf("runtime.SetFinalizer: finalizer already set");
 			goto throw;
 		}
 	}
 	addfinalizer(obj.data, finalizer.data);
 }
	// Copyright 2009 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	// See malloc.h for overview.
	//
	// TODO(rsc): double-check stats.

	package runtime
	#include "runtime.h"
	#include "malloc.h"
	#include "defs.h"
	#include "type.h"

	MHeap mheap;
	MStats mstats;

	// Allocate an object of at least size bytes.
	// Small objects are allocated from the per-thread cache's free lists.
	// Large objects (> 32 kB) are allocated straight from the heap.
	void*
	mallocgc(uintptr size, uint32 refflag, int32 dogc)
	{
	int32 sizeclass;
	MCache *c;
	uintptr npages;
	MSpan *s;
	void *v;
	uint32 *ref;

	if(gcwaiting && g != m->g0 && m->locks == 0)
	gosched();
	if(m->mallocing)
	throw("malloc/free - deadlock");
	m->mallocing = 1;
	if(size == 0)
	size = 1;

	mstats.nmalloc++;
	if(size <= MaxSmallSize) {
	// Allocate from mcache free lists.
	sizeclass = SizeToClass(size);
	size = class_to_size[sizeclass];
	c = m->mcache;
	v = MCache_Alloc(c, sizeclass, size);
	if(v == nil)
	throw("out of memory");
	mstats.alloc += size;
	} else {
	// TODO(rsc): Report tracebacks for very large allocations.

	// Allocate directly from heap.
	npages = size >> PageShift;
	if((size & PageMask) != 0)
	npages++;
	s = MHeap_Alloc(&mheap, npages, 0);
	if(s == nil)
	throw("out of memory");
	mstats.alloc += npages<<PageShift;
	v = (void*)(s->start << PageShift);
	}

	// setup for mark sweep
	if(!mlookup(v, nil, nil, &ref)) {
	printf("malloc %D; mlookup failed\n", (uint64)size);
	throw("malloc mlookup");
	}
	*ref = RefNone \| refflag;

	m->mallocing = 0;

	if(dogc && mstats.inuse_pages > mstats.next_gc)
	gc(0);
	return v;
	}

	void*
	malloc(uintptr size)
	{
	return mallocgc(size, 0, 0);
	}

	// Free the object whose base pointer is v.
	void
	free(void *v)
	{
	int32 sizeclass, size;
	uintptr page, tmp;
	MSpan *s;
	MCache *c;
	uint32 *ref;

	if(v == nil)
	return;

	if(m->mallocing)
	throw("malloc/free - deadlock");
	m->mallocing = 1;

	if(!mlookup(v, nil, nil, &ref)) {
	printf("free %p: not an allocated block\n", v);
	throw("free mlookup");
	}
	*ref = RefFree;

	// Find size class for v.
	page = (uintptr)v >> PageShift;
	sizeclass = MHeapMapCache_GET(&mheap.mapcache, page, tmp);
	if(sizeclass == 0) {
	// Missed in cache.
	s = MHeap_Lookup(&mheap, page);
	if(s == nil)
	throw("free - invalid pointer");
	sizeclass = s->sizeclass;
	if(sizeclass == 0) {
	// Large object.
	mstats.alloc -= s->npages<<PageShift;
	runtime_memclr(v, s->npages<<PageShift);
	MHeap_Free(&mheap, s);
	goto out;
	}
	MHeapMapCache_SET(&mheap.mapcache, page, sizeclass);
	}

	// Small object.
	c = m->mcache;
	size = class_to_size[sizeclass];
	runtime_memclr(v, size);
	mstats.alloc -= size;
	MCache_Free(c, v, sizeclass, size);

	out:
	m->mallocing = 0;
	}

	int32
	mlookup(void v, byte base, uintptr size, uint32 **ref)
	{
	uintptr n, nobj, i;
	byte *p;
	MSpan *s;

	mstats.nlookup++;
	s = MHeap_LookupMaybe(&mheap, (uintptr)v>>PageShift);
	if(s == nil) {
	if(base)
	*base = nil;
	if(size)
	*size = 0;
	if(ref)
	*ref = 0;
	return 0;
	}

	p = (byte*)((uintptr)s->start<<PageShift);
	if(s->sizeclass == 0) {
	// Large object.
	if(base)
	*base = p;
	if(size)
	*size = s->npages<<PageShift;
	if(ref)
	*ref = &s->gcref0;
	return 1;
	}

	if((byte)v >= (byte)s->gcref) {
	// pointers into the gc ref counts
	// do not count as pointers.
	return 0;
	}

	n = class_to_size[s->sizeclass];
	i = ((byte*)v - p)/n;
	if(base)
	base = p + in;
	if(size)
	*size = n;
	nobj = (s->npages << PageShift) / (n + RefcountOverhead);
	if((byte)s->gcref < p \|\| (byte)(s->gcref+nobj) > p+(s->npages<<PageShift)) {
	printf("odd span state=%d span=%p base=%p sizeclass=%d n=%D size=%D npages=%D\n",
	s->state, s, p, s->sizeclass, (uint64)nobj, (uint64)n, (uint64)s->npages);
	printf("s->base sizeclass %d v=%p base=%p gcref=%p blocksize=%D nobj=%D size=%D end=%p end=%p\n",
	s->sizeclass, v, p, s->gcref, (uint64)s->npages<<PageShift,
	(uint64)nobj, (uint64)n, s->gcref + nobj, p+(s->npages<<PageShift));
	throw("bad gcref");
	}
	if(ref)
	*ref = &s->gcref[i];

	return 1;
	}

	MCache*
	allocmcache(void)
	{
	return FixAlloc_Alloc(&mheap.cachealloc);
	}

	void
	mallocinit(void)
	{
	InitSizes();
	MHeap_Init(&mheap, SysAlloc);
	m->mcache = allocmcache();

	// See if it works.
	free(malloc(1));
	}

	// Runtime stubs.

	void*
	mal(uint32 n)
	{
	return mallocgc(n, 0, 1);
	}

	// Stack allocator uses malloc/free most of the time,
	// but if we're in the middle of malloc and need stack,
	// we have to do something else to avoid deadlock.
	// In that case, we fall back on a fixed-size free-list
	// allocator, assuming that inside malloc all the stack
	// frames are small, so that all the stack allocations
	// will be a single size, the minimum (right now, 5k).
	struct {
	Lock;
	FixAlloc;
	} stacks;

	void*
	stackalloc(uint32 n)
	{
	void *v;
	uint32 *ref;

	if(m->mallocing \|\| m->gcing) {
	lock(&stacks);
	if(stacks.size == 0)
	FixAlloc_Init(&stacks, n, SysAlloc, nil, nil);
	if(stacks.size != n) {
	printf("stackalloc: in malloc, size=%D want %d", (uint64)stacks.size, n);
	throw("stackalloc");
	}
	v = FixAlloc_Alloc(&stacks);
	unlock(&stacks);
	return v;
	}
	v = malloc(n);
	if(!mlookup(v, nil, nil, &ref))
	throw("stackalloc mlookup");
	*ref = RefStack;
	return v;
	}

	void
	stackfree(void *v)
	{
	if(m->mallocing \|\| m->gcing) {
	lock(&stacks);
	FixAlloc_Free(&stacks, v);
	unlock(&stacks);
	return;
	}
	free(v);
	}

	func Alloc(n uintptr) (p *byte) {
	p = malloc(n);
	}

	func Free(p *byte) {
	free(p);
	}

	func Lookup(p byte) (base byte, size uintptr) {
	mlookup(p, &base, &size, nil);
	}

	func GC() {
	gc(1);
	}

	func SetFinalizer(obj Eface, finalizer Eface) {
	byte *base;
	uintptr size;
	FuncType *ft;

	if(obj.type == nil) {
	printf("runtime.SetFinalizer: first argument is nil interface\n");
	throw:
	throw("runtime.SetFinalizer");
	}
	if(obj.type->kind != KindPtr) {
	printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.type->string);
	goto throw;
	}
	if(!mlookup(obj.data, &base, &size, nil) \|\| obj.data != base) {
	printf("runtime.SetFinalizer: pointer not at beginning of allocated block\n");
	goto throw;
	}
	if(finalizer.type != nil) {
	if(finalizer.type->kind != KindFunc) {
	badfunc:
	printf("runtime.SetFinalizer: second argument is %S, not func(%S)\n", finalizer.type->string, obj.type->string);
	goto throw;
	}
	ft = (FuncType*)finalizer.type;
	if(ft->dotdotdot \|\| ft->out.len != 0 \|\| ft->in.len != 1 \|\| (Type*)ft->in.array != obj.type)
	goto badfunc;
	if(getfinalizer(obj.data, 0)) {
	printf("runtime.SetFinalizer: finalizer already set");
	goto throw;
	}
	}
	addfinalizer(obj.data, finalizer.data);
	}