| // Copyright 2009 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // See malloc.h for overview. |
| // |
| // TODO(rsc): double-check stats. |
| |
| #include "runtime.h" |
| #include "arch_GOARCH.h" |
| #include "malloc.h" |
| #include "type.h" |
| #include "typekind.h" |
| #include "race.h" |
| #include "stack.h" |
| #include "textflag.h" |
| |
| // Mark mheap as 'no pointers', it does not contain interesting pointers but occupies ~45K. |
| #pragma dataflag NOPTR |
| MHeap runtime·mheap; |
| #pragma dataflag NOPTR |
| MStats runtime·memstats; |
| |
| int32 |
| runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp) |
| { |
| uintptr n, i; |
| byte *p; |
| MSpan *s; |
| |
| g->m->mcache->local_nlookup++; |
| if (sizeof(void*) == 4 && g->m->mcache->local_nlookup >= (1<<30)) { |
| // purge cache stats to prevent overflow |
| runtime·lock(&runtime·mheap.lock); |
| runtime·purgecachedstats(g->m->mcache); |
| runtime·unlock(&runtime·mheap.lock); |
| } |
| |
| s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); |
| if(sp) |
| *sp = s; |
| if(s == nil) { |
| if(base) |
| *base = nil; |
| if(size) |
| *size = 0; |
| return 0; |
| } |
| |
| p = (byte*)((uintptr)s->start<<PageShift); |
| if(s->sizeclass == 0) { |
| // Large object. |
| if(base) |
| *base = p; |
| if(size) |
| *size = s->npages<<PageShift; |
| return 1; |
| } |
| |
| n = s->elemsize; |
| if(base) { |
| i = ((byte*)v - p)/n; |
| *base = p + i*n; |
| } |
| if(size) |
| *size = n; |
| |
| return 1; |
| } |
| |
| #pragma textflag NOSPLIT |
| void |
| runtime·purgecachedstats(MCache *c) |
| { |
| MHeap *h; |
| int32 i; |
| |
| // Protected by either heap or GC lock. |
| h = &runtime·mheap; |
| mstats.heap_alloc += c->local_cachealloc; |
| c->local_cachealloc = 0; |
| mstats.tinyallocs += c->local_tinyallocs; |
| c->local_tinyallocs = 0; |
| mstats.nlookup += c->local_nlookup; |
| c->local_nlookup = 0; |
| h->largefree += c->local_largefree; |
| c->local_largefree = 0; |
| h->nlargefree += c->local_nlargefree; |
| c->local_nlargefree = 0; |
| for(i=0; i<nelem(c->local_nsmallfree); i++) { |
| h->nsmallfree[i] += c->local_nsmallfree[i]; |
| c->local_nsmallfree[i] = 0; |
| } |
| } |
| |
| // Size of the trailing by_size array differs between Go and C, |
| // and all data after by_size is local to C, not exported to Go. |
| // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. |
| // sizeof_C_MStats is what C thinks about size of Go struct. |
| uintptr runtime·sizeof_C_MStats = offsetof(MStats, by_size[61]); |
| |
| #define MaxArena32 (2U<<30) |
| |
| // For use by Go. If it were a C enum it would be made available automatically, |
| // but the value of MaxMem is too large for enum. |
| uintptr runtime·maxmem = MaxMem; |
| |
| void |
| runtime·mallocinit(void) |
| { |
| byte *p, *p1; |
| uintptr arena_size, bitmap_size, spans_size, p_size; |
| extern byte runtime·end[]; |
| uintptr limit; |
| uint64 i; |
| bool reserved; |
| |
| p = nil; |
| p_size = 0; |
| arena_size = 0; |
| bitmap_size = 0; |
| spans_size = 0; |
| reserved = false; |
| |
| // for 64-bit build |
| USED(p); |
| USED(p_size); |
| USED(arena_size); |
| USED(bitmap_size); |
| USED(spans_size); |
| |
| runtime·InitSizes(); |
| |
| if(runtime·class_to_size[TinySizeClass] != TinySize) |
| runtime·throw("bad TinySizeClass"); |
| |
| // limit = runtime·memlimit(); |
| // See https://code.google.com/p/go/issues/detail?id=5049 |
| // TODO(rsc): Fix after 1.1. |
| limit = 0; |
| |
| // Set up the allocation arena, a contiguous area of memory where |
| // allocated data will be found. The arena begins with a bitmap large |
| // enough to hold 4 bits per allocated word. |
| if(sizeof(void*) == 8 && (limit == 0 || limit > (1<<30))) { |
| // On a 64-bit machine, allocate from a single contiguous reservation. |
| // 128 GB (MaxMem) should be big enough for now. |
| // |
| // The code will work with the reservation at any address, but ask |
| // SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f). |
| // Allocating a 128 GB region takes away 37 bits, and the amd64 |
| // doesn't let us choose the top 17 bits, so that leaves the 11 bits |
| // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 means |
| // that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df. |
| // In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid |
| // UTF-8 sequences, and they are otherwise as far away from |
| // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0 |
| // addresses. An earlier attempt to use 0x11f8 caused out of memory errors |
| // on OS X during thread allocations. 0x00c0 causes conflicts with |
| // AddressSanitizer which reserves all memory up to 0x0100. |
| // These choices are both for debuggability and to reduce the |
| // odds of the conservative garbage collector not collecting memory |
| // because some non-pointer block of memory had a bit pattern |
| // that matched a memory address. |
| // |
| // Actually we reserve 136 GB (because the bitmap ends up being 8 GB) |
| // but it hardly matters: e0 00 is not valid UTF-8 either. |
| // |
| // If this fails we fall back to the 32 bit memory mechanism |
| arena_size = MaxMem; |
| bitmap_size = arena_size / (sizeof(void*)*8/4); |
| spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[0]); |
| spans_size = ROUND(spans_size, PageSize); |
| for(i = 0; i <= 0x7f; i++) { |
| p = (void*)(i<<40 | 0x00c0ULL<<32); |
| p_size = bitmap_size + spans_size + arena_size + PageSize; |
| p = runtime·SysReserve(p, p_size, &reserved); |
| if(p != nil) |
| break; |
| } |
| } |
| if (p == nil) { |
| // On a 32-bit machine, we can't typically get away |
| // with a giant virtual address space reservation. |
| // Instead we map the memory information bitmap |
| // immediately after the data segment, large enough |
| // to handle another 2GB of mappings (256 MB), |
| // along with a reservation for another 512 MB of memory. |
| // When that gets used up, we'll start asking the kernel |
| // for any memory anywhere and hope it's in the 2GB |
| // following the bitmap (presumably the executable begins |
| // near the bottom of memory, so we'll have to use up |
| // most of memory before the kernel resorts to giving out |
| // memory before the beginning of the text segment). |
| // |
| // Alternatively we could reserve 512 MB bitmap, enough |
| // for 4GB of mappings, and then accept any memory the |
| // kernel threw at us, but normally that's a waste of 512 MB |
| // of address space, which is probably too much in a 32-bit world. |
| bitmap_size = MaxArena32 / (sizeof(void*)*8/4); |
| arena_size = 512<<20; |
| spans_size = MaxArena32 / PageSize * sizeof(runtime·mheap.spans[0]); |
| if(limit > 0 && arena_size+bitmap_size+spans_size > limit) { |
| bitmap_size = (limit / 9) & ~((1<<PageShift) - 1); |
| arena_size = bitmap_size * 8; |
| spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[0]); |
| } |
| spans_size = ROUND(spans_size, PageSize); |
| |
| // SysReserve treats the address we ask for, end, as a hint, |
| // not as an absolute requirement. If we ask for the end |
| // of the data segment but the operating system requires |
| // a little more space before we can start allocating, it will |
| // give out a slightly higher pointer. Except QEMU, which |
| // is buggy, as usual: it won't adjust the pointer upward. |
| // So adjust it upward a little bit ourselves: 1/4 MB to get |
| // away from the running binary image and then round up |
| // to a MB boundary. |
| p = (byte*)ROUND((uintptr)runtime·end + (1<<18), 1<<20); |
| p_size = bitmap_size + spans_size + arena_size + PageSize; |
| p = runtime·SysReserve(p, p_size, &reserved); |
| if(p == nil) |
| runtime·throw("runtime: cannot reserve arena virtual address space"); |
| } |
| |
| // PageSize can be larger than OS definition of page size, |
| // so SysReserve can give us a PageSize-unaligned pointer. |
| // To overcome this we ask for PageSize more and round up the pointer. |
| p1 = (byte*)ROUND((uintptr)p, PageSize); |
| |
| runtime·mheap.spans = (MSpan**)p1; |
| runtime·mheap.bitmap = p1 + spans_size; |
| runtime·mheap.arena_start = p1 + spans_size + bitmap_size; |
| runtime·mheap.arena_used = runtime·mheap.arena_start; |
| runtime·mheap.arena_end = p + p_size; |
| runtime·mheap.arena_reserved = reserved; |
| |
| if(((uintptr)runtime·mheap.arena_start & (PageSize-1)) != 0) |
| runtime·throw("misrounded allocation in mallocinit"); |
| |
| // Initialize the rest of the allocator. |
| runtime·MHeap_Init(&runtime·mheap); |
| g->m->mcache = runtime·allocmcache(); |
| } |
| |
| void* |
| runtime·MHeap_SysAlloc(MHeap *h, uintptr n) |
| { |
| byte *p, *p_end; |
| uintptr p_size; |
| bool reserved; |
| |
| if(n > h->arena_end - h->arena_used) { |
| // We are in 32-bit mode, maybe we didn't use all possible address space yet. |
| // Reserve some more space. |
| byte *new_end; |
| |
| p_size = ROUND(n + PageSize, 256<<20); |
| new_end = h->arena_end + p_size; |
| if(new_end <= h->arena_start + MaxArena32) { |
| // TODO: It would be bad if part of the arena |
| // is reserved and part is not. |
| p = runtime·SysReserve(h->arena_end, p_size, &reserved); |
| if(p == h->arena_end) { |
| h->arena_end = new_end; |
| h->arena_reserved = reserved; |
| } |
| else if(p+p_size <= h->arena_start + MaxArena32) { |
| // Keep everything page-aligned. |
| // Our pages are bigger than hardware pages. |
| h->arena_end = p+p_size; |
| h->arena_used = p + (-(uintptr)p&(PageSize-1)); |
| h->arena_reserved = reserved; |
| } else { |
| uint64 stat; |
| stat = 0; |
| runtime·SysFree(p, p_size, &stat); |
| } |
| } |
| } |
| if(n <= h->arena_end - h->arena_used) { |
| // Keep taking from our reservation. |
| p = h->arena_used; |
| runtime·SysMap(p, n, h->arena_reserved, &mstats.heap_sys); |
| h->arena_used += n; |
| runtime·MHeap_MapBits(h); |
| runtime·MHeap_MapSpans(h); |
| if(raceenabled) |
| runtime·racemapshadow(p, n); |
| |
| if(((uintptr)p & (PageSize-1)) != 0) |
| runtime·throw("misrounded allocation in MHeap_SysAlloc"); |
| return p; |
| } |
| |
| // If using 64-bit, our reservation is all we have. |
| if(h->arena_end - h->arena_start >= MaxArena32) |
| return nil; |
| |
| // On 32-bit, once the reservation is gone we can |
| // try to get memory at a location chosen by the OS |
| // and hope that it is in the range we allocated bitmap for. |
| p_size = ROUND(n, PageSize) + PageSize; |
| p = runtime·sysAlloc(p_size, &mstats.heap_sys); |
| if(p == nil) |
| return nil; |
| |
| if(p < h->arena_start || p+p_size - h->arena_start >= MaxArena32) { |
| runtime·printf("runtime: memory allocated by OS (%p) not in usable range [%p,%p)\n", |
| p, h->arena_start, h->arena_start+MaxArena32); |
| runtime·SysFree(p, p_size, &mstats.heap_sys); |
| return nil; |
| } |
| |
| p_end = p + p_size; |
| p += -(uintptr)p & (PageSize-1); |
| if(p+n > h->arena_used) { |
| h->arena_used = p+n; |
| if(p_end > h->arena_end) |
| h->arena_end = p_end; |
| runtime·MHeap_MapBits(h); |
| runtime·MHeap_MapSpans(h); |
| if(raceenabled) |
| runtime·racemapshadow(p, n); |
| } |
| |
| if(((uintptr)p & (PageSize-1)) != 0) |
| runtime·throw("misrounded allocation in MHeap_SysAlloc"); |
| return p; |
| } |
| |
| void |
| runtime·setFinalizer_m(void) |
| { |
| FuncVal *fn; |
| void *arg; |
| uintptr nret; |
| Type *fint; |
| PtrType *ot; |
| |
| fn = g->m->ptrarg[0]; |
| arg = g->m->ptrarg[1]; |
| nret = g->m->scalararg[0]; |
| fint = g->m->ptrarg[2]; |
| ot = g->m->ptrarg[3]; |
| g->m->ptrarg[0] = nil; |
| g->m->ptrarg[1] = nil; |
| g->m->ptrarg[2] = nil; |
| g->m->ptrarg[3] = nil; |
| |
| g->m->scalararg[0] = runtime·addfinalizer(arg, fn, nret, fint, ot); |
| } |
| |
| void |
| runtime·removeFinalizer_m(void) |
| { |
| void *p; |
| |
| p = g->m->ptrarg[0]; |
| g->m->ptrarg[0] = nil; |
| runtime·removefinalizer(p); |
| } |
| |
| // mcallable cache refill |
| void |
| runtime·mcacheRefill_m(void) |
| { |
| runtime·MCache_Refill(g->m->mcache, (int32)g->m->scalararg[0]); |
| } |
| |
| void |
| runtime·largeAlloc_m(void) |
| { |
| uintptr npages, size; |
| MSpan *s; |
| void *v; |
| int32 flag; |
| |
| //runtime·printf("largeAlloc size=%D\n", g->m->scalararg[0]); |
| // Allocate directly from heap. |
| size = g->m->scalararg[0]; |
| flag = (int32)g->m->scalararg[1]; |
| if(size + PageSize < size) |
| runtime·throw("out of memory"); |
| npages = size >> PageShift; |
| if((size & PageMask) != 0) |
| npages++; |
| s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, !(flag & FlagNoZero)); |
| if(s == nil) |
| runtime·throw("out of memory"); |
| s->limit = (byte*)(s->start<<PageShift) + size; |
| v = (void*)(s->start << PageShift); |
| // setup for mark sweep |
| runtime·markspan(v, 0, 0, true); |
| g->m->ptrarg[0] = s; |
| } |