| // Code generated by mkmalloc.go; DO NOT EDIT. |
| // See overview in malloc_stubs.go. |
| |
| package runtime |
| |
| import ( |
| "internal/goarch" |
| "internal/goexperiment" |
| "internal/runtime/sys" |
| "unsafe" |
| ) |
| |
| func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(1<<1) | spanClass(0) |
| const elemsize = uintptr(8) |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 1 |
| const elemsize = 8 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallScanNoHeader(size, typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(0) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| if goarch.PtrSize == 8 && elemsize == 8 { |
| |
| c.scanAlloc += 8 |
| } else { |
| dataSize := size |
| x := uintptr(x) |
| |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(8)) { |
| throw("tried to write heap bits, but no heap bits in span") |
| } |
| |
| src0 := readUintptr(getGCMask(typ)) |
| |
| const elemsize = 8 |
| |
| var scanSize uintptr |
| src := src0 |
| if typ.Size_ == goarch.PtrSize { |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 |
| |
| scanSize = dataSize |
| } else { |
| |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") |
| } |
| scanSize = typ.PtrBytes |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { |
| src |= src0 << (i / goarch.PtrSize) |
| scanSize += typ.Size_ |
| } |
| } |
| |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) |
| dst := unsafe.Pointer(dstBase) |
| o := (x - span.base()) / goarch.PtrSize |
| i := o / ptrBits |
| j := o % ptrBits |
| var bits uintptr = elemsize / goarch.PtrSize |
| |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { |
| |
| bits0 := ptrBits - j |
| bits1 := bits - bits0 |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) |
| } else { |
| |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) |
| } |
| |
| const doubleCheck = false |
| if doubleCheck { |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) |
| } |
| if doubleCheckHeapSetType { |
| doubleCheckHeapType(x, dataSize, typ, nil, span) |
| } |
| c.scanAlloc += scanSize |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(2<<1) | spanClass(0) |
| const elemsize = uintptr(16) |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 2 |
| const elemsize = 16 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallScanNoHeader(size, typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(0) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| if goarch.PtrSize == 8 && elemsize == 8 { |
| |
| c.scanAlloc += 8 |
| } else { |
| dataSize := size |
| x := uintptr(x) |
| |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(16)) { |
| throw("tried to write heap bits, but no heap bits in span") |
| } |
| |
| src0 := readUintptr(getGCMask(typ)) |
| |
| const elemsize = 16 |
| |
| var scanSize uintptr |
| src := src0 |
| if typ.Size_ == goarch.PtrSize { |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 |
| |
| scanSize = dataSize |
| } else { |
| |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") |
| } |
| scanSize = typ.PtrBytes |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { |
| src |= src0 << (i / goarch.PtrSize) |
| scanSize += typ.Size_ |
| } |
| } |
| |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) |
| dst := unsafe.Pointer(dstBase) |
| o := (x - span.base()) / goarch.PtrSize |
| i := o / ptrBits |
| j := o % ptrBits |
| var bits uintptr = elemsize / goarch.PtrSize |
| |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { |
| |
| bits0 := ptrBits - j |
| bits1 := bits - bits0 |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) |
| } else { |
| |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) |
| } |
| |
| const doubleCheck = false |
| if doubleCheck { |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) |
| } |
| if doubleCheckHeapSetType { |
| doubleCheckHeapType(x, dataSize, typ, nil, span) |
| } |
| c.scanAlloc += scanSize |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(3<<1) | spanClass(0) |
| const elemsize = uintptr(24) |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 3 |
| const elemsize = 24 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallScanNoHeader(size, typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(0) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| if goarch.PtrSize == 8 && elemsize == 8 { |
| |
| c.scanAlloc += 8 |
| } else { |
| dataSize := size |
| x := uintptr(x) |
| |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(24)) { |
| throw("tried to write heap bits, but no heap bits in span") |
| } |
| |
| src0 := readUintptr(getGCMask(typ)) |
| |
| const elemsize = 24 |
| |
| var scanSize uintptr |
| src := src0 |
| if typ.Size_ == goarch.PtrSize { |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 |
| |
| scanSize = dataSize |
| } else { |
| |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") |
| } |
| scanSize = typ.PtrBytes |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { |
| src |= src0 << (i / goarch.PtrSize) |
| scanSize += typ.Size_ |
| } |
| } |
| |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) |
| dst := unsafe.Pointer(dstBase) |
| o := (x - span.base()) / goarch.PtrSize |
| i := o / ptrBits |
| j := o % ptrBits |
| var bits uintptr = elemsize / goarch.PtrSize |
| |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { |
| |
| bits0 := ptrBits - j |
| bits1 := bits - bits0 |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) |
| } else { |
| |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) |
| } |
| |
| const doubleCheck = false |
| if doubleCheck { |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) |
| } |
| if doubleCheckHeapSetType { |
| doubleCheckHeapType(x, dataSize, typ, nil, span) |
| } |
| c.scanAlloc += scanSize |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(4<<1) | spanClass(0) |
| const elemsize = uintptr(32) |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 4 |
| const elemsize = 32 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallScanNoHeader(size, typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(0) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| if goarch.PtrSize == 8 && elemsize == 8 { |
| |
| c.scanAlloc += 8 |
| } else { |
| dataSize := size |
| x := uintptr(x) |
| |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(32)) { |
| throw("tried to write heap bits, but no heap bits in span") |
| } |
| |
| src0 := readUintptr(getGCMask(typ)) |
| |
| const elemsize = 32 |
| |
| var scanSize uintptr |
| src := src0 |
| if typ.Size_ == goarch.PtrSize { |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 |
| |
| scanSize = dataSize |
| } else { |
| |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") |
| } |
| scanSize = typ.PtrBytes |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { |
| src |= src0 << (i / goarch.PtrSize) |
| scanSize += typ.Size_ |
| } |
| } |
| |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) |
| dst := unsafe.Pointer(dstBase) |
| o := (x - span.base()) / goarch.PtrSize |
| i := o / ptrBits |
| j := o % ptrBits |
| var bits uintptr = elemsize / goarch.PtrSize |
| |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { |
| |
| bits0 := ptrBits - j |
| bits1 := bits - bits0 |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) |
| } else { |
| |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) |
| } |
| |
| const doubleCheck = false |
| if doubleCheck { |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) |
| } |
| if doubleCheckHeapSetType { |
| doubleCheckHeapType(x, dataSize, typ, nil, span) |
| } |
| c.scanAlloc += scanSize |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(5<<1) | spanClass(0) |
| const elemsize = uintptr(48) |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 5 |
| const elemsize = 48 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallScanNoHeader(size, typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(0) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| if goarch.PtrSize == 8 && elemsize == 8 { |
| |
| c.scanAlloc += 8 |
| } else { |
| dataSize := size |
| x := uintptr(x) |
| |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(48)) { |
| throw("tried to write heap bits, but no heap bits in span") |
| } |
| |
| src0 := readUintptr(getGCMask(typ)) |
| |
| const elemsize = 48 |
| |
| var scanSize uintptr |
| src := src0 |
| if typ.Size_ == goarch.PtrSize { |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 |
| |
| scanSize = dataSize |
| } else { |
| |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") |
| } |
| scanSize = typ.PtrBytes |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { |
| src |= src0 << (i / goarch.PtrSize) |
| scanSize += typ.Size_ |
| } |
| } |
| |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) |
| dst := unsafe.Pointer(dstBase) |
| o := (x - span.base()) / goarch.PtrSize |
| i := o / ptrBits |
| j := o % ptrBits |
| var bits uintptr = elemsize / goarch.PtrSize |
| |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { |
| |
| bits0 := ptrBits - j |
| bits1 := bits - bits0 |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) |
| } else { |
| |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) |
| } |
| |
| const doubleCheck = false |
| if doubleCheck { |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) |
| } |
| if doubleCheckHeapSetType { |
| doubleCheckHeapType(x, dataSize, typ, nil, span) |
| } |
| c.scanAlloc += scanSize |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(6<<1) | spanClass(0) |
| const elemsize = uintptr(64) |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 6 |
| const elemsize = 64 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallScanNoHeader(size, typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(0) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| if goarch.PtrSize == 8 && elemsize == 8 { |
| |
| c.scanAlloc += 8 |
| } else { |
| dataSize := size |
| x := uintptr(x) |
| |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(64)) { |
| throw("tried to write heap bits, but no heap bits in span") |
| } |
| |
| src0 := readUintptr(getGCMask(typ)) |
| |
| const elemsize = 64 |
| |
| var scanSize uintptr |
| src := src0 |
| if typ.Size_ == goarch.PtrSize { |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 |
| |
| scanSize = dataSize |
| } else { |
| |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") |
| } |
| scanSize = typ.PtrBytes |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { |
| src |= src0 << (i / goarch.PtrSize) |
| scanSize += typ.Size_ |
| } |
| } |
| |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) |
| dst := unsafe.Pointer(dstBase) |
| o := (x - span.base()) / goarch.PtrSize |
| i := o / ptrBits |
| j := o % ptrBits |
| var bits uintptr = elemsize / goarch.PtrSize |
| |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { |
| |
| bits0 := ptrBits - j |
| bits1 := bits - bits0 |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) |
| } else { |
| |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) |
| } |
| |
| const doubleCheck = false |
| if doubleCheck { |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) |
| } |
| if doubleCheckHeapSetType { |
| doubleCheckHeapType(x, dataSize, typ, nil, span) |
| } |
| c.scanAlloc += scanSize |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(7<<1) | spanClass(0) |
| const elemsize = uintptr(80) |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 7 |
| const elemsize = 80 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallScanNoHeader(size, typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(0) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| if goarch.PtrSize == 8 && elemsize == 8 { |
| |
| c.scanAlloc += 8 |
| } else { |
| dataSize := size |
| x := uintptr(x) |
| |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(80)) { |
| throw("tried to write heap bits, but no heap bits in span") |
| } |
| |
| src0 := readUintptr(getGCMask(typ)) |
| |
| const elemsize = 80 |
| |
| var scanSize uintptr |
| src := src0 |
| if typ.Size_ == goarch.PtrSize { |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 |
| |
| scanSize = dataSize |
| } else { |
| |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") |
| } |
| scanSize = typ.PtrBytes |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { |
| src |= src0 << (i / goarch.PtrSize) |
| scanSize += typ.Size_ |
| } |
| } |
| |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) |
| dst := unsafe.Pointer(dstBase) |
| o := (x - span.base()) / goarch.PtrSize |
| i := o / ptrBits |
| j := o % ptrBits |
| var bits uintptr = elemsize / goarch.PtrSize |
| |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { |
| |
| bits0 := ptrBits - j |
| bits1 := bits - bits0 |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) |
| } else { |
| |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) |
| } |
| |
| const doubleCheck = false |
| if doubleCheck { |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) |
| } |
| if doubleCheckHeapSetType { |
| doubleCheckHeapType(x, dataSize, typ, nil, span) |
| } |
| c.scanAlloc += scanSize |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| return mallocgcTinySlowPath(size, typ, needzero) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const elemsize = 16 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| doubleCheckTiny(size, typ, mp) |
| } |
| mp.mallocing = 1 |
| |
| c := getMCache(mp) |
| off := c.tinyoffset |
| |
| if size&7 == 0 { |
| off = alignUp(off, 8) |
| } else if goarch.PtrSize == 4 && size == 12 { |
| |
| off = alignUp(off, 8) |
| } else if size&3 == 0 { |
| off = alignUp(off, 4) |
| } else if size&1 == 0 { |
| off = alignUp(off, 2) |
| } |
| if off+size <= maxTinySize && c.tiny != 0 { |
| |
| x := unsafe.Pointer(c.tiny + off) |
| c.tinyoffset = off + size |
| c.tinyAllocs++ |
| mp.mallocing = 0 |
| releasem(mp) |
| const elemsize = 0 |
| |
| return x |
| } |
| |
| checkGCTrigger := false |
| span := c.alloc[tinySpanClass] |
| |
| const nbytes = 8192 |
| const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16) |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base()) |
| } |
| } |
| } |
| v := nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(tinySpanClass) |
| } |
| x := unsafe.Pointer(v) |
| (*[2]uint64)(x)[0] = 0 |
| (*[2]uint64)(x)[1] = 0 |
| |
| if !raceenabled && (size < c.tinyoffset || c.tiny == 0) { |
| |
| c.tiny = uintptr(x) |
| c.tinyoffset = size |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(2<<1) | spanClass(1) |
| const elemsize = uintptr(16) |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 2 |
| const elemsize = 16 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallNoScan(typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(1) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { |
| |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| return x |
| |
| } |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if needzero && span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(3<<1) | spanClass(1) |
| const elemsize = uintptr(24) |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 3 |
| const elemsize = 24 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallNoScan(typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(1) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { |
| |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| return x |
| |
| } |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if needzero && span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(4<<1) | spanClass(1) |
| const elemsize = uintptr(32) |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 4 |
| const elemsize = 32 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallNoScan(typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(1) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { |
| |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| return x |
| |
| } |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if needzero && span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(5<<1) | spanClass(1) |
| const elemsize = uintptr(48) |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 5 |
| const elemsize = 48 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallNoScan(typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(1) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { |
| |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| return x |
| |
| } |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if needzero && span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(6<<1) | spanClass(1) |
| const elemsize = uintptr(64) |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 6 |
| const elemsize = 64 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallNoScan(typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(1) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { |
| |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| return x |
| |
| } |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if needzero && span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) |
| |
| if forceSlowPath { |
| |
| const spc = spanClass(7<<1) | spanClass(1) |
| const elemsize = uintptr(80) |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) |
| |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| const sizeclass = 7 |
| const elemsize = 80 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallNoScan(typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| const spc = spanClass(sizeclass<<1) | spanClass(1) |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { |
| |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| return x |
| |
| } |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if needzero && span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| |
| publicationBarrier() |
| |
| span.freeIndexForScan = span.freeindex |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| return x |
| } |
| |
| func mallocgcTinySlowPath(size uintptr, typ *_type, needzero bool) unsafe.Pointer { |
| |
| gp := getg() |
| if goexperiment.RuntimeSecret && gp.secret > 0 { |
| return mallocgcSmallNoScanSC2(size, typ, needzero) |
| } |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| if debug.malloc { |
| if x := preMallocgcDebug(size, typ); x != nil { |
| return x |
| } |
| } |
| |
| if gcBlackenEnabled != 0 { |
| |
| assistG := getg() |
| if assistG.m.curg != nil { |
| assistG = assistG.m.curg |
| } |
| assistG.gcAssistBytes -= int64(size) |
| if assistG.gcAssistBytes < 0 { |
| gcAssistAlloc(assistG) |
| } |
| } |
| |
| const elemsize = 16 |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| doubleCheckTiny(size, typ, mp) |
| } |
| mp.mallocing = 1 |
| |
| c := getMCache(mp) |
| off := c.tinyoffset |
| |
| if size&7 == 0 { |
| off = alignUp(off, 8) |
| } else if goarch.PtrSize == 4 && size == 12 { |
| |
| off = alignUp(off, 8) |
| } else if size&3 == 0 { |
| off = alignUp(off, 4) |
| } else if size&1 == 0 { |
| off = alignUp(off, 2) |
| } |
| if off+size <= maxTinySize && c.tiny != 0 { |
| |
| x := unsafe.Pointer(c.tiny + off) |
| c.tinyoffset = off + size |
| c.tinyAllocs++ |
| mp.mallocing = 0 |
| releasem(mp) |
| const elemsize = 0 |
| if gcBlackenEnabled != 0 && elemsize != 0 { |
| if assistG := getg().m.curg; assistG != nil { |
| assistG.gcAssistBytes -= int64(elemsize - size) |
| } |
| } |
| |
| if debug.malloc { |
| postMallocgcDebug(x, elemsize, typ) |
| } |
| |
| return x |
| } |
| |
| checkGCTrigger := false |
| span := c.alloc[tinySpanClass] |
| |
| const nbytes = 8192 |
| const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16) |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base()) |
| } |
| } |
| } |
| v := nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(tinySpanClass) |
| } |
| x := unsafe.Pointer(v) |
| (*[2]uint64)(x)[0] = 0 |
| (*[2]uint64)(x)[1] = 0 |
| |
| if !raceenabled && (size < c.tinyoffset || c.tiny == 0) { |
| |
| c.tiny = uintptr(x) |
| c.tinyoffset = size |
| } |
| |
| publicationBarrier() |
| |
| if writeBarrier.enabled { |
| |
| gcmarknewobject(span, uintptr(x)) |
| } else { |
| |
| span.freeIndexForScan = span.freeindex |
| } |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| if gcBlackenEnabled != 0 && elemsize != 0 { |
| if assistG := getg().m.curg; assistG != nil { |
| assistG.gcAssistBytes -= int64(elemsize - size) |
| } |
| } |
| |
| if debug.malloc { |
| postMallocgcDebug(x, elemsize, typ) |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallScanSlowPath(size uintptr, typ *_type, needzero bool, spc spanClass, elemsize uintptr) unsafe.Pointer { |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| if debug.malloc { |
| if x := preMallocgcDebug(size, typ); x != nil { |
| return x |
| } |
| } |
| |
| if gcBlackenEnabled != 0 { |
| |
| assistG := getg() |
| if assistG.m.curg != nil { |
| assistG = assistG.m.curg |
| } |
| assistG.gcAssistBytes -= int64(size) |
| if assistG.gcAssistBytes < 0 { |
| gcAssistAlloc(assistG) |
| } |
| } |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallScanNoHeader(size, typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| if goarch.PtrSize == 8 && elemsize == 8 { |
| |
| c.scanAlloc += 8 |
| } else { |
| dataSize := size |
| x := uintptr(x) |
| |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(elemsize_)) { |
| throw("tried to write heap bits, but no heap bits in span") |
| } |
| |
| src0 := readUintptr(getGCMask(typ)) |
| |
| var scanSize uintptr |
| src := src0 |
| if typ.Size_ == goarch.PtrSize { |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 |
| |
| scanSize = dataSize |
| } else { |
| |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") |
| } |
| scanSize = typ.PtrBytes |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { |
| src |= src0 << (i / goarch.PtrSize) |
| scanSize += typ.Size_ |
| } |
| } |
| |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) |
| dst := unsafe.Pointer(dstBase) |
| o := (x - span.base()) / goarch.PtrSize |
| i := o / ptrBits |
| j := o % ptrBits |
| var bits uintptr = elemsize / goarch.PtrSize |
| |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { |
| |
| bits0 := ptrBits - j |
| bits1 := bits - bits0 |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) |
| } else { |
| |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) |
| } |
| |
| const doubleCheck = false |
| if doubleCheck { |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) |
| } |
| if doubleCheckHeapSetType { |
| doubleCheckHeapType(x, dataSize, typ, nil, span) |
| } |
| c.scanAlloc += scanSize |
| } |
| |
| publicationBarrier() |
| |
| if writeBarrier.enabled { |
| |
| gcmarknewobject(span, uintptr(x)) |
| } else { |
| |
| span.freeIndexForScan = span.freeindex |
| } |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| gp := getg() |
| if goexperiment.RuntimeSecret && gp.secret > 0 { |
| |
| addSecret(x, size) |
| } |
| |
| if gcBlackenEnabled != 0 && elemsize != 0 { |
| if assistG := getg().m.curg; assistG != nil { |
| assistG.gcAssistBytes -= int64(elemsize - size) |
| } |
| } |
| |
| if debug.malloc { |
| postMallocgcDebug(x, elemsize, typ) |
| } |
| |
| return x |
| } |
| |
| func mallocgcSmallNoScanSlowPath(size uintptr, typ *_type, needzero bool, spc spanClass, elemsize uintptr) unsafe.Pointer { |
| |
| if doubleCheckMalloc { |
| if gcphase == _GCmarktermination { |
| throw("mallocgc called with gcphase == _GCmarktermination") |
| } |
| } |
| |
| lockRankMayQueueFinalizer() |
| |
| if debug.malloc { |
| if x := preMallocgcDebug(size, typ); x != nil { |
| return x |
| } |
| } |
| |
| if gcBlackenEnabled != 0 { |
| |
| assistG := getg() |
| if assistG.m.curg != nil { |
| assistG = assistG.m.curg |
| } |
| assistG.gcAssistBytes -= int64(size) |
| if assistG.gcAssistBytes < 0 { |
| gcAssistAlloc(assistG) |
| } |
| } |
| |
| mp := acquirem() |
| if doubleCheckMalloc { |
| |
| doubleCheckSmallNoScan(typ, mp) |
| |
| } |
| mp.mallocing = 1 |
| |
| checkGCTrigger := false |
| c := getMCache(mp) |
| |
| span := c.alloc[spc] |
| |
| var v gclinkptr |
| var x unsafe.Pointer |
| |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { |
| |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| goto post |
| |
| } |
| |
| { |
| |
| var nextFreeFastResult gclinkptr |
| if span.allocCache != 0 { |
| theBit := sys.TrailingZeros64(span.allocCache) |
| result := span.freeindex + uint16(theBit) |
| if result < span.nelems { |
| freeidx := result + 1 |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { |
| span.allocCache >>= uint(theBit + 1) |
| span.freeindex = freeidx |
| span.allocCount++ |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) |
| } |
| } |
| } |
| v = nextFreeFastResult |
| if v == 0 { |
| v, span, checkGCTrigger = c.nextFree(spc) |
| } |
| x = unsafe.Pointer(v) |
| } |
| |
| if needzero && span.needzero != 0 { |
| memclrNoHeapPointers(x, elemsize) |
| } |
| |
| publicationBarrier() |
| |
| if writeBarrier.enabled { |
| |
| gcmarknewobject(span, uintptr(x)) |
| } else { |
| |
| span.freeIndexForScan = span.freeindex |
| } |
| |
| c.nextSample -= int64(elemsize) |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { |
| profilealloc(mp, x, elemsize) |
| } |
| mp.mallocing = 0 |
| releasem(mp) |
| |
| if checkGCTrigger { |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { |
| gcStart(t) |
| } |
| } |
| |
| post: |
| gp := getg() |
| if goexperiment.RuntimeSecret && gp.secret > 0 { |
| |
| addSecret(x, size) |
| } |
| |
| if gcBlackenEnabled != 0 && elemsize != 0 { |
| if assistG := getg().m.curg; assistG != nil { |
| assistG.gcAssistBytes -= int64(elemsize - size) |
| } |
| } |
| |
| if debug.malloc { |
| postMallocgcDebug(x, elemsize, typ) |
| } |
| |
| return x |
| } |
