| // Code generated by mkmalloc.go; DO NOT EDIT. | |
| // See overview in malloc_stubs.go. | |
| package runtime | |
| import ( | |
| "internal/goarch" | |
| "internal/goexperiment" | |
| "internal/runtime/sys" | |
| "unsafe" | |
| ) | |
| func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(1<<1) | spanClass(0) | |
| const elemsize = uintptr(8) | |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 1 | |
| const elemsize = 8 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallScanNoHeader(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(0) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| if goarch.PtrSize == 8 && elemsize == 8 { | |
| c.scanAlloc += 8 | |
| } else { | |
| dataSize := size | |
| x := uintptr(x) | |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(8)) { | |
| throw("tried to write heap bits, but no heap bits in span") | |
| } | |
| src0 := readUintptr(getGCMask(typ)) | |
| const elemsize = 8 | |
| var scanSize uintptr | |
| src := src0 | |
| if typ.Size_ == goarch.PtrSize { | |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 | |
| scanSize = dataSize | |
| } else { | |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { | |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") | |
| } | |
| scanSize = typ.PtrBytes | |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { | |
| src |= src0 << (i / goarch.PtrSize) | |
| scanSize += typ.Size_ | |
| } | |
| } | |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) | |
| dst := unsafe.Pointer(dstBase) | |
| o := (x - span.base()) / goarch.PtrSize | |
| i := o / ptrBits | |
| j := o % ptrBits | |
| var bits uintptr = elemsize / goarch.PtrSize | |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 | |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { | |
| bits0 := ptrBits - j | |
| bits1 := bits - bits0 | |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) | |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) | |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) | |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) | |
| } else { | |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) | |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) | |
| } | |
| const doubleCheck = false | |
| if doubleCheck { | |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) | |
| } | |
| if doubleCheckHeapSetType { | |
| doubleCheckHeapType(x, dataSize, typ, nil, span) | |
| } | |
| c.scanAlloc += scanSize | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(2<<1) | spanClass(0) | |
| const elemsize = uintptr(16) | |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 2 | |
| const elemsize = 16 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallScanNoHeader(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(0) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| if goarch.PtrSize == 8 && elemsize == 8 { | |
| c.scanAlloc += 8 | |
| } else { | |
| dataSize := size | |
| x := uintptr(x) | |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(16)) { | |
| throw("tried to write heap bits, but no heap bits in span") | |
| } | |
| src0 := readUintptr(getGCMask(typ)) | |
| const elemsize = 16 | |
| var scanSize uintptr | |
| src := src0 | |
| if typ.Size_ == goarch.PtrSize { | |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 | |
| scanSize = dataSize | |
| } else { | |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { | |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") | |
| } | |
| scanSize = typ.PtrBytes | |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { | |
| src |= src0 << (i / goarch.PtrSize) | |
| scanSize += typ.Size_ | |
| } | |
| } | |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) | |
| dst := unsafe.Pointer(dstBase) | |
| o := (x - span.base()) / goarch.PtrSize | |
| i := o / ptrBits | |
| j := o % ptrBits | |
| var bits uintptr = elemsize / goarch.PtrSize | |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 | |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { | |
| bits0 := ptrBits - j | |
| bits1 := bits - bits0 | |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) | |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) | |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) | |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) | |
| } else { | |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) | |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) | |
| } | |
| const doubleCheck = false | |
| if doubleCheck { | |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) | |
| } | |
| if doubleCheckHeapSetType { | |
| doubleCheckHeapType(x, dataSize, typ, nil, span) | |
| } | |
| c.scanAlloc += scanSize | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(3<<1) | spanClass(0) | |
| const elemsize = uintptr(24) | |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 3 | |
| const elemsize = 24 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallScanNoHeader(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(0) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| if goarch.PtrSize == 8 && elemsize == 8 { | |
| c.scanAlloc += 8 | |
| } else { | |
| dataSize := size | |
| x := uintptr(x) | |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(24)) { | |
| throw("tried to write heap bits, but no heap bits in span") | |
| } | |
| src0 := readUintptr(getGCMask(typ)) | |
| const elemsize = 24 | |
| var scanSize uintptr | |
| src := src0 | |
| if typ.Size_ == goarch.PtrSize { | |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 | |
| scanSize = dataSize | |
| } else { | |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { | |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") | |
| } | |
| scanSize = typ.PtrBytes | |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { | |
| src |= src0 << (i / goarch.PtrSize) | |
| scanSize += typ.Size_ | |
| } | |
| } | |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) | |
| dst := unsafe.Pointer(dstBase) | |
| o := (x - span.base()) / goarch.PtrSize | |
| i := o / ptrBits | |
| j := o % ptrBits | |
| var bits uintptr = elemsize / goarch.PtrSize | |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 | |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { | |
| bits0 := ptrBits - j | |
| bits1 := bits - bits0 | |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) | |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) | |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) | |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) | |
| } else { | |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) | |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) | |
| } | |
| const doubleCheck = false | |
| if doubleCheck { | |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) | |
| } | |
| if doubleCheckHeapSetType { | |
| doubleCheckHeapType(x, dataSize, typ, nil, span) | |
| } | |
| c.scanAlloc += scanSize | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(4<<1) | spanClass(0) | |
| const elemsize = uintptr(32) | |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 4 | |
| const elemsize = 32 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallScanNoHeader(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(0) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| if goarch.PtrSize == 8 && elemsize == 8 { | |
| c.scanAlloc += 8 | |
| } else { | |
| dataSize := size | |
| x := uintptr(x) | |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(32)) { | |
| throw("tried to write heap bits, but no heap bits in span") | |
| } | |
| src0 := readUintptr(getGCMask(typ)) | |
| const elemsize = 32 | |
| var scanSize uintptr | |
| src := src0 | |
| if typ.Size_ == goarch.PtrSize { | |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 | |
| scanSize = dataSize | |
| } else { | |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { | |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") | |
| } | |
| scanSize = typ.PtrBytes | |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { | |
| src |= src0 << (i / goarch.PtrSize) | |
| scanSize += typ.Size_ | |
| } | |
| } | |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) | |
| dst := unsafe.Pointer(dstBase) | |
| o := (x - span.base()) / goarch.PtrSize | |
| i := o / ptrBits | |
| j := o % ptrBits | |
| var bits uintptr = elemsize / goarch.PtrSize | |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 | |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { | |
| bits0 := ptrBits - j | |
| bits1 := bits - bits0 | |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) | |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) | |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) | |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) | |
| } else { | |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) | |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) | |
| } | |
| const doubleCheck = false | |
| if doubleCheck { | |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) | |
| } | |
| if doubleCheckHeapSetType { | |
| doubleCheckHeapType(x, dataSize, typ, nil, span) | |
| } | |
| c.scanAlloc += scanSize | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(5<<1) | spanClass(0) | |
| const elemsize = uintptr(48) | |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 5 | |
| const elemsize = 48 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallScanNoHeader(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(0) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| if goarch.PtrSize == 8 && elemsize == 8 { | |
| c.scanAlloc += 8 | |
| } else { | |
| dataSize := size | |
| x := uintptr(x) | |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(48)) { | |
| throw("tried to write heap bits, but no heap bits in span") | |
| } | |
| src0 := readUintptr(getGCMask(typ)) | |
| const elemsize = 48 | |
| var scanSize uintptr | |
| src := src0 | |
| if typ.Size_ == goarch.PtrSize { | |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 | |
| scanSize = dataSize | |
| } else { | |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { | |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") | |
| } | |
| scanSize = typ.PtrBytes | |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { | |
| src |= src0 << (i / goarch.PtrSize) | |
| scanSize += typ.Size_ | |
| } | |
| } | |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) | |
| dst := unsafe.Pointer(dstBase) | |
| o := (x - span.base()) / goarch.PtrSize | |
| i := o / ptrBits | |
| j := o % ptrBits | |
| var bits uintptr = elemsize / goarch.PtrSize | |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 | |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { | |
| bits0 := ptrBits - j | |
| bits1 := bits - bits0 | |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) | |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) | |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) | |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) | |
| } else { | |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) | |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) | |
| } | |
| const doubleCheck = false | |
| if doubleCheck { | |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) | |
| } | |
| if doubleCheckHeapSetType { | |
| doubleCheckHeapType(x, dataSize, typ, nil, span) | |
| } | |
| c.scanAlloc += scanSize | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(6<<1) | spanClass(0) | |
| const elemsize = uintptr(64) | |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 6 | |
| const elemsize = 64 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallScanNoHeader(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(0) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| if goarch.PtrSize == 8 && elemsize == 8 { | |
| c.scanAlloc += 8 | |
| } else { | |
| dataSize := size | |
| x := uintptr(x) | |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(64)) { | |
| throw("tried to write heap bits, but no heap bits in span") | |
| } | |
| src0 := readUintptr(getGCMask(typ)) | |
| const elemsize = 64 | |
| var scanSize uintptr | |
| src := src0 | |
| if typ.Size_ == goarch.PtrSize { | |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 | |
| scanSize = dataSize | |
| } else { | |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { | |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") | |
| } | |
| scanSize = typ.PtrBytes | |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { | |
| src |= src0 << (i / goarch.PtrSize) | |
| scanSize += typ.Size_ | |
| } | |
| } | |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) | |
| dst := unsafe.Pointer(dstBase) | |
| o := (x - span.base()) / goarch.PtrSize | |
| i := o / ptrBits | |
| j := o % ptrBits | |
| var bits uintptr = elemsize / goarch.PtrSize | |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 | |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { | |
| bits0 := ptrBits - j | |
| bits1 := bits - bits0 | |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) | |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) | |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) | |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) | |
| } else { | |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) | |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) | |
| } | |
| const doubleCheck = false | |
| if doubleCheck { | |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) | |
| } | |
| if doubleCheckHeapSetType { | |
| doubleCheckHeapType(x, dataSize, typ, nil, span) | |
| } | |
| c.scanAlloc += scanSize | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(7<<1) | spanClass(0) | |
| const elemsize = uintptr(80) | |
| return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 7 | |
| const elemsize = 80 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallScanNoHeader(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(0) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| if goarch.PtrSize == 8 && elemsize == 8 { | |
| c.scanAlloc += 8 | |
| } else { | |
| dataSize := size | |
| x := uintptr(x) | |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(80)) { | |
| throw("tried to write heap bits, but no heap bits in span") | |
| } | |
| src0 := readUintptr(getGCMask(typ)) | |
| const elemsize = 80 | |
| var scanSize uintptr | |
| src := src0 | |
| if typ.Size_ == goarch.PtrSize { | |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 | |
| scanSize = dataSize | |
| } else { | |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { | |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") | |
| } | |
| scanSize = typ.PtrBytes | |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { | |
| src |= src0 << (i / goarch.PtrSize) | |
| scanSize += typ.Size_ | |
| } | |
| } | |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) | |
| dst := unsafe.Pointer(dstBase) | |
| o := (x - span.base()) / goarch.PtrSize | |
| i := o / ptrBits | |
| j := o % ptrBits | |
| var bits uintptr = elemsize / goarch.PtrSize | |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 | |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { | |
| bits0 := ptrBits - j | |
| bits1 := bits - bits0 | |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) | |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) | |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) | |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) | |
| } else { | |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) | |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) | |
| } | |
| const doubleCheck = false | |
| if doubleCheck { | |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) | |
| } | |
| if doubleCheckHeapSetType { | |
| doubleCheckHeapType(x, dataSize, typ, nil, span) | |
| } | |
| c.scanAlloc += scanSize | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| return mallocgcTinySlowPath(size, typ, needzero) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const elemsize = 16 | |
| if doubleCheckMalloc { | |
| doubleCheckTiny(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| c := getMCache(mp) | |
| off := c.tinyoffset | |
| if size&7 == 0 { | |
| off = alignUp(off, 8) | |
| } else if goarch.PtrSize == 4 && size == 12 { | |
| off = alignUp(off, 8) | |
| } else if size&3 == 0 { | |
| off = alignUp(off, 4) | |
| } else if size&1 == 0 { | |
| off = alignUp(off, 2) | |
| } | |
| if off+size <= maxTinySize && c.tiny != 0 { | |
| x := unsafe.Pointer(c.tiny + off) | |
| c.tinyoffset = off + size | |
| c.tinyAllocs++ | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| const elemsize = 0 | |
| return x | |
| } | |
| checkGCTrigger := false | |
| span := c.alloc[tinySpanClass] | |
| const nbytes = 8192 | |
| const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16) | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base()) | |
| } | |
| } | |
| } | |
| v := nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(tinySpanClass) | |
| } | |
| x := unsafe.Pointer(v) | |
| (*[2]uint64)(x)[0] = 0 | |
| (*[2]uint64)(x)[1] = 0 | |
| if !raceenabled && (size < c.tinyoffset || c.tiny == 0) { | |
| c.tiny = uintptr(x) | |
| c.tinyoffset = size | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(2<<1) | spanClass(1) | |
| const elemsize = uintptr(16) | |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 2 | |
| const elemsize = 16 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallNoScan(typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(1) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { | |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| return x | |
| } | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if needzero && span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(3<<1) | spanClass(1) | |
| const elemsize = uintptr(24) | |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 3 | |
| const elemsize = 24 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallNoScan(typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(1) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { | |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| return x | |
| } | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if needzero && span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(4<<1) | spanClass(1) | |
| const elemsize = uintptr(32) | |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 4 | |
| const elemsize = 32 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallNoScan(typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(1) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { | |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| return x | |
| } | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if needzero && span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(5<<1) | spanClass(1) | |
| const elemsize = uintptr(48) | |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 5 | |
| const elemsize = 48 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallNoScan(typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(1) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { | |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| return x | |
| } | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if needzero && span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(6<<1) | spanClass(1) | |
| const elemsize = uintptr(64) | |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 6 | |
| const elemsize = 64 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallNoScan(typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(1) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { | |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| return x | |
| } | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if needzero && span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| mp = acquirem() | |
| forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) | |
| if forceSlowPath { | |
| releasem(mp) | |
| const spc = spanClass(7<<1) | spanClass(1) | |
| const elemsize = uintptr(80) | |
| return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize) | |
| } | |
| lockRankMayQueueFinalizer() | |
| const sizeclass = 7 | |
| const elemsize = 80 | |
| if doubleCheckMalloc { | |
| doubleCheckSmallNoScan(typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| const spc = spanClass(sizeclass<<1) | spanClass(1) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { | |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| return x | |
| } | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if needzero && span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| publicationBarrier() | |
| span.freeIndexForScan = span.freeindex | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| return x | |
| } | |
| func mallocgcTinySlowPath(size uintptr, typ *_type, needzero bool) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| gp := getg() | |
| if goexperiment.RuntimeSecret && gp.secret > 0 { | |
| return mallocgcSmallNoScanSC2(size, typ, needzero) | |
| } | |
| lockRankMayQueueFinalizer() | |
| if debug.malloc { | |
| if x := preMallocgcDebug(size, typ); x != nil { | |
| return x | |
| } | |
| } | |
| if gcBlackenEnabled != 0 { | |
| assistG := getg() | |
| if assistG.m.curg != nil { | |
| assistG = assistG.m.curg | |
| } | |
| assistG.gcAssistBytes -= int64(size) | |
| if assistG.gcAssistBytes < 0 { | |
| gcAssistAlloc(assistG) | |
| } | |
| } | |
| mp = acquirem() | |
| const elemsize = 16 | |
| if doubleCheckMalloc { | |
| doubleCheckTiny(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| c := getMCache(mp) | |
| off := c.tinyoffset | |
| if size&7 == 0 { | |
| off = alignUp(off, 8) | |
| } else if goarch.PtrSize == 4 && size == 12 { | |
| off = alignUp(off, 8) | |
| } else if size&3 == 0 { | |
| off = alignUp(off, 4) | |
| } else if size&1 == 0 { | |
| off = alignUp(off, 2) | |
| } | |
| if off+size <= maxTinySize && c.tiny != 0 { | |
| x := unsafe.Pointer(c.tiny + off) | |
| c.tinyoffset = off + size | |
| c.tinyAllocs++ | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| const elemsize = 0 | |
| if gcBlackenEnabled != 0 && elemsize != 0 { | |
| if assistG := getg().m.curg; assistG != nil { | |
| assistG.gcAssistBytes -= int64(elemsize - size) | |
| } | |
| } | |
| if debug.malloc { | |
| postMallocgcDebug(x, elemsize, typ) | |
| } | |
| return x | |
| } | |
| checkGCTrigger := false | |
| span := c.alloc[tinySpanClass] | |
| const nbytes = 8192 | |
| const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16) | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base()) | |
| } | |
| } | |
| } | |
| v := nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(tinySpanClass) | |
| } | |
| x := unsafe.Pointer(v) | |
| (*[2]uint64)(x)[0] = 0 | |
| (*[2]uint64)(x)[1] = 0 | |
| if !raceenabled && (size < c.tinyoffset || c.tiny == 0) { | |
| c.tiny = uintptr(x) | |
| c.tinyoffset = size | |
| } | |
| publicationBarrier() | |
| if writeBarrier.enabled { | |
| gcmarknewobject(span, uintptr(x)) | |
| } else { | |
| span.freeIndexForScan = span.freeindex | |
| } | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| if gcBlackenEnabled != 0 && elemsize != 0 { | |
| if assistG := getg().m.curg; assistG != nil { | |
| assistG.gcAssistBytes -= int64(elemsize - size) | |
| } | |
| } | |
| if debug.malloc { | |
| postMallocgcDebug(x, elemsize, typ) | |
| } | |
| return x | |
| } | |
| func mallocgcSmallScanSlowPath(size uintptr, typ *_type, needzero bool, spc spanClass, elemsize uintptr) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| lockRankMayQueueFinalizer() | |
| if debug.malloc { | |
| if x := preMallocgcDebug(size, typ); x != nil { | |
| return x | |
| } | |
| } | |
| if gcBlackenEnabled != 0 { | |
| assistG := getg() | |
| if assistG.m.curg != nil { | |
| assistG = assistG.m.curg | |
| } | |
| assistG.gcAssistBytes -= int64(size) | |
| if assistG.gcAssistBytes < 0 { | |
| gcAssistAlloc(assistG) | |
| } | |
| } | |
| mp = acquirem() | |
| if doubleCheckMalloc { | |
| doubleCheckSmallScanNoHeader(size, typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| if goarch.PtrSize == 8 && elemsize == 8 { | |
| c.scanAlloc += 8 | |
| } else { | |
| dataSize := size | |
| x := uintptr(x) | |
| if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(elemsize_)) { | |
| throw("tried to write heap bits, but no heap bits in span") | |
| } | |
| src0 := readUintptr(getGCMask(typ)) | |
| var scanSize uintptr | |
| src := src0 | |
| if typ.Size_ == goarch.PtrSize { | |
| src = (1 << (dataSize / goarch.PtrSize)) - 1 | |
| scanSize = dataSize | |
| } else { | |
| if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { | |
| throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") | |
| } | |
| scanSize = typ.PtrBytes | |
| for i := typ.Size_; i < dataSize; i += typ.Size_ { | |
| src |= src0 << (i / goarch.PtrSize) | |
| scanSize += typ.Size_ | |
| } | |
| } | |
| dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) | |
| dst := unsafe.Pointer(dstBase) | |
| o := (x - span.base()) / goarch.PtrSize | |
| i := o / ptrBits | |
| j := o % ptrBits | |
| var bits uintptr = elemsize / goarch.PtrSize | |
| var bitsIsPowerOfTwo = bits&(bits-1) == 0 | |
| if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { | |
| bits0 := ptrBits - j | |
| bits1 := bits - bits0 | |
| dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) | |
| dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) | |
| *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) | |
| *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) | |
| } else { | |
| dst := (*uintptr)(add(dst, i*goarch.PtrSize)) | |
| *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) | |
| } | |
| const doubleCheck = false | |
| if doubleCheck { | |
| writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) | |
| } | |
| if doubleCheckHeapSetType { | |
| doubleCheckHeapType(x, dataSize, typ, nil, span) | |
| } | |
| c.scanAlloc += scanSize | |
| } | |
| publicationBarrier() | |
| if writeBarrier.enabled { | |
| gcmarknewobject(span, uintptr(x)) | |
| } else { | |
| span.freeIndexForScan = span.freeindex | |
| } | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| gp := getg() | |
| if goexperiment.RuntimeSecret && gp.secret > 0 { | |
| addSecret(x, size) | |
| } | |
| if gcBlackenEnabled != 0 && elemsize != 0 { | |
| if assistG := getg().m.curg; assistG != nil { | |
| assistG.gcAssistBytes -= int64(elemsize - size) | |
| } | |
| } | |
| if debug.malloc { | |
| postMallocgcDebug(x, elemsize, typ) | |
| } | |
| return x | |
| } | |
| func mallocgcSmallNoScanSlowPath(size uintptr, typ *_type, needzero bool, spc spanClass, elemsize uintptr) unsafe.Pointer { | |
| if doubleCheckMalloc { | |
| if gcphase == _GCmarktermination { | |
| throw("mallocgc called with gcphase == _GCmarktermination") | |
| } | |
| } | |
| var mp *m | |
| lockRankMayQueueFinalizer() | |
| if debug.malloc { | |
| if x := preMallocgcDebug(size, typ); x != nil { | |
| return x | |
| } | |
| } | |
| if gcBlackenEnabled != 0 { | |
| assistG := getg() | |
| if assistG.m.curg != nil { | |
| assistG = assistG.m.curg | |
| } | |
| assistG.gcAssistBytes -= int64(size) | |
| if assistG.gcAssistBytes < 0 { | |
| gcAssistAlloc(assistG) | |
| } | |
| } | |
| mp = acquirem() | |
| if doubleCheckMalloc { | |
| doubleCheckSmallNoScan(typ, mp) | |
| } | |
| mp.mallocing = 1 | |
| checkGCTrigger := false | |
| c := getMCache(mp) | |
| span := c.alloc[spc] | |
| var v gclinkptr | |
| var x unsafe.Pointer | |
| if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { | |
| x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| goto post | |
| } | |
| { | |
| var nextFreeFastResult gclinkptr | |
| if span.allocCache != 0 { | |
| theBit := sys.TrailingZeros64(span.allocCache) | |
| result := span.freeindex + uint16(theBit) | |
| if result < span.nelems { | |
| freeidx := result + 1 | |
| if !(freeidx%64 == 0 && freeidx != span.nelems) { | |
| span.allocCache >>= uint(theBit + 1) | |
| span.freeindex = freeidx | |
| span.allocCount++ | |
| nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) | |
| } | |
| } | |
| } | |
| v = nextFreeFastResult | |
| if v == 0 { | |
| v, span, checkGCTrigger = c.nextFree(spc) | |
| } | |
| x = unsafe.Pointer(v) | |
| } | |
| if needzero && span.needzero != 0 { | |
| memclrNoHeapPointers(x, elemsize) | |
| } | |
| publicationBarrier() | |
| if writeBarrier.enabled { | |
| gcmarknewobject(span, uintptr(x)) | |
| } else { | |
| span.freeIndexForScan = span.freeindex | |
| } | |
| c.nextSample -= int64(elemsize) | |
| if c.nextSample < 0 || MemProfileRate != c.memProfRate { | |
| profilealloc(mp, x, elemsize) | |
| } | |
| mp.mallocing = 0 | |
| releasem(mp) | |
| if checkGCTrigger { | |
| if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { | |
| gcStart(t) | |
| } | |
| } | |
| post: | |
| gp := getg() | |
| if goexperiment.RuntimeSecret && gp.secret > 0 { | |
| addSecret(x, size) | |
| } | |
| if gcBlackenEnabled != 0 && elemsize != 0 { | |
| if assistG := getg().m.curg; assistG != nil { | |
| assistG.gcAssistBytes -= int64(elemsize - size) | |
| } | |
| } | |
| if debug.malloc { | |
| postMallocgcDebug(x, elemsize, typ) | |
| } | |
| return x | |
| } |