| // Copyright 2014 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package runtime |
| |
| import "unsafe" |
| |
| // Declarations for runtime services implemented in C or assembly. |
| |
| const ptrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ideal const |
| const regSize = 4 << (^uintreg(0) >> 63) // unsafe.Sizeof(uintreg(0)) but an ideal const |
| |
| // Should be a built-in for unsafe.Pointer? |
| //go:nosplit |
| func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { |
| return unsafe.Pointer(uintptr(p) + x) |
| } |
| |
| // n must be a power of 2 |
| func roundup(p unsafe.Pointer, n uintptr) unsafe.Pointer { |
| delta := -uintptr(p) & (n - 1) |
| return unsafe.Pointer(uintptr(p) + delta) |
| } |
| |
| func getg() *g |
| |
| // mcall switches from the g to the g0 stack and invokes fn(g), |
| // where g is the goroutine that made the call. |
| // mcall saves g's current PC/SP in g->sched so that it can be restored later. |
| // It is up to fn to arrange for that later execution, typically by recording |
| // g in a data structure, causing something to call ready(g) later. |
| // mcall returns to the original goroutine g later, when g has been rescheduled. |
| // fn must not return at all; typically it ends by calling schedule, to let the m |
| // run other goroutines. |
| // |
| // mcall can only be called from g stacks (not g0, not gsignal). |
| //go:noescape |
| func mcall(fn func(*g)) |
| |
| // systemstack runs fn on a system stack. |
| // If systemstack is called from the per-OS-thread (g0) stack, or |
| // if systemstack is called from the signal handling (gsignal) stack, |
| // systemstack calls fn directly and returns. |
| // Otherwise, systemstack is being called from the limited stack |
| // of an ordinary goroutine. In this case, systemstack switches |
| // to the per-OS-thread stack, calls fn, and switches back. |
| // It is common to use a func literal as the argument, in order |
| // to share inputs and outputs with the code around the call |
| // to system stack: |
| // |
| // ... set up y ... |
| // systemstack(func() { |
| // x = bigcall(y) |
| // }) |
| // ... use x ... |
| // |
| //go:noescape |
| func systemstack(fn func()) |
| |
| func badsystemstack() { |
| gothrow("systemstack called from unexpected goroutine") |
| } |
| |
| // memclr clears n bytes starting at ptr. |
| // in memclr_*.s |
| //go:noescape |
| func memclr(ptr unsafe.Pointer, n uintptr) |
| |
| // memmove copies n bytes from "from" to "to". |
| // in memmove_*.s |
| //go:noescape |
| func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) |
| |
| // exported value for testing |
| var hashLoad = loadFactor |
| |
| // in asm_*.s |
| func fastrand1() uint32 |
| |
| // in asm_*.s |
| //go:noescape |
| func memeq(a, b unsafe.Pointer, size uintptr) bool |
| |
| // noescape hides a pointer from escape analysis. noescape is |
| // the identity function but escape analysis doesn't think the |
| // output depends on the input. noescape is inlined and currently |
| // compiles down to a single xor instruction. |
| // USE CAREFULLY! |
| //go:nosplit |
| func noescape(p unsafe.Pointer) unsafe.Pointer { |
| x := uintptr(p) |
| return unsafe.Pointer(x ^ 0) |
| } |
| |
| func cgocallback(fn, frame unsafe.Pointer, framesize uintptr) |
| func gogo(buf *gobuf) |
| func gosave(buf *gobuf) |
| func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32 |
| |
| //go:noescape |
| func jmpdefer(fv *funcval, argp uintptr) |
| func exit1(code int32) |
| func asminit() |
| func setg(gg *g) |
| func breakpoint() |
| |
| func reflectcall(fn, arg unsafe.Pointer, n uint32, retoffset uint32) |
| func procyield(cycles uint32) |
| func cgocallback_gofunc(fv *funcval, frame unsafe.Pointer, framesize uintptr) |
| func goexit() |
| |
| //go:noescape |
| func cas(ptr *uint32, old, new uint32) bool |
| |
| // casp cannot have a go:noescape annotation, because |
| // while ptr and old do not escape, new does. If new is marked as |
| // not escaping, the compiler will make incorrect escape analysis |
| // decisions about the value being xchg'ed. |
| // Instead, make casp a wrapper around the actual atomic. |
| // When calling the wrapper we mark ptr as noescape explicitly. |
| |
| //go:nosplit |
| func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool { |
| return casp1((*unsafe.Pointer)(noescape(unsafe.Pointer(ptr))), noescape(old), new) |
| } |
| |
| func casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool |
| |
| func nop() // call to prevent inlining of function body |
| |
| //go:noescape |
| func casuintptr(ptr *uintptr, old, new uintptr) bool |
| |
| //go:noescape |
| func atomicstoreuintptr(ptr *uintptr, new uintptr) |
| |
| //go:noescape |
| func atomicloaduintptr(ptr *uintptr) uintptr |
| |
| //go:noescape |
| func atomicloaduint(ptr *uint) uint |
| |
| //go:noescape |
| func setcallerpc(argp unsafe.Pointer, pc uintptr) |
| |
| // getcallerpc returns the program counter (PC) of its caller's caller. |
| // getcallersp returns the stack pointer (SP) of its caller's caller. |
| // For both, the argp must be a pointer to the caller's first function argument. |
| // The implementation may or may not use argp, depending on |
| // the architecture. |
| // |
| // For example: |
| // |
| // func f(arg1, arg2, arg3 int) { |
| // pc := getcallerpc(unsafe.Pointer(&arg1)) |
| // sp := getcallersp(unsafe.Pointer(&arg1)) |
| // } |
| // |
| // These two lines find the PC and SP immediately following |
| // the call to f (where f will return). |
| // |
| // The call to getcallerpc and getcallersp must be done in the |
| // frame being asked about. It would not be correct for f to pass &arg1 |
| // to another function g and let g call getcallerpc/getcallersp. |
| // The call inside g might return information about g's caller or |
| // information about f's caller or complete garbage. |
| // |
| // The result of getcallersp is correct at the time of the return, |
| // but it may be invalidated by any subsequent call to a function |
| // that might relocate the stack in order to grow or shrink it. |
| // A general rule is that the result of getcallersp should be used |
| // immediately and can only be passed to nosplit functions. |
| |
| //go:noescape |
| func getcallerpc(argp unsafe.Pointer) uintptr |
| |
| //go:noescape |
| func getcallersp(argp unsafe.Pointer) uintptr |
| |
| //go:noescape |
| func asmcgocall(fn, arg unsafe.Pointer) |
| |
| //go:noescape |
| func asmcgocall_errno(fn, arg unsafe.Pointer) int32 |
| |
| // argp used in Defer structs when there is no argp. |
| const _NoArgs = ^uintptr(0) |
| |
| func morestack() |
| func rt0_go() |
| |
| // return0 is a stub used to return 0 from deferproc. |
| // It is called at the very end of deferproc to signal |
| // the calling Go function that it should not jump |
| // to deferreturn. |
| // in asm_*.s |
| func return0() |
| |
| // thunk to call time.now. |
| func timenow() (sec int64, nsec int32) |
| |
| // in asm_*.s |
| // not called directly; definitions here supply type information for traceback. |
| func call16(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call32(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call64(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call128(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call256(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call512(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call1024(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call2048(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call4096(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call8192(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call16384(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call32768(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call65536(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call131072(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call262144(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call524288(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call1048576(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call2097152(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call4194304(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call8388608(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call16777216(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call33554432(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call67108864(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call134217728(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call268435456(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call536870912(fn, arg unsafe.Pointer, n, retoffset uint32) |
| func call1073741824(fn, arg unsafe.Pointer, n, retoffset uint32) |
| |
| func systemstack_switch() |
| |
| func prefetcht0(addr uintptr) |
| func prefetcht1(addr uintptr) |
| func prefetcht2(addr uintptr) |
| func prefetchnta(addr uintptr) |