| // Copyright 2014 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package runtime |
| |
| import ( |
| "internal/abi" |
| "unsafe" |
| ) |
| |
| // Should be a built-in for unsafe.Pointer? |
| // |
| // add should be an internal detail, |
| // but widely used packages access it using linkname. |
| // Notable members of the hall of shame include: |
| // - fortio.org/log |
| // |
| // Do not remove or change the type signature. |
| // See go.dev/issue/67401. |
| // |
| //go:linkname add |
| //go:nosplit |
| func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { |
| return unsafe.Pointer(uintptr(p) + x) |
| } |
| |
| // getg returns the pointer to the current g. |
| // The compiler rewrites calls to this function into instructions |
| // that fetch the g directly (from TLS or from the dedicated register). |
| func getg() *g |
| |
| // mcall switches from the g to the g0 stack and invokes fn(g), |
| // where g is the goroutine that made the call. |
| // mcall saves g's current PC/SP in g->sched so that it can be restored later. |
| // It is up to fn to arrange for that later execution, typically by recording |
| // g in a data structure, causing something to call ready(g) later. |
| // mcall returns to the original goroutine g later, when g has been rescheduled. |
| // fn must not return at all; typically it ends by calling schedule, to let the m |
| // run other goroutines. |
| // |
| // mcall can only be called from g stacks (not g0, not gsignal). |
| // |
| // This must NOT be go:noescape: if fn is a stack-allocated closure, |
| // fn puts g on a run queue, and g executes before fn returns, the |
| // closure will be invalidated while it is still executing. |
| func mcall(fn func(*g)) |
| |
| // systemstack runs fn on a system stack. |
| // If systemstack is called from the per-OS-thread (g0) stack, or |
| // if systemstack is called from the signal handling (gsignal) stack, |
| // systemstack calls fn directly and returns. |
| // Otherwise, systemstack is being called from the limited stack |
| // of an ordinary goroutine. In this case, systemstack switches |
| // to the per-OS-thread stack, calls fn, and switches back. |
| // It is common to use a func literal as the argument, in order |
| // to share inputs and outputs with the code around the call |
| // to system stack: |
| // |
| // ... set up y ... |
| // systemstack(func() { |
| // x = bigcall(y) |
| // }) |
| // ... use x ... |
| // |
| //go:noescape |
| func systemstack(fn func()) |
| |
| //go:nosplit |
| //go:nowritebarrierrec |
| func badsystemstack() { |
| writeErrStr("fatal: systemstack called from unexpected goroutine") |
| } |
| |
| // memclrNoHeapPointers clears n bytes starting at ptr. |
| // |
| // Usually you should use typedmemclr. memclrNoHeapPointers should be |
| // used only when the caller knows that *ptr contains no heap pointers |
| // because either: |
| // |
| // *ptr is initialized memory and its type is pointer-free, or |
| // |
| // *ptr is uninitialized memory (e.g., memory that's being reused |
| // for a new allocation) and hence contains only "junk". |
| // |
| // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n |
| // is a multiple of the pointer size, then any pointer-aligned, |
| // pointer-sized portion is cleared atomically. Despite the function |
| // name, this is necessary because this function is the underlying |
| // implementation of typedmemclr and memclrHasPointers. See the doc of |
| // memmove for more details. |
| // |
| // The (CPU-specific) implementations of this function are in memclr_*.s. |
| // |
| // memclrNoHeapPointers should be an internal detail, |
| // but widely used packages access it using linkname. |
| // Notable members of the hall of shame include: |
| // - github.com/bytedance/sonic |
| // - github.com/chenzhuoyu/iasm |
| // - github.com/dgraph-io/ristretto |
| // - github.com/outcaste-io/ristretto |
| // |
| // Do not remove or change the type signature. |
| // See go.dev/issue/67401. |
| // |
| //go:linkname memclrNoHeapPointers |
| //go:noescape |
| func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) |
| |
| //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers |
| func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { |
| memclrNoHeapPointers(ptr, n) |
| } |
| |
| // memmove copies n bytes from "from" to "to". |
| // |
| // memmove ensures that any pointer in "from" is written to "to" with |
| // an indivisible write, so that racy reads cannot observe a |
| // half-written pointer. This is necessary to prevent the garbage |
| // collector from observing invalid pointers, and differs from memmove |
| // in unmanaged languages. However, memmove is only required to do |
| // this if "from" and "to" may contain pointers, which can only be the |
| // case if "from", "to", and "n" are all be word-aligned. |
| // |
| // Implementations are in memmove_*.s. |
| // |
| // Outside assembly calls memmove. |
| // |
| // memmove should be an internal detail, |
| // but widely used packages access it using linkname. |
| // Notable members of the hall of shame include: |
| // - github.com/bytedance/sonic |
| // - github.com/cloudwego/dynamicgo |
| // - github.com/ebitengine/purego |
| // - github.com/tetratelabs/wazero |
| // - github.com/ugorji/go/codec |
| // - gvisor.dev/gvisor |
| // - github.com/sagernet/gvisor |
| // |
| // Do not remove or change the type signature. |
| // See go.dev/issue/67401. |
| // |
| //go:linkname memmove |
| //go:noescape |
| func memmove(to, from unsafe.Pointer, n uintptr) |
| |
| //go:linkname reflect_memmove reflect.memmove |
| func reflect_memmove(to, from unsafe.Pointer, n uintptr) { |
| memmove(to, from, n) |
| } |
| |
| // exported value for testing |
| const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) |
| |
| // in internal/bytealg/equal_*.s |
| // |
| // memequal should be an internal detail, |
| // but widely used packages access it using linkname. |
| // Notable members of the hall of shame include: |
| // - github.com/bytedance/sonic |
| // |
| // Do not remove or change the type signature. |
| // See go.dev/issue/67401. |
| // |
| //go:linkname memequal |
| //go:noescape |
| func memequal(a, b unsafe.Pointer, size uintptr) bool |
| |
| // noescape hides a pointer from escape analysis. noescape is |
| // the identity function but escape analysis doesn't think the |
| // output depends on the input. noescape is inlined and currently |
| // compiles down to zero instructions. |
| // USE CAREFULLY! |
| // |
| // noescape should be an internal detail, |
| // but widely used packages access it using linkname. |
| // Notable members of the hall of shame include: |
| // - github.com/bytedance/gopkg |
| // - github.com/ebitengine/purego |
| // - github.com/hamba/avro/v2 |
| // - github.com/puzpuzpuz/xsync/v3 |
| // - github.com/songzhibin97/gkit |
| // |
| // Do not remove or change the type signature. |
| // See go.dev/issue/67401. |
| // |
| //go:linkname noescape |
| //go:nosplit |
| func noescape(p unsafe.Pointer) unsafe.Pointer { |
| x := uintptr(p) |
| return unsafe.Pointer(x ^ 0) |
| } |
| |
| // noEscapePtr hides a pointer from escape analysis. See noescape. |
| // USE CAREFULLY! |
| // |
| //go:nosplit |
| func noEscapePtr[T any](p *T) *T { |
| x := uintptr(unsafe.Pointer(p)) |
| return (*T)(unsafe.Pointer(x ^ 0)) |
| } |
| |
| // Not all cgocallback frames are actually cgocallback, |
| // so not all have these arguments. Mark them uintptr so that the GC |
| // does not misinterpret memory when the arguments are not present. |
| // cgocallback is not called from Go, only from crosscall2. |
| // This in turn calls cgocallbackg, which is where we'll find |
| // pointer-declared arguments. |
| // |
| // When fn is nil (frame is saved g), call dropm instead, |
| // this is used when the C thread is exiting. |
| func cgocallback(fn, frame, ctxt uintptr) |
| |
| func gogo(buf *gobuf) |
| |
| func asminit() |
| func setg(gg *g) |
| func breakpoint() |
| |
| // reflectcall calls fn with arguments described by stackArgs, stackArgsSize, |
| // frameSize, and regArgs. |
| // |
| // Arguments passed on the stack and space for return values passed on the stack |
| // must be laid out at the space pointed to by stackArgs (with total length |
| // stackArgsSize) according to the ABI. |
| // |
| // stackRetOffset must be some value <= stackArgsSize that indicates the |
| // offset within stackArgs where the return value space begins. |
| // |
| // frameSize is the total size of the argument frame at stackArgs and must |
| // therefore be >= stackArgsSize. It must include additional space for spilling |
| // register arguments for stack growth and preemption. |
| // |
| // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, |
| // since frameSize will be redundant with stackArgsSize. |
| // |
| // Arguments passed in registers must be laid out in regArgs according to the ABI. |
| // regArgs will hold any return values passed in registers after the call. |
| // |
| // reflectcall copies stack arguments from stackArgs to the goroutine stack, and |
| // then copies back stackArgsSize-stackRetOffset bytes back to the return space |
| // in stackArgs once fn has completed. It also "unspills" argument registers from |
| // regArgs before calling fn, and spills them back into regArgs immediately |
| // following the call to fn. If there are results being returned on the stack, |
| // the caller should pass the argument frame type as stackArgsType so that |
| // reflectcall can execute appropriate write barriers during the copy. |
| // |
| // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which |
| // registers on the return path will contain Go pointers. It will then store |
| // these pointers in regArgs.Ptrs such that they are visible to the GC. |
| // |
| // Package reflect passes a frame type. In package runtime, there is only |
| // one call that copies results back, in callbackWrap in syscall_windows.go, and it |
| // does NOT pass a frame type, meaning there are no write barriers invoked. See that |
| // call site for justification. |
| // |
| // Package reflect accesses this symbol through a linkname. |
| // |
| // Arguments passed through to reflectcall do not escape. The type is used |
| // only in a very limited callee of reflectcall, the stackArgs are copied, and |
| // regArgs is only used in the reflectcall frame. |
| // |
| //go:noescape |
| func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| |
| // procyield should be an internal detail, |
| // but widely used packages access it using linkname. |
| // Notable members of the hall of shame include: |
| // - github.com/sagernet/sing-tun |
| // - github.com/slackhq/nebula |
| // - github.com/tailscale/wireguard-go |
| // |
| // Do not remove or change the type signature. |
| // See go.dev/issue/67401. |
| // |
| //go:linkname procyield |
| func procyield(cycles uint32) |
| |
| type neverCallThisFunction struct{} |
| |
| // goexit is the return stub at the top of every goroutine call stack. |
| // Each goroutine stack is constructed as if goexit called the |
| // goroutine's entry point function, so that when the entry point |
| // function returns, it will return to goexit, which will call goexit1 |
| // to perform the actual exit. |
| // |
| // This function must never be called directly. Call goexit1 instead. |
| // gentraceback assumes that goexit terminates the stack. A direct |
| // call on the stack will cause gentraceback to stop walking the stack |
| // prematurely and if there is leftover state it may panic. |
| func goexit(neverCallThisFunction) |
| |
| // publicationBarrier performs a store/store barrier (a "publication" |
| // or "export" barrier). Some form of synchronization is required |
| // between initializing an object and making that object accessible to |
| // another processor. Without synchronization, the initialization |
| // writes and the "publication" write may be reordered, allowing the |
| // other processor to follow the pointer and observe an uninitialized |
| // object. In general, higher-level synchronization should be used, |
| // such as locking or an atomic pointer write. publicationBarrier is |
| // for when those aren't an option, such as in the implementation of |
| // the memory manager. |
| // |
| // There's no corresponding barrier for the read side because the read |
| // side naturally has a data dependency order. All architectures that |
| // Go supports or seems likely to ever support automatically enforce |
| // data dependency ordering. |
| func publicationBarrier() |
| |
| // getcallerpc returns the program counter (PC) of its caller's caller. |
| // getcallersp returns the stack pointer (SP) of its caller's caller. |
| // The implementation may be a compiler intrinsic; there is not |
| // necessarily code implementing this on every platform. |
| // |
| // For example: |
| // |
| // func f(arg1, arg2, arg3 int) { |
| // pc := getcallerpc() |
| // sp := getcallersp() |
| // } |
| // |
| // These two lines find the PC and SP immediately following |
| // the call to f (where f will return). |
| // |
| // The call to getcallerpc and getcallersp must be done in the |
| // frame being asked about. |
| // |
| // The result of getcallersp is correct at the time of the return, |
| // but it may be invalidated by any subsequent call to a function |
| // that might relocate the stack in order to grow or shrink it. |
| // A general rule is that the result of getcallersp should be used |
| // immediately and can only be passed to nosplit functions. |
| |
| //go:noescape |
| func getcallerpc() uintptr |
| |
| //go:noescape |
| func getcallersp() uintptr // implemented as an intrinsic on all platforms |
| |
| // getclosureptr returns the pointer to the current closure. |
| // getclosureptr can only be used in an assignment statement |
| // at the entry of a function. Moreover, go:nosplit directive |
| // must be specified at the declaration of caller function, |
| // so that the function prolog does not clobber the closure register. |
| // for example: |
| // |
| // //go:nosplit |
| // func f(arg1, arg2, arg3 int) { |
| // dx := getclosureptr() |
| // } |
| // |
| // The compiler rewrites calls to this function into instructions that fetch the |
| // pointer from a well-known register (DX on x86 architecture, etc.) directly. |
| // |
| // WARNING: PGO-based devirtualization cannot detect that caller of |
| // getclosureptr require closure context, and thus must maintain a list of |
| // these functions, which is in |
| // cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall. |
| func getclosureptr() uintptr |
| |
| //go:noescape |
| func asmcgocall(fn, arg unsafe.Pointer) int32 |
| |
| func morestack() |
| |
| func morestack_noctxt() |
| |
| func rt0_go() |
| |
| // return0 is a stub used to return 0 from deferproc. |
| // It is called at the very end of deferproc to signal |
| // the calling Go function that it should not jump |
| // to deferreturn. |
| // in asm_*.s |
| func return0() |
| |
| // in asm_*.s |
| // not called directly; definitions here supply type information for traceback. |
| // These must have the same signature (arg pointer map) as reflectcall. |
| func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) |
| |
| func systemstack_switch() |
| |
| // alignUp rounds n up to a multiple of a. a must be a power of 2. |
| // |
| //go:nosplit |
| func alignUp(n, a uintptr) uintptr { |
| return (n + a - 1) &^ (a - 1) |
| } |
| |
| // alignDown rounds n down to a multiple of a. a must be a power of 2. |
| // |
| //go:nosplit |
| func alignDown(n, a uintptr) uintptr { |
| return n &^ (a - 1) |
| } |
| |
| // divRoundUp returns ceil(n / a). |
| func divRoundUp(n, a uintptr) uintptr { |
| // a is generally a power of two. This will get inlined and |
| // the compiler will optimize the division. |
| return (n + a - 1) / a |
| } |
| |
| // checkASM reports whether assembly runtime checks have passed. |
| func checkASM() bool |
| |
| func memequal_varlen(a, b unsafe.Pointer) bool |
| |
| // bool2int returns 0 if x is false or 1 if x is true. |
| func bool2int(x bool) int { |
| // Avoid branches. In the SSA compiler, this compiles to |
| // exactly what you would want it to. |
| return int(*(*uint8)(unsafe.Pointer(&x))) |
| } |
| |
| // abort crashes the runtime in situations where even throw might not |
| // work. In general it should do something a debugger will recognize |
| // (e.g., an INT3 on x86). A crash in abort is recognized by the |
| // signal handler, which will attempt to tear down the runtime |
| // immediately. |
| func abort() |
| |
| // Called from compiled code; declared for vet; do NOT call from Go. |
| func gcWriteBarrier1() |
| |
| // gcWriteBarrier2 should be an internal detail, |
| // but widely used packages access it using linkname. |
| // Notable members of the hall of shame include: |
| // - github.com/bytedance/sonic |
| // |
| // Do not remove or change the type signature. |
| // See go.dev/issue/67401. |
| // |
| //go:linkname gcWriteBarrier2 |
| func gcWriteBarrier2() |
| |
| func gcWriteBarrier3() |
| func gcWriteBarrier4() |
| func gcWriteBarrier5() |
| func gcWriteBarrier6() |
| func gcWriteBarrier7() |
| func gcWriteBarrier8() |
| func duffzero() |
| func duffcopy() |
| |
| // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. |
| func addmoduledata() |
| |
| // Injected by the signal handler for panicking signals. |
| // Initializes any registers that have fixed meaning at calls but |
| // are scratch in bodies and calls sigpanic. |
| // On many platforms it just jumps to sigpanic. |
| func sigpanic0() |
| |
| // intArgRegs is used by the various register assignment |
| // algorithm implementations in the runtime. These include:. |
| // - Finalizers (mfinal.go) |
| // - Windows callbacks (syscall_windows.go) |
| // |
| // Both are stripped-down versions of the algorithm since they |
| // only have to deal with a subset of cases (finalizers only |
| // take a pointer or interface argument, Go Windows callbacks |
| // don't support floating point). |
| // |
| // It should be modified with care and are generally only |
| // modified when testing this package. |
| // |
| // It should never be set higher than its internal/abi |
| // constant counterparts, because the system relies on a |
| // structure that is at least large enough to hold the |
| // registers the system supports. |
| // |
| // Protected by finlock. |
| var intArgRegs = abi.IntArgRegs |