Keith Randall | 61dca94 | 2014-06-16 23:03:03 -0700 | [diff] [blame] | 1 | // Copyright 2014 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | package runtime |
| 6 | |
| 7 | import "unsafe" |
| 8 | |
| 9 | // Declarations for runtime services implemented in C or assembly. |
| 10 | // C implementations of these functions are in stubs.goc. |
| 11 | // Assembly implementations are in various files, see comments with |
| 12 | // each function. |
| 13 | |
Russ Cox | 3a7f664 | 2014-08-29 16:20:48 -0400 | [diff] [blame] | 14 | const ptrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ideal const |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 15 | |
Keith Randall | 61dca94 | 2014-06-16 23:03:03 -0700 | [diff] [blame] | 16 | //go:noescape |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 17 | func racereadpc(addr unsafe.Pointer, callpc, pc uintptr) |
| 18 | |
| 19 | //go:noescape |
| 20 | func racewritepc(addr unsafe.Pointer, callpc, pc uintptr) |
| 21 | |
| 22 | //go:noescape |
Keith Randall | 61dca94 | 2014-06-16 23:03:03 -0700 | [diff] [blame] | 23 | func racereadrangepc(addr unsafe.Pointer, len int, callpc, pc uintptr) |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 24 | |
Keith Randall | cc9ec52 | 2014-07-31 12:43:40 -0700 | [diff] [blame] | 25 | //go:noescape |
| 26 | func racewriterangepc(addr unsafe.Pointer, len int, callpc, pc uintptr) |
| 27 | |
Keith Randall | 9a1e142 | 2014-08-24 12:31:03 +0400 | [diff] [blame] | 28 | //go:noescape |
| 29 | func raceacquire(addr unsafe.Pointer) |
| 30 | |
| 31 | //go:noescape |
| 32 | func racerelease(addr unsafe.Pointer) |
| 33 | |
| 34 | //go:noescape |
| 35 | func raceacquireg(gp *g, addr unsafe.Pointer) |
| 36 | |
| 37 | //go:noescape |
| 38 | func racereleaseg(gp *g, addr unsafe.Pointer) |
| 39 | |
Keith Randall | c46bcd4 | 2014-08-28 13:23:10 -0700 | [diff] [blame] | 40 | func racefingo() |
| 41 | |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 42 | // Should be a built-in for unsafe.Pointer? |
| 43 | func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { |
| 44 | return unsafe.Pointer(uintptr(p) + x) |
| 45 | } |
| 46 | |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 47 | // n must be a power of 2 |
| 48 | func roundup(p unsafe.Pointer, n uintptr) unsafe.Pointer { |
Russ Cox | 6c67dd9 | 2014-08-24 20:28:29 -0400 | [diff] [blame] | 49 | delta := -uintptr(p) & (n - 1) |
| 50 | return unsafe.Pointer(uintptr(p) + delta) |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 51 | } |
| 52 | |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 53 | // in stubs.goc |
Dmitriy Vyukov | 684de04 | 2014-08-21 20:41:09 +0400 | [diff] [blame] | 54 | func getg() *g |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 55 | func acquirem() *m |
| 56 | func releasem(mp *m) |
Dmitriy Vyukov | 30940cf | 2014-08-18 16:33:39 +0400 | [diff] [blame] | 57 | func gomcache() *mcache |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 58 | |
Keith Randall | e359bea | 2014-08-06 14:33:57 -0700 | [diff] [blame] | 59 | // An mFunction represents a C function that runs on the M stack. It |
| 60 | // can be called from Go using mcall or onM. Through the magic of |
| 61 | // linking, an mFunction variable and the corresponding C code entry |
| 62 | // point live at the same address. |
| 63 | type mFunction byte |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 64 | |
Keith Randall | e359bea | 2014-08-06 14:33:57 -0700 | [diff] [blame] | 65 | // in asm_*.s |
| 66 | func mcall(fn *mFunction) |
| 67 | func onM(fn *mFunction) |
| 68 | |
| 69 | // C functions that run on the M stack. Call these like |
| 70 | // mcall(&mcacheRefill_m) |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 71 | // Arguments should be passed in m->scalararg[x] and |
| 72 | // m->ptrarg[x]. Return values can be passed in those |
| 73 | // same slots. |
Keith Randall | e359bea | 2014-08-06 14:33:57 -0700 | [diff] [blame] | 74 | var ( |
| 75 | mcacheRefill_m, |
| 76 | largeAlloc_m, |
| 77 | mprofMalloc_m, |
| 78 | gc_m, |
Dmitriy Vyukov | 9f38b6c | 2014-08-29 18:44:38 +0400 | [diff] [blame] | 79 | scavenge_m, |
Keith Randall | e359bea | 2014-08-06 14:33:57 -0700 | [diff] [blame] | 80 | setFinalizer_m, |
Keith Randall | c46bcd4 | 2014-08-28 13:23:10 -0700 | [diff] [blame] | 81 | removeFinalizer_m, |
Dmitriy Vyukov | aac7f1a | 2014-08-07 13:34:30 +0400 | [diff] [blame] | 82 | markallocated_m, |
| 83 | unrollgcprog_m, |
Dmitriy Vyukov | 5d40742 | 2014-08-19 11:49:59 +0400 | [diff] [blame] | 84 | unrollgcproginplace_m, |
Dmitriy Vyukov | 684de04 | 2014-08-21 20:41:09 +0400 | [diff] [blame] | 85 | gosched_m, |
Sanjay Menakuru | ef50462 | 2014-08-24 20:27:00 -0700 | [diff] [blame] | 86 | setgcpercent_m, |
| 87 | setmaxthreads_m, |
Dmitriy Vyukov | 684de04 | 2014-08-21 20:41:09 +0400 | [diff] [blame] | 88 | ready_m, |
Russ Cox | 3a7f664 | 2014-08-29 16:20:48 -0400 | [diff] [blame] | 89 | park_m mFunction |
Keith Randall | e359bea | 2014-08-06 14:33:57 -0700 | [diff] [blame] | 90 | ) |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 91 | |
Russ Cox | d21638b | 2014-08-27 21:59:49 -0400 | [diff] [blame] | 92 | func blockevent(int64, int32) |
| 93 | |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 94 | // memclr clears n bytes starting at ptr. |
| 95 | // in memclr_*.s |
Keith Randall | 483cb61 | 2014-08-07 13:58:42 -0700 | [diff] [blame] | 96 | //go:noescape |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 97 | func memclr(ptr unsafe.Pointer, n uintptr) |
| 98 | |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 99 | func racemalloc(p unsafe.Pointer, size uintptr) |
| 100 | func tracealloc(p unsafe.Pointer, size uintptr, typ *_type) |
| 101 | |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 102 | // memmove copies n bytes from "from" to "to". |
| 103 | // in memmove_*.s |
Keith Randall | 483cb61 | 2014-08-07 13:58:42 -0700 | [diff] [blame] | 104 | //go:noescape |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 105 | func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) |
| 106 | |
| 107 | // in asm_*.s |
| 108 | func fastrand2() uint32 |
| 109 | |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 110 | const ( |
Dmitriy Vyukov | 9f38b6c | 2014-08-29 18:44:38 +0400 | [diff] [blame] | 111 | concurrentSweep = true |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 112 | ) |
| 113 | |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 114 | func gosched() |
| 115 | func starttheworld() |
| 116 | func stoptheworld() |
| 117 | func clearpools() |
| 118 | |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 119 | // exported value for testing |
| 120 | var hashLoad = loadFactor |
| 121 | |
| 122 | // in asm_*.s |
| 123 | //go:noescape |
Keith Randall | 7aa4e5a | 2014-08-07 14:52:55 -0700 | [diff] [blame] | 124 | func memeq(a, b unsafe.Pointer, size uintptr) bool |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 125 | |
Keith Randall | 483cb61 | 2014-08-07 13:58:42 -0700 | [diff] [blame] | 126 | // Code pointers for the nohash/noequal algorithms. Used for producing better error messages. |
Keith Randall | 0c6b55e | 2014-07-16 14:16:19 -0700 | [diff] [blame] | 127 | var nohashcode uintptr |
Keith Randall | 483cb61 | 2014-08-07 13:58:42 -0700 | [diff] [blame] | 128 | var noequalcode uintptr |
Dave Cheney | ec5d7ba | 2014-07-18 16:30:38 +1000 | [diff] [blame] | 129 | |
| 130 | // Go version of runtime.throw. |
| 131 | // in panic.c |
Dave Cheney | 355c38d | 2014-07-23 07:08:52 +1000 | [diff] [blame] | 132 | func gothrow(s string) |
Keith Randall | 4aa5043 | 2014-07-30 09:01:52 -0700 | [diff] [blame] | 133 | |
Keith Randall | a2a9768 | 2014-07-31 15:07:05 -0700 | [diff] [blame] | 134 | // noescape hides a pointer from escape analysis. noescape is |
| 135 | // the identity function but escape analysis doesn't think the |
| 136 | // output depends on the input. noescape is inlined and currently |
| 137 | // compiles down to a single xor instruction. |
| 138 | // USE CAREFULLY! |
| 139 | func noescape(p unsafe.Pointer) unsafe.Pointer { |
| 140 | x := uintptr(p) |
| 141 | return unsafe.Pointer(x ^ 0) |
| 142 | } |
Keith Randall | 483cb61 | 2014-08-07 13:58:42 -0700 | [diff] [blame] | 143 | |
Russ Cox | 3a7f664 | 2014-08-29 16:20:48 -0400 | [diff] [blame] | 144 | func entersyscall() |
| 145 | func entersyscallblock() |
Dmitriy Vyukov | afb2260 | 2014-08-22 22:13:01 +0400 | [diff] [blame] | 146 | func exitsyscall() |
Rémy Oudompheng | 39ffa8b | 2014-08-26 08:34:46 +0200 | [diff] [blame] | 147 | |
Brad Fitzpatrick | 9a5654a | 2014-08-27 09:31:32 -0700 | [diff] [blame] | 148 | func goroutineheader(gp *g) |
Rémy Oudompheng | 39ffa8b | 2014-08-26 08:34:46 +0200 | [diff] [blame] | 149 | func traceback(pc, sp, lr uintptr, gp *g) |
| 150 | func tracebackothers(gp *g) |
Russ Cox | 25f6b02 | 2014-08-27 11:32:17 -0400 | [diff] [blame] | 151 | |
| 152 | func cgocallback(fn, frame unsafe.Pointer, framesize uintptr) |
| 153 | func gogo(buf *gobuf) |
| 154 | func gosave(buf *gobuf) |
| 155 | func open(name *byte, mode, perm int32) int32 |
| 156 | func read(fd int32, p unsafe.Pointer, n int32) int32 |
Russ Cox | 25f6b02 | 2014-08-27 11:32:17 -0400 | [diff] [blame] | 157 | func close(fd int32) int32 |
| 158 | func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32 |
| 159 | func jmpdefer(fv *funcval, argp unsafe.Pointer) |
| 160 | func exit1(code int32) |
| 161 | func asminit() |
Russ Cox | 25f6b02 | 2014-08-27 11:32:17 -0400 | [diff] [blame] | 162 | func setg(gg *g) |
| 163 | func exit(code int32) |
| 164 | func breakpoint() |
| 165 | func asmcgocall(fn, arg unsafe.Pointer) |
| 166 | func nanotime() int64 |
| 167 | func usleep(usec uint32) |
| 168 | func cputicks() int64 |
| 169 | func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer |
| 170 | func munmap(addr unsafe.Pointer, n uintptr) |
| 171 | func madvise(addr unsafe.Pointer, n uintptr, flags int32) |
Russ Cox | 25f6b02 | 2014-08-27 11:32:17 -0400 | [diff] [blame] | 172 | func newstackcall(fv *funcval, addr unsafe.Pointer, size uint32) |
Keith Randall | c46bcd4 | 2014-08-28 13:23:10 -0700 | [diff] [blame] | 173 | func reflectcall(fn, arg unsafe.Pointer, n uint32, retoffset uint32) |
Russ Cox | 25f6b02 | 2014-08-27 11:32:17 -0400 | [diff] [blame] | 174 | func procyield(cycles uint32) |
| 175 | func osyield() |
| 176 | func cgocallback_gofunc(fv *funcval, frame unsafe.Pointer, framesize uintptr) |
Russ Cox | d21638b | 2014-08-27 21:59:49 -0400 | [diff] [blame] | 177 | func persistentalloc(size, align uintptr, stat *uint64) unsafe.Pointer |
| 178 | func readgogc() int32 |
Keith Randall | c46bcd4 | 2014-08-28 13:23:10 -0700 | [diff] [blame] | 179 | func purgecachedstats(c *mcache) |
Keith Randall | ef64d9f | 2014-08-28 23:26:50 -0700 | [diff] [blame] | 180 | func gostringnocopy(b *byte) string |
Russ Cox | d21638b | 2014-08-27 21:59:49 -0400 | [diff] [blame] | 181 | |
| 182 | //go:noescape |
Russ Cox | 597b266 | 2014-08-28 23:26:40 -0400 | [diff] [blame] | 183 | func write(fd uintptr, p unsafe.Pointer, n int32) int32 |
| 184 | |
| 185 | //go:noescape |
Russ Cox | d21638b | 2014-08-27 21:59:49 -0400 | [diff] [blame] | 186 | func cas(ptr *uint32, old, new uint32) bool |
| 187 | |
| 188 | //go:noescape |
| 189 | func cas64(ptr *uint64, old, new uint64) bool |
| 190 | |
| 191 | //go:noescape |
| 192 | func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool |
| 193 | |
| 194 | //go:noescape |
| 195 | func casuintptr(ptr *uintptr, old, new uintptr) bool |
| 196 | |
| 197 | //go:noescape |
| 198 | func xadd(ptr *uint32, delta int32) uint32 |
| 199 | |
| 200 | //go:noescape |
| 201 | func xadd64(ptr *uint64, delta int64) uint64 |
| 202 | |
| 203 | //go:noescape |
| 204 | func xchg(ptr *uint32, new uint32) uint32 |
| 205 | |
| 206 | //go:noescape |
| 207 | func xchg64(ptr *uint64, new uint64) uint64 |
| 208 | |
| 209 | //go:noescape |
| 210 | func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer |
| 211 | |
| 212 | //go:noescape |
| 213 | func atomicstore(ptr *uint32, val uint32) |
| 214 | |
| 215 | //go:noescape |
| 216 | func atomicstore64(ptr *uint64, val uint64) |
| 217 | |
| 218 | //go:noescape |
| 219 | func atomicstorep(ptr unsafe.Pointer, val unsafe.Pointer) |
| 220 | |
| 221 | //go:noescape |
Russ Cox | 3a7f664 | 2014-08-29 16:20:48 -0400 | [diff] [blame] | 222 | func atomicstoreuintptr(ptr *uintptr, new uintptr) |
| 223 | |
| 224 | //go:noescape |
Russ Cox | d21638b | 2014-08-27 21:59:49 -0400 | [diff] [blame] | 225 | func atomicload(ptr *uint32) uint32 |
| 226 | |
| 227 | //go:noescape |
| 228 | func atomicload64(ptr *uint64) uint64 |
| 229 | |
| 230 | //go:noescape |
| 231 | func atomicloadp(ptr unsafe.Pointer) unsafe.Pointer |
| 232 | |
| 233 | //go:noescape |
Russ Cox | 3a7f664 | 2014-08-29 16:20:48 -0400 | [diff] [blame] | 234 | func atomicloaduintptr(ptr *uintptr) uintptr |
| 235 | |
| 236 | //go:noescape |
Keith Randall | 47d6af2 | 2014-08-30 11:03:28 -0700 | [diff] [blame^] | 237 | func atomicloaduint(ptr *uint) uint |
| 238 | |
| 239 | //go:noescape |
Russ Cox | d21638b | 2014-08-27 21:59:49 -0400 | [diff] [blame] | 240 | func atomicor8(ptr *uint8, val uint8) |
| 241 | |
| 242 | //go:noescape |
| 243 | func setcallerpc(argp unsafe.Pointer, pc uintptr) |
| 244 | |
| 245 | //go:noescape |
| 246 | func getcallerpc(argp unsafe.Pointer) uintptr |
| 247 | |
| 248 | //go:noescape |
| 249 | func getcallersp(argp unsafe.Pointer) uintptr |