blob: 237457f68488a804b9abe2c9107e4b953bcd4300 [file] [log] [blame]
Keith Randall61dca942014-06-16 23:03:03 -07001// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package runtime
6
7import "unsafe"
8
9// Declarations for runtime services implemented in C or assembly.
10// C implementations of these functions are in stubs.goc.
11// Assembly implementations are in various files, see comments with
12// each function.
13
Russ Cox3a7f6642014-08-29 16:20:48 -040014const ptrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ideal const
Keith Randall0c6b55e2014-07-16 14:16:19 -070015
Keith Randall61dca942014-06-16 23:03:03 -070016//go:noescape
Keith Randall0c6b55e2014-07-16 14:16:19 -070017func racereadpc(addr unsafe.Pointer, callpc, pc uintptr)
18
19//go:noescape
20func racewritepc(addr unsafe.Pointer, callpc, pc uintptr)
21
22//go:noescape
Keith Randall61dca942014-06-16 23:03:03 -070023func racereadrangepc(addr unsafe.Pointer, len int, callpc, pc uintptr)
Keith Randall0c6b55e2014-07-16 14:16:19 -070024
Keith Randallcc9ec522014-07-31 12:43:40 -070025//go:noescape
26func racewriterangepc(addr unsafe.Pointer, len int, callpc, pc uintptr)
27
Keith Randall9a1e1422014-08-24 12:31:03 +040028//go:noescape
29func raceacquire(addr unsafe.Pointer)
30
31//go:noescape
32func racerelease(addr unsafe.Pointer)
33
34//go:noescape
35func raceacquireg(gp *g, addr unsafe.Pointer)
36
37//go:noescape
38func racereleaseg(gp *g, addr unsafe.Pointer)
39
Keith Randallc46bcd42014-08-28 13:23:10 -070040func racefingo()
41
Keith Randall0c6b55e2014-07-16 14:16:19 -070042// Should be a built-in for unsafe.Pointer?
43func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
44 return unsafe.Pointer(uintptr(p) + x)
45}
46
Keith Randall4aa50432014-07-30 09:01:52 -070047// n must be a power of 2
48func roundup(p unsafe.Pointer, n uintptr) unsafe.Pointer {
Russ Cox6c67dd92014-08-24 20:28:29 -040049 delta := -uintptr(p) & (n - 1)
50 return unsafe.Pointer(uintptr(p) + delta)
Keith Randall4aa50432014-07-30 09:01:52 -070051}
52
Keith Randall0c6b55e2014-07-16 14:16:19 -070053// in stubs.goc
Dmitriy Vyukov684de042014-08-21 20:41:09 +040054func getg() *g
Keith Randall4aa50432014-07-30 09:01:52 -070055func acquirem() *m
56func releasem(mp *m)
Dmitriy Vyukov30940cf2014-08-18 16:33:39 +040057func gomcache() *mcache
Keith Randall4aa50432014-07-30 09:01:52 -070058
Keith Randalle359bea2014-08-06 14:33:57 -070059// An mFunction represents a C function that runs on the M stack. It
60// can be called from Go using mcall or onM. Through the magic of
61// linking, an mFunction variable and the corresponding C code entry
62// point live at the same address.
63type mFunction byte
Keith Randall4aa50432014-07-30 09:01:52 -070064
Keith Randalle359bea2014-08-06 14:33:57 -070065// in asm_*.s
66func mcall(fn *mFunction)
67func onM(fn *mFunction)
68
69// C functions that run on the M stack. Call these like
70// mcall(&mcacheRefill_m)
Keith Randall4aa50432014-07-30 09:01:52 -070071// Arguments should be passed in m->scalararg[x] and
72// m->ptrarg[x]. Return values can be passed in those
73// same slots.
Keith Randalle359bea2014-08-06 14:33:57 -070074var (
75 mcacheRefill_m,
76 largeAlloc_m,
77 mprofMalloc_m,
78 gc_m,
Dmitriy Vyukov9f38b6c2014-08-29 18:44:38 +040079 scavenge_m,
Keith Randalle359bea2014-08-06 14:33:57 -070080 setFinalizer_m,
Keith Randallc46bcd42014-08-28 13:23:10 -070081 removeFinalizer_m,
Dmitriy Vyukovaac7f1a2014-08-07 13:34:30 +040082 markallocated_m,
83 unrollgcprog_m,
Dmitriy Vyukov5d407422014-08-19 11:49:59 +040084 unrollgcproginplace_m,
Dmitriy Vyukov684de042014-08-21 20:41:09 +040085 gosched_m,
Sanjay Menakuruef504622014-08-24 20:27:00 -070086 setgcpercent_m,
87 setmaxthreads_m,
Dmitriy Vyukov684de042014-08-21 20:41:09 +040088 ready_m,
Russ Cox3a7f6642014-08-29 16:20:48 -040089 park_m mFunction
Keith Randalle359bea2014-08-06 14:33:57 -070090)
Keith Randall0c6b55e2014-07-16 14:16:19 -070091
Russ Coxd21638b2014-08-27 21:59:49 -040092func blockevent(int64, int32)
93
Keith Randall0c6b55e2014-07-16 14:16:19 -070094// memclr clears n bytes starting at ptr.
95// in memclr_*.s
Keith Randall483cb612014-08-07 13:58:42 -070096//go:noescape
Keith Randall0c6b55e2014-07-16 14:16:19 -070097func memclr(ptr unsafe.Pointer, n uintptr)
98
Keith Randall4aa50432014-07-30 09:01:52 -070099func racemalloc(p unsafe.Pointer, size uintptr)
100func tracealloc(p unsafe.Pointer, size uintptr, typ *_type)
101
Keith Randall0c6b55e2014-07-16 14:16:19 -0700102// memmove copies n bytes from "from" to "to".
103// in memmove_*.s
Keith Randall483cb612014-08-07 13:58:42 -0700104//go:noescape
Keith Randall0c6b55e2014-07-16 14:16:19 -0700105func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
106
107// in asm_*.s
108func fastrand2() uint32
109
Keith Randall4aa50432014-07-30 09:01:52 -0700110const (
Dmitriy Vyukov9f38b6c2014-08-29 18:44:38 +0400111 concurrentSweep = true
Keith Randall4aa50432014-07-30 09:01:52 -0700112)
113
Keith Randall4aa50432014-07-30 09:01:52 -0700114func gosched()
115func starttheworld()
116func stoptheworld()
117func clearpools()
118
Keith Randall0c6b55e2014-07-16 14:16:19 -0700119// exported value for testing
120var hashLoad = loadFactor
121
122// in asm_*.s
123//go:noescape
Keith Randall7aa4e5a2014-08-07 14:52:55 -0700124func memeq(a, b unsafe.Pointer, size uintptr) bool
Keith Randall0c6b55e2014-07-16 14:16:19 -0700125
Keith Randall483cb612014-08-07 13:58:42 -0700126// Code pointers for the nohash/noequal algorithms. Used for producing better error messages.
Keith Randall0c6b55e2014-07-16 14:16:19 -0700127var nohashcode uintptr
Keith Randall483cb612014-08-07 13:58:42 -0700128var noequalcode uintptr
Dave Cheneyec5d7ba2014-07-18 16:30:38 +1000129
130// Go version of runtime.throw.
131// in panic.c
Dave Cheney355c38d2014-07-23 07:08:52 +1000132func gothrow(s string)
Keith Randall4aa50432014-07-30 09:01:52 -0700133
Keith Randalla2a97682014-07-31 15:07:05 -0700134// noescape hides a pointer from escape analysis. noescape is
135// the identity function but escape analysis doesn't think the
136// output depends on the input. noescape is inlined and currently
137// compiles down to a single xor instruction.
138// USE CAREFULLY!
139func noescape(p unsafe.Pointer) unsafe.Pointer {
140 x := uintptr(p)
141 return unsafe.Pointer(x ^ 0)
142}
Keith Randall483cb612014-08-07 13:58:42 -0700143
Russ Cox3a7f6642014-08-29 16:20:48 -0400144func entersyscall()
145func entersyscallblock()
Dmitriy Vyukovafb22602014-08-22 22:13:01 +0400146func exitsyscall()
Rémy Oudompheng39ffa8b2014-08-26 08:34:46 +0200147
Brad Fitzpatrick9a5654a2014-08-27 09:31:32 -0700148func goroutineheader(gp *g)
Rémy Oudompheng39ffa8b2014-08-26 08:34:46 +0200149func traceback(pc, sp, lr uintptr, gp *g)
150func tracebackothers(gp *g)
Russ Cox25f6b022014-08-27 11:32:17 -0400151
152func cgocallback(fn, frame unsafe.Pointer, framesize uintptr)
153func gogo(buf *gobuf)
154func gosave(buf *gobuf)
155func open(name *byte, mode, perm int32) int32
156func read(fd int32, p unsafe.Pointer, n int32) int32
Russ Cox25f6b022014-08-27 11:32:17 -0400157func close(fd int32) int32
158func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32
159func jmpdefer(fv *funcval, argp unsafe.Pointer)
160func exit1(code int32)
161func asminit()
Russ Cox25f6b022014-08-27 11:32:17 -0400162func setg(gg *g)
163func exit(code int32)
164func breakpoint()
165func asmcgocall(fn, arg unsafe.Pointer)
166func nanotime() int64
167func usleep(usec uint32)
168func cputicks() int64
169func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer
170func munmap(addr unsafe.Pointer, n uintptr)
171func madvise(addr unsafe.Pointer, n uintptr, flags int32)
Russ Cox25f6b022014-08-27 11:32:17 -0400172func newstackcall(fv *funcval, addr unsafe.Pointer, size uint32)
Keith Randallc46bcd42014-08-28 13:23:10 -0700173func reflectcall(fn, arg unsafe.Pointer, n uint32, retoffset uint32)
Russ Cox25f6b022014-08-27 11:32:17 -0400174func procyield(cycles uint32)
175func osyield()
176func cgocallback_gofunc(fv *funcval, frame unsafe.Pointer, framesize uintptr)
Russ Coxd21638b2014-08-27 21:59:49 -0400177func persistentalloc(size, align uintptr, stat *uint64) unsafe.Pointer
178func readgogc() int32
Keith Randallc46bcd42014-08-28 13:23:10 -0700179func purgecachedstats(c *mcache)
Keith Randallef64d9f2014-08-28 23:26:50 -0700180func gostringnocopy(b *byte) string
Russ Coxd21638b2014-08-27 21:59:49 -0400181
182//go:noescape
Russ Cox597b2662014-08-28 23:26:40 -0400183func write(fd uintptr, p unsafe.Pointer, n int32) int32
184
185//go:noescape
Russ Coxd21638b2014-08-27 21:59:49 -0400186func cas(ptr *uint32, old, new uint32) bool
187
188//go:noescape
189func cas64(ptr *uint64, old, new uint64) bool
190
191//go:noescape
192func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool
193
194//go:noescape
195func casuintptr(ptr *uintptr, old, new uintptr) bool
196
197//go:noescape
198func xadd(ptr *uint32, delta int32) uint32
199
200//go:noescape
201func xadd64(ptr *uint64, delta int64) uint64
202
203//go:noescape
204func xchg(ptr *uint32, new uint32) uint32
205
206//go:noescape
207func xchg64(ptr *uint64, new uint64) uint64
208
209//go:noescape
210func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer
211
212//go:noescape
213func atomicstore(ptr *uint32, val uint32)
214
215//go:noescape
216func atomicstore64(ptr *uint64, val uint64)
217
218//go:noescape
219func atomicstorep(ptr unsafe.Pointer, val unsafe.Pointer)
220
221//go:noescape
Russ Cox3a7f6642014-08-29 16:20:48 -0400222func atomicstoreuintptr(ptr *uintptr, new uintptr)
223
224//go:noescape
Russ Coxd21638b2014-08-27 21:59:49 -0400225func atomicload(ptr *uint32) uint32
226
227//go:noescape
228func atomicload64(ptr *uint64) uint64
229
230//go:noescape
231func atomicloadp(ptr unsafe.Pointer) unsafe.Pointer
232
233//go:noescape
Russ Cox3a7f6642014-08-29 16:20:48 -0400234func atomicloaduintptr(ptr *uintptr) uintptr
235
236//go:noescape
Keith Randall47d6af22014-08-30 11:03:28 -0700237func atomicloaduint(ptr *uint) uint
238
239//go:noescape
Russ Coxd21638b2014-08-27 21:59:49 -0400240func atomicor8(ptr *uint8, val uint8)
241
242//go:noescape
243func setcallerpc(argp unsafe.Pointer, pc uintptr)
244
245//go:noescape
246func getcallerpc(argp unsafe.Pointer) uintptr
247
248//go:noescape
249func getcallersp(argp unsafe.Pointer) uintptr