blob: 9591437eec747520210b70706da190f0616a8b0d [file] [log] [blame]
Rob Pike8e82a672008-06-30 11:50:36 -07001// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
Russ Cox55889402011-12-19 15:51:13 -05005#include "zasm_GOOS_GOARCH.h"
Rob Pike8e82a672008-06-30 11:50:36 -07006
Russ Cox68b42552010-11-04 14:00:19 -04007TEXT _rt0_amd64(SB),7,$-8
Rob Pike8e82a672008-06-30 11:50:36 -07008 // copy arguments forward on an even stack
Devon H. O'Dell0489a262009-11-17 08:20:58 -08009 MOVQ 0(DI), AX // argc
10 LEAQ 8(DI), BX // argv
Rob Pike8e82a672008-06-30 11:50:36 -070011 SUBQ $(4*8+7), SP // 2args 2auto
Ian Lance Taylora4f8d362010-04-09 14:15:15 -070012 ANDQ $~15, SP
Rob Pike8e82a672008-06-30 11:50:36 -070013 MOVQ AX, 16(SP)
14 MOVQ BX, 24(SP)
Dmitriy Vyukov428062d2011-12-07 16:53:17 +030015
16 // create istack out of the given (operating system) stack.
17 // initcgo may update stackguard.
18 MOVQ $runtime·g0(SB), DI
Alex Brainman8d6958f2012-01-20 12:59:44 +110019 LEAQ (-64*1024+104)(SP), BX
Dmitriy Vyukov428062d2011-12-07 16:53:17 +030020 MOVQ BX, g_stackguard(DI)
21 MOVQ SP, g_stackbase(DI)
Rob Pike8e82a672008-06-30 11:50:36 -070022
Ian Lance Taylora4f8d362010-04-09 14:15:15 -070023 // if there is an initcgo, call it.
24 MOVQ initcgo(SB), AX
25 TESTQ AX, AX
Russ Coxe473f422010-08-04 17:50:22 -070026 JZ needtls
Alex Brainman8d6958f2012-01-20 12:59:44 +110027 // g0 already in DI
28 MOVQ DI, CX // Win64 uses CX for first parameter
29 CALL AX
Wei Guangjing9f636592011-07-19 10:47:33 -040030 CMPL runtime·iswindows(SB), $0
31 JEQ ok
Ian Lance Taylora4f8d362010-04-09 14:15:15 -070032
Russ Coxe473f422010-08-04 17:50:22 -070033needtls:
Akshat Kumara72bebf2012-08-31 13:21:13 -040034 // skip TLS setup on Plan 9
35 CMPL runtime·isplan9(SB), $1
36 JEQ ok
37
Russ Cox68b42552010-11-04 14:00:19 -040038 LEAQ runtime·tls0(SB), DI
39 CALL runtime·settls(SB)
Russ Coxe473f422010-08-04 17:50:22 -070040
41 // store through it, to make sure it works
42 get_tls(BX)
43 MOVQ $0x123, g(BX)
Russ Cox68b42552010-11-04 14:00:19 -040044 MOVQ runtime·tls0(SB), AX
Russ Coxe473f422010-08-04 17:50:22 -070045 CMPQ AX, $0x123
46 JEQ 2(PC)
47 MOVL AX, 0 // abort
48ok:
49 // set the per-goroutine and per-mach "registers"
50 get_tls(BX)
Russ Cox68b42552010-11-04 14:00:19 -040051 LEAQ runtime·g0(SB), CX
Russ Coxe473f422010-08-04 17:50:22 -070052 MOVQ CX, g(BX)
Russ Cox68b42552010-11-04 14:00:19 -040053 LEAQ runtime·m0(SB), AX
Russ Coxe473f422010-08-04 17:50:22 -070054 MOVQ AX, m(BX)
55
56 // save m->g0 = g0
57 MOVQ CX, m_g0(AX)
Rob Pike8e82a672008-06-30 11:50:36 -070058
Ken Thompson8f53bc02008-12-15 15:07:35 -080059 CLD // convention is D is always left cleared
Russ Cox68b42552010-11-04 14:00:19 -040060 CALL runtime·check(SB)
Rob Pike8e82a672008-06-30 11:50:36 -070061
Rob Pike8e82a672008-06-30 11:50:36 -070062 MOVL 16(SP), AX // copy argc
63 MOVL AX, 0(SP)
64 MOVQ 24(SP), AX // copy argv
65 MOVQ AX, 8(SP)
Russ Cox68b42552010-11-04 14:00:19 -040066 CALL runtime·args(SB)
67 CALL runtime·osinit(SB)
68 CALL runtime·schedinit(SB)
Russ Coxf7f63292008-08-05 14:21:42 -070069
Ken Thompson751ce3a2008-07-11 19:16:39 -070070 // create a new goroutine to start program
Russ Cox1903ad72013-02-21 17:01:13 -050071 PUSHQ $runtime·main·f(SB) // entry
Russ Cox7343e032009-06-17 15:12:16 -070072 PUSHQ $0 // arg size
Russ Cox68b42552010-11-04 14:00:19 -040073 CALL runtime·newproc(SB)
Russ Coxebd1eef2008-09-22 13:47:59 -070074 POPQ AX
75 POPQ AX
Russ Cox79e1db22008-12-04 08:30:54 -080076
Russ Coxebd1eef2008-09-22 13:47:59 -070077 // start this M
Russ Cox68b42552010-11-04 14:00:19 -040078 CALL runtime·mstart(SB)
Rob Pike8e82a672008-06-30 11:50:36 -070079
Russ Cox36aa7d42012-03-08 14:03:56 -050080 MOVL $0xf1, 0xf1 // crash
Rob Pike8e82a672008-06-30 11:50:36 -070081 RET
82
Russ Cox1903ad72013-02-21 17:01:13 -050083DATA runtime·main·f+0(SB)/8,$runtime·main(SB)
84GLOBL runtime·main·f(SB),8,$8
85
Russ Cox68b42552010-11-04 14:00:19 -040086TEXT runtime·breakpoint(SB),7,$0
Ken Thompson751ce3a2008-07-11 19:16:39 -070087 BYTE $0xcc
Rob Pike8e82a672008-06-30 11:50:36 -070088 RET
89
Russ Cox1707a992012-02-14 01:23:15 -050090TEXT runtime·asminit(SB),7,$0
91 // No per-thread init.
92 RET
93
Ken Thompson751ce3a2008-07-11 19:16:39 -070094/*
95 * go-routine
96 */
Rob Piked3204ef2008-06-30 14:39:47 -070097
Russ Coxf9ca3b52011-03-07 10:37:42 -050098// void gosave(Gobuf*)
Russ Cox7343e032009-06-17 15:12:16 -070099// save state in Gobuf; setjmp
Russ Cox68b42552010-11-04 14:00:19 -0400100TEXT runtime·gosave(SB), 7, $0
Ken Thompson751ce3a2008-07-11 19:16:39 -0700101 MOVQ 8(SP), AX // gobuf
Russ Cox7343e032009-06-17 15:12:16 -0700102 LEAQ 8(SP), BX // caller's SP
103 MOVQ BX, gobuf_sp(AX)
104 MOVQ 0(SP), BX // caller's PC
105 MOVQ BX, gobuf_pc(AX)
Russ Coxe473f422010-08-04 17:50:22 -0700106 get_tls(CX)
107 MOVQ g(CX), BX
108 MOVQ BX, gobuf_g(AX)
Ken Thompson751ce3a2008-07-11 19:16:39 -0700109 RET
110
Russ Cox7343e032009-06-17 15:12:16 -0700111// void gogo(Gobuf*, uintptr)
112// restore state from Gobuf; longjmp
Russ Cox68b42552010-11-04 14:00:19 -0400113TEXT runtime·gogo(SB), 7, $0
Russ Cox7343e032009-06-17 15:12:16 -0700114 MOVQ 16(SP), AX // return 2nd arg
115 MOVQ 8(SP), BX // gobuf
Russ Coxe473f422010-08-04 17:50:22 -0700116 MOVQ gobuf_g(BX), DX
117 MOVQ 0(DX), CX // make sure g != nil
118 get_tls(CX)
119 MOVQ DX, g(CX)
Russ Cox7343e032009-06-17 15:12:16 -0700120 MOVQ gobuf_sp(BX), SP // restore SP
121 MOVQ gobuf_pc(BX), BX
122 JMP BX
123
Russ Cox6066fdc2013-02-22 10:47:54 -0500124// void gogocall(Gobuf*, void (*fn)(void), uintptr r0)
Russ Cox7343e032009-06-17 15:12:16 -0700125// restore state from Gobuf but then call fn.
126// (call fn, returning to state in Gobuf)
Russ Cox68b42552010-11-04 14:00:19 -0400127TEXT runtime·gogocall(SB), 7, $0
Russ Cox6066fdc2013-02-22 10:47:54 -0500128 MOVQ 24(SP), DX // context
Russ Cox7343e032009-06-17 15:12:16 -0700129 MOVQ 16(SP), AX // fn
130 MOVQ 8(SP), BX // gobuf
Russ Cox6066fdc2013-02-22 10:47:54 -0500131 MOVQ gobuf_g(BX), DI
Russ Coxe473f422010-08-04 17:50:22 -0700132 get_tls(CX)
Russ Cox6066fdc2013-02-22 10:47:54 -0500133 MOVQ DI, g(CX)
134 MOVQ 0(DI), CX // make sure g != nil
Russ Cox7343e032009-06-17 15:12:16 -0700135 MOVQ gobuf_sp(BX), SP // restore SP
136 MOVQ gobuf_pc(BX), BX
137 PUSHQ BX
138 JMP AX
139 POPQ BX // not reached
140
Russ Cox1903ad72013-02-21 17:01:13 -0500141// void gogocallfn(Gobuf*, FuncVal*)
142// restore state from Gobuf but then call fn.
143// (call fn, returning to state in Gobuf)
144TEXT runtime·gogocallfn(SB), 7, $0
Russ Cox6066fdc2013-02-22 10:47:54 -0500145 MOVQ 16(SP), DX // fn
Russ Cox1903ad72013-02-21 17:01:13 -0500146 MOVQ 8(SP), BX // gobuf
Russ Cox6066fdc2013-02-22 10:47:54 -0500147 MOVQ gobuf_g(BX), AX
Russ Cox1903ad72013-02-21 17:01:13 -0500148 get_tls(CX)
Russ Cox6066fdc2013-02-22 10:47:54 -0500149 MOVQ AX, g(CX)
150 MOVQ 0(AX), CX // make sure g != nil
Russ Cox1903ad72013-02-21 17:01:13 -0500151 MOVQ gobuf_sp(BX), SP // restore SP
152 MOVQ gobuf_pc(BX), BX
153 PUSHQ BX
Russ Cox6066fdc2013-02-22 10:47:54 -0500154 MOVQ 0(DX), BX
Russ Cox1903ad72013-02-21 17:01:13 -0500155 JMP BX
156 POPQ BX // not reached
157
Russ Coxf9ca3b52011-03-07 10:37:42 -0500158// void mcall(void (*fn)(G*))
159// Switch to m->g0's stack, call fn(g).
Russ Cox370276a2011-04-27 23:21:12 -0400160// Fn must never return. It should gogo(&g->sched)
Russ Coxf9ca3b52011-03-07 10:37:42 -0500161// to keep running g.
162TEXT runtime·mcall(SB), 7, $0
163 MOVQ fn+0(FP), DI
164
165 get_tls(CX)
166 MOVQ g(CX), AX // save state in g->gobuf
167 MOVQ 0(SP), BX // caller's PC
168 MOVQ BX, (g_sched+gobuf_pc)(AX)
169 LEAQ 8(SP), BX // caller's SP
170 MOVQ BX, (g_sched+gobuf_sp)(AX)
171 MOVQ AX, (g_sched+gobuf_g)(AX)
172
173 // switch to m->g0 & its stack, call fn
174 MOVQ m(CX), BX
175 MOVQ m_g0(BX), SI
176 CMPQ SI, AX // if g == m->g0 call badmcall
177 JNE 2(PC)
178 CALL runtime·badmcall(SB)
179 MOVQ SI, g(CX) // g = m->g0
180 MOVQ (g_sched+gobuf_sp)(SI), SP // sp = m->g0->gobuf.sp
181 PUSHQ AX
182 CALL DI
183 POPQ AX
184 CALL runtime·badmcall2(SB)
185 RET
186
Rob Pike2da97832008-07-12 11:30:53 -0700187/*
188 * support for morestack
189 */
190
Russ Cox7343e032009-06-17 15:12:16 -0700191// Called during function prolog when more stack is needed.
Russ Coxe473f422010-08-04 17:50:22 -0700192// Caller has already done get_tls(CX); MOVQ m(CX), BX.
Russ Cox68b42552010-11-04 14:00:19 -0400193TEXT runtime·morestack(SB),7,$0
Russ Coxe473f422010-08-04 17:50:22 -0700194 // Cannot grow scheduler stack (m->g0).
195 MOVQ m_g0(BX), SI
196 CMPQ g(CX), SI
197 JNE 2(PC)
198 INT $3
Russ Cox6066fdc2013-02-22 10:47:54 -0500199
200 MOVQ DX, m_cret(BX)
Russ Coxe473f422010-08-04 17:50:22 -0700201
Russ Cox7343e032009-06-17 15:12:16 -0700202 // Called from f.
203 // Set m->morebuf to f's caller.
204 MOVQ 8(SP), AX // f's caller's PC
Russ Coxe473f422010-08-04 17:50:22 -0700205 MOVQ AX, (m_morebuf+gobuf_pc)(BX)
Russ Cox7343e032009-06-17 15:12:16 -0700206 LEAQ 16(SP), AX // f's caller's SP
Russ Coxe473f422010-08-04 17:50:22 -0700207 MOVQ AX, (m_morebuf+gobuf_sp)(BX)
Russ Cox141a4a12011-01-14 14:05:20 -0500208 MOVQ AX, m_moreargp(BX)
Russ Coxe473f422010-08-04 17:50:22 -0700209 get_tls(CX)
210 MOVQ g(CX), SI
211 MOVQ SI, (m_morebuf+gobuf_g)(BX)
Russ Cox7343e032009-06-17 15:12:16 -0700212
213 // Set m->morepc to f's PC.
214 MOVQ 0(SP), AX
Russ Coxe473f422010-08-04 17:50:22 -0700215 MOVQ AX, m_morepc(BX)
Russ Cox7343e032009-06-17 15:12:16 -0700216
Russ Coxf9ca3b52011-03-07 10:37:42 -0500217 // Call newstack on m->g0's stack.
Russ Coxe473f422010-08-04 17:50:22 -0700218 MOVQ m_g0(BX), BP
219 MOVQ BP, g(CX)
Russ Coxf9ca3b52011-03-07 10:37:42 -0500220 MOVQ (g_sched+gobuf_sp)(BP), SP
Russ Cox68b42552010-11-04 14:00:19 -0400221 CALL runtime·newstack(SB)
Russ Cox7343e032009-06-17 15:12:16 -0700222 MOVQ $0, 0x1003 // crash if newstack returns
223 RET
224
Russ Coxbba278a2009-07-08 18:16:09 -0700225// Called from reflection library. Mimics morestack,
226// reuses stack growth code to create a frame
227// with the desired args running the desired function.
228//
229// func call(fn *byte, arg *byte, argsize uint32).
230TEXT reflect·call(SB), 7, $0
Russ Coxe473f422010-08-04 17:50:22 -0700231 get_tls(CX)
232 MOVQ m(CX), BX
233
Russ Coxbba278a2009-07-08 18:16:09 -0700234 // Save our caller's state as the PC and SP to
235 // restore when returning from f.
236 MOVQ 0(SP), AX // our caller's PC
Russ Coxe473f422010-08-04 17:50:22 -0700237 MOVQ AX, (m_morebuf+gobuf_pc)(BX)
Russ Coxbba278a2009-07-08 18:16:09 -0700238 LEAQ 8(SP), AX // our caller's SP
Russ Coxe473f422010-08-04 17:50:22 -0700239 MOVQ AX, (m_morebuf+gobuf_sp)(BX)
240 MOVQ g(CX), AX
241 MOVQ AX, (m_morebuf+gobuf_g)(BX)
Russ Coxbba278a2009-07-08 18:16:09 -0700242
243 // Set up morestack arguments to call f on a new stack.
Russ Cox83727cc2010-03-29 21:48:22 -0700244 // We set f's frame size to 1, as a hint to newstack
245 // that this is a call from reflect·call.
246 // If it turns out that f needs a larger frame than
247 // the default stack, f's usual stack growth prolog will
248 // allocate a new segment (and recopy the arguments).
Russ Coxbba278a2009-07-08 18:16:09 -0700249 MOVQ 8(SP), AX // fn
Russ Coxe473f422010-08-04 17:50:22 -0700250 MOVQ 16(SP), DX // arg frame
Russ Coxbba278a2009-07-08 18:16:09 -0700251 MOVL 24(SP), CX // arg size
252
Russ Coxe473f422010-08-04 17:50:22 -0700253 MOVQ AX, m_morepc(BX) // f's PC
Russ Cox141a4a12011-01-14 14:05:20 -0500254 MOVQ DX, m_moreargp(BX) // argument frame pointer
255 MOVL CX, m_moreargsize(BX) // f's argument size
256 MOVL $1, m_moreframesize(BX) // f's frame size
Russ Coxbba278a2009-07-08 18:16:09 -0700257
Russ Coxf9ca3b52011-03-07 10:37:42 -0500258 // Call newstack on m->g0's stack.
Russ Coxe473f422010-08-04 17:50:22 -0700259 MOVQ m_g0(BX), BP
260 get_tls(CX)
261 MOVQ BP, g(CX)
Russ Coxf9ca3b52011-03-07 10:37:42 -0500262 MOVQ (g_sched+gobuf_sp)(BP), SP
Russ Cox68b42552010-11-04 14:00:19 -0400263 CALL runtime·newstack(SB)
Russ Coxbba278a2009-07-08 18:16:09 -0700264 MOVQ $0, 0x1103 // crash if newstack returns
265 RET
266
Russ Cox7343e032009-06-17 15:12:16 -0700267// Return point when leaving stack.
Russ Cox68b42552010-11-04 14:00:19 -0400268TEXT runtime·lessstack(SB), 7, $0
Russ Cox7343e032009-06-17 15:12:16 -0700269 // Save return value in m->cret
Russ Coxe473f422010-08-04 17:50:22 -0700270 get_tls(CX)
271 MOVQ m(CX), BX
272 MOVQ AX, m_cret(BX)
Russ Cox7343e032009-06-17 15:12:16 -0700273
Russ Coxf9ca3b52011-03-07 10:37:42 -0500274 // Call oldstack on m->g0's stack.
275 MOVQ m_g0(BX), BP
276 MOVQ BP, g(CX)
277 MOVQ (g_sched+gobuf_sp)(BP), SP
Russ Cox68b42552010-11-04 14:00:19 -0400278 CALL runtime·oldstack(SB)
Russ Cox7343e032009-06-17 15:12:16 -0700279 MOVQ $0, 0x1004 // crash if oldstack returns
280 RET
281
Ken Thompson1ed7f182009-05-01 18:07:33 -0700282// morestack trampolines
Russ Cox68b42552010-11-04 14:00:19 -0400283TEXT runtime·morestack00(SB),7,$0
Russ Coxe473f422010-08-04 17:50:22 -0700284 get_tls(CX)
285 MOVQ m(CX), BX
Ken Thompson1ed7f182009-05-01 18:07:33 -0700286 MOVQ $0, AX
Russ Cox141a4a12011-01-14 14:05:20 -0500287 MOVQ AX, m_moreframesize(BX)
Russ Cox68b42552010-11-04 14:00:19 -0400288 MOVQ $runtime·morestack(SB), AX
Ken Thompson1ed7f182009-05-01 18:07:33 -0700289 JMP AX
290
Russ Cox68b42552010-11-04 14:00:19 -0400291TEXT runtime·morestack01(SB),7,$0
Russ Coxe473f422010-08-04 17:50:22 -0700292 get_tls(CX)
293 MOVQ m(CX), BX
Ken Thompson1ed7f182009-05-01 18:07:33 -0700294 SHLQ $32, AX
Russ Cox141a4a12011-01-14 14:05:20 -0500295 MOVQ AX, m_moreframesize(BX)
Russ Cox68b42552010-11-04 14:00:19 -0400296 MOVQ $runtime·morestack(SB), AX
Ken Thompson1ed7f182009-05-01 18:07:33 -0700297 JMP AX
298
Russ Cox68b42552010-11-04 14:00:19 -0400299TEXT runtime·morestack10(SB),7,$0
Russ Coxe473f422010-08-04 17:50:22 -0700300 get_tls(CX)
301 MOVQ m(CX), BX
Ken Thompson1ed7f182009-05-01 18:07:33 -0700302 MOVLQZX AX, AX
Russ Cox141a4a12011-01-14 14:05:20 -0500303 MOVQ AX, m_moreframesize(BX)
Russ Cox68b42552010-11-04 14:00:19 -0400304 MOVQ $runtime·morestack(SB), AX
Ken Thompson1ed7f182009-05-01 18:07:33 -0700305 JMP AX
306
Russ Cox68b42552010-11-04 14:00:19 -0400307TEXT runtime·morestack11(SB),7,$0
Russ Coxe473f422010-08-04 17:50:22 -0700308 get_tls(CX)
309 MOVQ m(CX), BX
Russ Cox141a4a12011-01-14 14:05:20 -0500310 MOVQ AX, m_moreframesize(BX)
Russ Cox68b42552010-11-04 14:00:19 -0400311 MOVQ $runtime·morestack(SB), AX
Ken Thompson1ed7f182009-05-01 18:07:33 -0700312 JMP AX
313
Ken Thompson5963f592009-05-03 19:09:14 -0700314// subcases of morestack01
315// with const of 8,16,...48
Russ Cox68b42552010-11-04 14:00:19 -0400316TEXT runtime·morestack8(SB),7,$0
Ken Thompson5963f592009-05-03 19:09:14 -0700317 PUSHQ $1
Russ Cox68b42552010-11-04 14:00:19 -0400318 MOVQ $morestack<>(SB), AX
Ken Thompson5963f592009-05-03 19:09:14 -0700319 JMP AX
320
Russ Cox68b42552010-11-04 14:00:19 -0400321TEXT runtime·morestack16(SB),7,$0
Ken Thompson5963f592009-05-03 19:09:14 -0700322 PUSHQ $2
Russ Cox68b42552010-11-04 14:00:19 -0400323 MOVQ $morestack<>(SB), AX
Ken Thompson5963f592009-05-03 19:09:14 -0700324 JMP AX
325
Russ Cox68b42552010-11-04 14:00:19 -0400326TEXT runtime·morestack24(SB),7,$0
Ken Thompson5963f592009-05-03 19:09:14 -0700327 PUSHQ $3
Russ Cox68b42552010-11-04 14:00:19 -0400328 MOVQ $morestack<>(SB), AX
Ken Thompson5963f592009-05-03 19:09:14 -0700329 JMP AX
330
Russ Cox68b42552010-11-04 14:00:19 -0400331TEXT runtime·morestack32(SB),7,$0
Ken Thompson5963f592009-05-03 19:09:14 -0700332 PUSHQ $4
Russ Cox68b42552010-11-04 14:00:19 -0400333 MOVQ $morestack<>(SB), AX
Ken Thompson5963f592009-05-03 19:09:14 -0700334 JMP AX
335
Russ Cox68b42552010-11-04 14:00:19 -0400336TEXT runtime·morestack40(SB),7,$0
Ken Thompson5963f592009-05-03 19:09:14 -0700337 PUSHQ $5
Russ Cox68b42552010-11-04 14:00:19 -0400338 MOVQ $morestack<>(SB), AX
Ken Thompson5963f592009-05-03 19:09:14 -0700339 JMP AX
340
Russ Cox68b42552010-11-04 14:00:19 -0400341TEXT runtime·morestack48(SB),7,$0
Ken Thompson5963f592009-05-03 19:09:14 -0700342 PUSHQ $6
Russ Cox68b42552010-11-04 14:00:19 -0400343 MOVQ $morestack<>(SB), AX
Ken Thompson5963f592009-05-03 19:09:14 -0700344 JMP AX
345
Russ Cox68b42552010-11-04 14:00:19 -0400346TEXT morestack<>(SB),7,$0
Russ Coxe473f422010-08-04 17:50:22 -0700347 get_tls(CX)
348 MOVQ m(CX), BX
Russ Cox7343e032009-06-17 15:12:16 -0700349 POPQ AX
350 SHLQ $35, AX
Russ Cox141a4a12011-01-14 14:05:20 -0500351 MOVQ AX, m_moreframesize(BX)
Russ Cox68b42552010-11-04 14:00:19 -0400352 MOVQ $runtime·morestack(SB), AX
Rob Pike2da97832008-07-12 11:30:53 -0700353 JMP AX
354
Russ Coxd28acc42008-08-04 16:43:49 -0700355// bool cas(int32 *val, int32 old, int32 new)
356// Atomically:
357// if(*val == old){
358// *val = new;
359// return 1;
Ken Thompson1e1cc4e2009-01-27 12:03:53 -0800360// } else
Russ Coxd28acc42008-08-04 16:43:49 -0700361// return 0;
Russ Cox68b42552010-11-04 14:00:19 -0400362TEXT runtime·cas(SB), 7, $0
Russ Coxd28acc42008-08-04 16:43:49 -0700363 MOVQ 8(SP), BX
364 MOVL 16(SP), AX
365 MOVL 20(SP), CX
366 LOCK
367 CMPXCHGL CX, 0(BX)
368 JZ 3(PC)
369 MOVL $0, AX
370 RET
371 MOVL $1, AX
372 RET
Ken Thompson1e1cc4e2009-01-27 12:03:53 -0800373
Dmitriy Vyukov46675712012-04-05 18:47:43 +0400374// bool runtime·cas64(uint64 *val, uint64 *old, uint64 new)
375// Atomically:
376// if(*val == *old){
377// *val = new;
378// return 1;
379// } else {
380// *old = *val
381// return 0;
382// }
383TEXT runtime·cas64(SB), 7, $0
384 MOVQ 8(SP), BX
385 MOVQ 16(SP), BP
386 MOVQ 0(BP), AX
387 MOVQ 24(SP), CX
388 LOCK
389 CMPXCHGQ CX, 0(BX)
390 JNZ cas64_fail
391 MOVL $1, AX
392 RET
393cas64_fail:
394 MOVQ AX, 0(BP)
395 MOVL $0, AX
396 RET
397
Russ Cox67793502011-02-16 13:21:13 -0500398// bool casp(void **val, void *old, void *new)
399// Atomically:
400// if(*val == old){
401// *val = new;
402// return 1;
403// } else
404// return 0;
405TEXT runtime·casp(SB), 7, $0
406 MOVQ 8(SP), BX
407 MOVQ 16(SP), AX
408 MOVQ 24(SP), CX
409 LOCK
410 CMPXCHGQ CX, 0(BX)
411 JZ 3(PC)
412 MOVL $0, AX
413 RET
414 MOVL $1, AX
415 RET
416
Dmitriy Vyukov491aa152011-07-15 11:27:16 -0400417// uint32 xadd(uint32 volatile *val, int32 delta)
418// Atomically:
419// *val += delta;
420// return *val;
421TEXT runtime·xadd(SB), 7, $0
422 MOVQ 8(SP), BX
423 MOVL 16(SP), AX
424 MOVL AX, CX
425 LOCK
426 XADDL AX, 0(BX)
427 ADDL CX, AX
428 RET
429
Dmitriy Vyukov46675712012-04-05 18:47:43 +0400430TEXT runtime·xadd64(SB), 7, $0
431 MOVQ 8(SP), BX
432 MOVQ 16(SP), AX
433 MOVQ AX, CX
434 LOCK
435 XADDQ AX, 0(BX)
436 ADDQ CX, AX
437 RET
438
Dmitriy Vyukov4e5086b2011-07-29 12:44:06 -0400439TEXT runtime·xchg(SB), 7, $0
440 MOVQ 8(SP), BX
441 MOVL 16(SP), AX
442 XCHGL AX, 0(BX)
443 RET
444
445TEXT runtime·procyield(SB),7,$0
446 MOVL 8(SP), AX
447again:
448 PAUSE
449 SUBL $1, AX
450 JNZ again
451 RET
452
Dmitriy Vyukov86a659c2011-07-13 11:22:41 -0700453TEXT runtime·atomicstorep(SB), 7, $0
454 MOVQ 8(SP), BX
455 MOVQ 16(SP), AX
456 XCHGQ AX, 0(BX)
457 RET
458
Dmitriy Vyukov91f0f182011-07-29 13:47:24 -0400459TEXT runtime·atomicstore(SB), 7, $0
460 MOVQ 8(SP), BX
461 MOVL 16(SP), AX
462 XCHGL AX, 0(BX)
463 RET
464
Dmitriy Vyukov46675712012-04-05 18:47:43 +0400465TEXT runtime·atomicstore64(SB), 7, $0
466 MOVQ 8(SP), BX
467 MOVQ 16(SP), AX
468 XCHGQ AX, 0(BX)
469 RET
470
Russ Coxaa3222d82009-06-02 23:02:12 -0700471// void jmpdefer(fn, sp);
472// called from deferreturn.
Ken Thompson1e1cc4e2009-01-27 12:03:53 -0800473// 1. pop the caller
474// 2. sub 5 bytes from the callers return
475// 3. jmp to the argument
Russ Cox68b42552010-11-04 14:00:19 -0400476TEXT runtime·jmpdefer(SB), 7, $0
Russ Cox6066fdc2013-02-22 10:47:54 -0500477 MOVQ 8(SP), DX // fn
Russ Coxaa3222d82009-06-02 23:02:12 -0700478 MOVQ 16(SP), BX // caller sp
479 LEAQ -8(BX), SP // caller sp after CALL
480 SUBQ $5, (SP) // return to CALL again
Russ Cox6066fdc2013-02-22 10:47:54 -0500481 MOVQ 0(DX), BX
Russ Cox1903ad72013-02-21 17:01:13 -0500482 JMP BX // but first run the deferred function
Russ Cox133a1582009-10-03 10:37:12 -0700483
Russ Coxf9ca3b52011-03-07 10:37:42 -0500484// Dummy function to use in saved gobuf.PC,
485// to match SP pointing at a return address.
486// The gobuf.PC is unused by the contortions here
487// but setting it to return will make the traceback code work.
488TEXT return<>(SB),7,$0
489 RET
490
491// asmcgocall(void(*fn)(void*), void *arg)
Russ Coxadd89dd2009-10-12 10:26:38 -0700492// Call fn(arg) on the scheduler stack,
493// aligned appropriately for the gcc ABI.
Russ Coxf9ca3b52011-03-07 10:37:42 -0500494// See cgocall.c for more details.
495TEXT runtime·asmcgocall(SB),7,$0
496 MOVQ fn+0(FP), AX
497 MOVQ arg+8(FP), BX
498 MOVQ SP, DX
Russ Coxadd89dd2009-10-12 10:26:38 -0700499
500 // Figure out if we need to switch to m->g0 stack.
Russ Coxf9ca3b52011-03-07 10:37:42 -0500501 // We get called to create new OS threads too, and those
502 // come in on the m->g0 stack already.
503 get_tls(CX)
504 MOVQ m(CX), BP
505 MOVQ m_g0(BP), SI
506 MOVQ g(CX), DI
507 CMPQ SI, DI
508 JEQ 6(PC)
509 MOVQ SP, (g_sched+gobuf_sp)(DI)
510 MOVQ $return<>(SB), (g_sched+gobuf_pc)(DI)
511 MOVQ DI, (g_sched+gobuf_g)(DI)
512 MOVQ SI, g(CX)
513 MOVQ (g_sched+gobuf_sp)(SI), SP
Russ Coxadd89dd2009-10-12 10:26:38 -0700514
515 // Now on a scheduling stack (a pthread-created stack).
Alex Brainman7f075ec2012-09-03 12:12:51 +1000516 // Make sure we have enough room for 4 stack-backed fast-call
517 // registers as per windows amd64 calling convention.
518 SUBQ $64, SP
Russ Cox133a1582009-10-03 10:37:12 -0700519 ANDQ $~15, SP // alignment for gcc ABI
Alex Brainman7f075ec2012-09-03 12:12:51 +1000520 MOVQ DI, 48(SP) // save g
521 MOVQ DX, 40(SP) // save SP
Russ Coxf9ca3b52011-03-07 10:37:42 -0500522 MOVQ BX, DI // DI = first argument in AMD64 ABI
Wei Guangjing9f636592011-07-19 10:47:33 -0400523 MOVQ BX, CX // CX = first argument in Win64
Russ Coxf9ca3b52011-03-07 10:37:42 -0500524 CALL AX
Russ Coxadd89dd2009-10-12 10:26:38 -0700525
Russ Coxe473f422010-08-04 17:50:22 -0700526 // Restore registers, g, stack pointer.
Russ Coxf9ca3b52011-03-07 10:37:42 -0500527 get_tls(CX)
Alex Brainman7f075ec2012-09-03 12:12:51 +1000528 MOVQ 48(SP), DI
Russ Coxf9ca3b52011-03-07 10:37:42 -0500529 MOVQ DI, g(CX)
Alex Brainman7f075ec2012-09-03 12:12:51 +1000530 MOVQ 40(SP), SP
Russ Cox133a1582009-10-03 10:37:12 -0700531 RET
532
Russ Coxf9ca3b52011-03-07 10:37:42 -0500533// cgocallback(void (*fn)(void*), void *frame, uintptr framesize)
Russ Cox3d2dfc52013-02-22 16:08:56 -0500534// Turn the fn into a Go func (by taking its address) and call
535// cgocallback_gofunc.
Russ Coxf9ca3b52011-03-07 10:37:42 -0500536TEXT runtime·cgocallback(SB),7,$24
Russ Cox3d2dfc52013-02-22 16:08:56 -0500537 LEAQ fn+0(FP), AX
538 MOVQ AX, 0(SP)
539 MOVQ frame+8(FP), AX
540 MOVQ AX, 8(SP)
541 MOVQ framesize+16(FP), AX
542 MOVQ AX, 16(SP)
543 MOVQ $runtime·cgocallback_gofunc(SB), AX
544 CALL AX
545 RET
546
547// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize)
548// See cgocall.c for more details.
549TEXT runtime·cgocallback_gofunc(SB),7,$24
Russ Cox6c976392013-02-20 17:48:23 -0500550 // If m is nil, Go did not create the current thread.
551 // Call needm to obtain one for temporary use.
552 // In this case, we're running on the thread stack, so there's
553 // lots of space, but the linker doesn't know. Hide the call from
554 // the linker analysis by using an indirect call through AX.
555 get_tls(CX)
556#ifdef GOOS_windows
557 CMPQ CX, $0
558 JNE 3(PC)
559 PUSHQ $0
560 JMP needm
561#endif
562 MOVQ m(CX), BP
563 PUSHQ BP
564 CMPQ BP, $0
565 JNE havem
566needm:
567 MOVQ $runtime·needm(SB), AX
568 CALL AX
Russ Coxe473f422010-08-04 17:50:22 -0700569 get_tls(CX)
Russ Coxf9ca3b52011-03-07 10:37:42 -0500570 MOVQ m(CX), BP
Russ Cox9b732382012-03-08 12:12:40 -0500571
Russ Cox6c976392013-02-20 17:48:23 -0500572havem:
573 // Now there's a valid m, and we're running on its m->g0.
574 // Save current m->g0->sched.sp on stack and then set it to SP.
575 // Save current sp in m->g0->sched.sp in preparation for
576 // switch back to m->curg stack.
Russ Coxf9ca3b52011-03-07 10:37:42 -0500577 MOVQ m_g0(BP), SI
578 PUSHQ (g_sched+gobuf_sp)(SI)
579 MOVQ SP, (g_sched+gobuf_sp)(SI)
Ian Lance Taylor2d0ff3f2010-04-09 13:30:11 -0700580
Alex Brainman72e83482011-08-18 12:17:09 -0400581 // Switch to m->curg stack and call runtime.cgocallbackg
Russ Coxf9ca3b52011-03-07 10:37:42 -0500582 // with the three arguments. Because we are taking over
583 // the execution of m->curg but *not* resuming what had
584 // been running, we need to save that information (m->curg->gobuf)
585 // so that we can restore it when we're done.
586 // We can restore m->curg->gobuf.sp easily, because calling
Alex Brainman72e83482011-08-18 12:17:09 -0400587 // runtime.cgocallbackg leaves SP unchanged upon return.
Russ Coxf9ca3b52011-03-07 10:37:42 -0500588 // To save m->curg->gobuf.pc, we push it onto the stack.
589 // This has the added benefit that it looks to the traceback
Alex Brainman72e83482011-08-18 12:17:09 -0400590 // routine like cgocallbackg is going to return to that
591 // PC (because we defined cgocallbackg to have
Russ Coxf9ca3b52011-03-07 10:37:42 -0500592 // a frame size of 24, the same amount that we use below),
593 // so that the traceback will seamlessly trace back into
594 // the earlier calls.
Russ Cox6c976392013-02-20 17:48:23 -0500595 MOVQ fn+0(FP), AX
596 MOVQ frame+8(FP), BX
597 MOVQ framesize+16(FP), DX
598
Russ Coxf9ca3b52011-03-07 10:37:42 -0500599 MOVQ m_curg(BP), SI
600 MOVQ SI, g(CX)
601 MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI
Ian Lance Taylor2d0ff3f2010-04-09 13:30:11 -0700602
Russ Coxf9ca3b52011-03-07 10:37:42 -0500603 // Push gobuf.pc
604 MOVQ (g_sched+gobuf_pc)(SI), BP
605 SUBQ $8, DI
606 MOVQ BP, 0(DI)
607
608 // Push arguments to cgocallbackg.
609 // Frame size here must match the frame size above
610 // to trick traceback routines into doing the right thing.
611 SUBQ $24, DI
612 MOVQ AX, 0(DI)
613 MOVQ BX, 8(DI)
614 MOVQ DX, 16(DI)
615
616 // Switch stack and make the call.
617 MOVQ DI, SP
618 CALL runtime·cgocallbackg(SB)
619
620 // Restore g->gobuf (== m->curg->gobuf) from saved values.
Russ Coxe473f422010-08-04 17:50:22 -0700621 get_tls(CX)
Russ Coxf9ca3b52011-03-07 10:37:42 -0500622 MOVQ g(CX), SI
623 MOVQ 24(SP), BP
624 MOVQ BP, (g_sched+gobuf_pc)(SI)
625 LEAQ (24+8)(SP), DI
626 MOVQ DI, (g_sched+gobuf_sp)(SI)
627
628 // Switch back to m->g0's stack and restore m->g0->sched.sp.
629 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
630 // so we do not have to restore it.)
631 MOVQ m(CX), BP
632 MOVQ m_g0(BP), SI
633 MOVQ SI, g(CX)
634 MOVQ (g_sched+gobuf_sp)(SI), SP
635 POPQ (g_sched+gobuf_sp)(SI)
Russ Cox6c976392013-02-20 17:48:23 -0500636
637 // If the m on entry was nil, we called needm above to borrow an m
638 // for the duration of the call. Since the call is over, return it with dropm.
639 POPQ BP
640 CMPQ BP, $0
641 JNE 3(PC)
642 MOVQ $runtime·dropm(SB), AX
643 CALL AX
Russ Coxf9ca3b52011-03-07 10:37:42 -0500644
645 // Done!
Ian Lance Taylor2d0ff3f2010-04-09 13:30:11 -0700646 RET
647
Russ Cox6c976392013-02-20 17:48:23 -0500648// void setmg(M*, G*); set m and g. for use by needm.
649TEXT runtime·setmg(SB), 7, $0
650 MOVQ mm+0(FP), AX
651#ifdef GOOS_windows
652 CMPQ AX, $0
653 JNE settls
654 MOVQ $0, 0x28(GS)
655 RET
656settls:
657 LEAQ m_tls(AX), AX
658 MOVQ AX, 0x28(GS)
659#endif
660 get_tls(CX)
661 MOVQ mm+0(FP), AX
662 MOVQ AX, m(CX)
663 MOVQ gg+8(FP), BX
664 MOVQ BX, g(CX)
665 RET
666
Devon H. O'Dell5a4a08f2009-12-08 18:19:30 -0800667// check that SP is in range [g->stackbase, g->stackguard)
Russ Cox68b42552010-11-04 14:00:19 -0400668TEXT runtime·stackcheck(SB), 7, $0
Russ Coxe473f422010-08-04 17:50:22 -0700669 get_tls(CX)
670 MOVQ g(CX), AX
671 CMPQ g_stackbase(AX), SP
Russ Cox01eaf782010-03-30 10:53:16 -0700672 JHI 2(PC)
673 INT $3
Russ Coxe473f422010-08-04 17:50:22 -0700674 CMPQ SP, g_stackguard(AX)
Russ Cox01eaf782010-03-30 10:53:16 -0700675 JHI 2(PC)
676 INT $3
677 RET
678
Russ Cox68b42552010-11-04 14:00:19 -0400679TEXT runtime·memclr(SB),7,$0
Russ Cox6c196012010-04-05 12:51:09 -0700680 MOVQ 8(SP), DI // arg 1 addr
Russ Coxafc69282011-01-25 16:35:36 -0500681 MOVQ 16(SP), CX // arg 2 count
Quan Yong Zhai47410a22011-07-23 15:46:58 -0400682 MOVQ CX, BX
683 ANDQ $7, BX
Russ Coxafc69282011-01-25 16:35:36 -0500684 SHRQ $3, CX
Russ Cox6c196012010-04-05 12:51:09 -0700685 MOVQ $0, AX
686 CLD
687 REP
688 STOSQ
Quan Yong Zhai47410a22011-07-23 15:46:58 -0400689 MOVQ BX, CX
690 REP
691 STOSB
Russ Cox6c196012010-04-05 12:51:09 -0700692 RET
693
Russ Cox68b42552010-11-04 14:00:19 -0400694TEXT runtime·getcallerpc(SB),7,$0
Russ Cox6c196012010-04-05 12:51:09 -0700695 MOVQ x+0(FP),AX // addr of first arg
696 MOVQ -8(AX),AX // get calling pc
697 RET
698
Russ Cox68b42552010-11-04 14:00:19 -0400699TEXT runtime·setcallerpc(SB),7,$0
Russ Cox6c196012010-04-05 12:51:09 -0700700 MOVQ x+0(FP),AX // addr of first arg
701 MOVQ x+8(FP), BX
702 MOVQ BX, -8(AX) // set calling pc
703 RET
704
Russ Cox68b42552010-11-04 14:00:19 -0400705TEXT runtime·getcallersp(SB),7,$0
Russ Cox6c196012010-04-05 12:51:09 -0700706 MOVQ sp+0(FP), AX
707 RET
708
Damian Gryski8e765da2012-02-02 14:09:27 -0500709// int64 runtime·cputicks(void)
710TEXT runtime·cputicks(SB),7,$0
711 RDTSC
712 SHLQ $32, DX
713 ADDQ DX, AX
714 RET
715
Russ Cox9e5db8c2012-03-15 15:22:30 -0400716TEXT runtime·stackguard(SB),7,$0
717 MOVQ SP, DX
718 MOVQ DX, sp+0(FP)
719 get_tls(CX)
720 MOVQ g(CX), BX
721 MOVQ g_stackguard(BX), DX
722 MOVQ DX, guard+8(FP)
723 RET
724
Russ Cox68b42552010-11-04 14:00:19 -0400725GLOBL runtime·tls0(SB), $64