blob: 7baef373247551d0b8de92862e271b1b256d45e6 [file] [log] [blame]
Michael Mundaye6f36f02016-03-18 19:09:39 -04001// Copyright 2016 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "go_tls.h"
7#include "funcdata.h"
8#include "textflag.h"
9
Bill O'Farrellc2efb2f2017-10-31 19:18:48 -040010// _rt0_s390x_lib is common startup code for s390x systems when
11// using -buildmode=c-archive or -buildmode=c-shared. The linker will
12// arrange to invoke this function as a global constructor (for
13// c-archive) or when the shared library is loaded (for c-shared).
14// We expect argc and argv to be passed in the usual C ABI registers
15// R2 and R3.
16TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0
17 STMG R6, R15, 48(R15)
18 MOVD R2, _rt0_s390x_lib_argc<>(SB)
19 MOVD R3, _rt0_s390x_lib_argv<>(SB)
20
21 // Save R6-R15 in the register save area of the calling function.
22 STMG R6, R15, 48(R15)
23
24 // Allocate 80 bytes on the stack.
25 MOVD $-80(R15), R15
26
27 // Save F8-F15 in our stack frame.
28 FMOVD F8, 16(R15)
29 FMOVD F9, 24(R15)
30 FMOVD F10, 32(R15)
31 FMOVD F11, 40(R15)
32 FMOVD F12, 48(R15)
33 FMOVD F13, 56(R15)
34 FMOVD F14, 64(R15)
35 FMOVD F15, 72(R15)
36
37 // Synchronous initialization.
38 MOVD $runtime·libpreinit(SB), R1
39 BL R1
40
41 // Create a new thread to finish Go runtime initialization.
42 MOVD _cgo_sys_thread_create(SB), R1
43 CMP R1, $0
44 BEQ nocgo
45 MOVD $_rt0_s390x_lib_go(SB), R2
46 MOVD $0, R3
47 BL R1
48 BR restore
49
50nocgo:
51 MOVD $0x800000, R1 // stacksize
52 MOVD R1, 0(R15)
53 MOVD $_rt0_s390x_lib_go(SB), R1
54 MOVD R1, 8(R15) // fn
55 MOVD $runtime·newosproc(SB), R1
56 BL R1
57
58restore:
59 // Restore F8-F15 from our stack frame.
60 FMOVD 16(R15), F8
61 FMOVD 24(R15), F9
62 FMOVD 32(R15), F10
63 FMOVD 40(R15), F11
64 FMOVD 48(R15), F12
65 FMOVD 56(R15), F13
66 FMOVD 64(R15), F14
67 FMOVD 72(R15), F15
68 MOVD $80(R15), R15
69
70 // Restore R6-R15.
71 LMG 48(R15), R6, R15
72 RET
73
74// _rt0_s390x_lib_go initializes the Go runtime.
75// This is started in a separate thread by _rt0_s390x_lib.
76TEXT _rt0_s390x_lib_go(SB), NOSPLIT|NOFRAME, $0
77 MOVD _rt0_s390x_lib_argc<>(SB), R2
78 MOVD _rt0_s390x_lib_argv<>(SB), R3
79 MOVD $runtime·rt0_go(SB), R1
80 BR R1
81
82DATA _rt0_s390x_lib_argc<>(SB)/8, $0
83GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8
84DATA _rt0_s90x_lib_argv<>(SB)/8, $0
85GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8
86
Michael Mundaye6f36f02016-03-18 19:09:39 -040087TEXT runtime·rt0_go(SB),NOSPLIT,$0
88 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
89 // C TLS base pointer in AR0:AR1
90
91 // initialize essential registers
92 XOR R0, R0
93
94 SUB $24, R15
95 MOVW R2, 8(R15) // argc
96 MOVD R3, 16(R15) // argv
97
98 // create istack out of the given (operating system) stack.
99 // _cgo_init may update stackguard.
100 MOVD $runtime·g0(SB), g
101 MOVD R15, R11
102 SUB $(64*1024), R11
103 MOVD R11, g_stackguard0(g)
104 MOVD R11, g_stackguard1(g)
105 MOVD R11, (g_stack+stack_lo)(g)
106 MOVD R15, (g_stack+stack_hi)(g)
107
108 // if there is a _cgo_init, call it using the gcc ABI.
109 MOVD _cgo_init(SB), R11
110 CMPBEQ R11, $0, nocgo
111 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR
112 SLD $32, R4, R4
113 MOVW AR1, R4 // arg 2: TLS base pointer
114 MOVD $setg_gcc<>(SB), R3 // arg 1: setg
115 MOVD g, R2 // arg 0: G
116 // C functions expect 160 bytes of space on caller stack frame
117 // and an 8-byte aligned stack pointer
118 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI)
119 SUB $160, R15 // reserve 160 bytes
120 MOVD $~7, R6
121 AND R6, R15 // 8-byte align
122 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14)
123 MOVD R9, R15 // restore stack
124 XOR R0, R0 // zero R0
125
126nocgo:
127 // update stackguard after _cgo_init
128 MOVD (g_stack+stack_lo)(g), R2
129 ADD $const__StackGuard, R2
130 MOVD R2, g_stackguard0(g)
131 MOVD R2, g_stackguard1(g)
132
133 // set the per-goroutine and per-mach "registers"
134 MOVD $runtime·m0(SB), R2
135
136 // save m->g0 = g0
137 MOVD g, m_g0(R2)
138 // save m0 to g0->m
139 MOVD R2, g_m(g)
140
141 BL runtime·check(SB)
142
143 // argc/argv are already prepared on stack
144 BL runtime·args(SB)
145 BL runtime·osinit(SB)
146 BL runtime·schedinit(SB)
147
148 // create a new goroutine to start program
149 MOVD $runtime·mainPC(SB), R2 // entry
150 SUB $24, R15
151 MOVD R2, 16(R15)
Michael Mundayf1515a02016-09-12 13:33:00 -0400152 MOVD $0, 8(R15)
153 MOVD $0, 0(R15)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400154 BL runtime·newproc(SB)
155 ADD $24, R15
156
157 // start this M
158 BL runtime·mstart(SB)
159
Michael Mundayf1515a02016-09-12 13:33:00 -0400160 MOVD $0, 1(R0)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400161 RET
162
163DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
164GLOBL runtime·mainPC(SB),RODATA,$8
165
166TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
Michael Mundayf1515a02016-09-12 13:33:00 -0400167 MOVD $0, 2(R0)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400168 RET
169
170TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
171 RET
172
173/*
174 * go-routine
175 */
176
177// void gosave(Gobuf*)
178// save state in Gobuf; setjmp
179TEXT runtime·gosave(SB), NOSPLIT, $-8-8
180 MOVD buf+0(FP), R3
181 MOVD R15, gobuf_sp(R3)
182 MOVD LR, gobuf_pc(R3)
183 MOVD g, gobuf_g(R3)
184 MOVD $0, gobuf_lr(R3)
185 MOVD $0, gobuf_ret(R3)
Austin Clements70c107c2016-10-19 15:49:31 -0400186 // Assert ctxt is zero. See func save.
187 MOVD gobuf_ctxt(R3), R3
188 CMPBEQ R3, $0, 2(PC)
189 BL runtime·badctxt(SB)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400190 RET
191
192// void gogo(Gobuf*)
193// restore state from Gobuf; longjmp
Austin Clements70c107c2016-10-19 15:49:31 -0400194TEXT runtime·gogo(SB), NOSPLIT, $16-8
Michael Mundaye6f36f02016-03-18 19:09:39 -0400195 MOVD buf+0(FP), R5
196 MOVD gobuf_g(R5), g // make sure g is not nil
197 BL runtime·save_g(SB)
198
199 MOVD 0(g), R4
200 MOVD gobuf_sp(R5), R15
201 MOVD gobuf_lr(R5), LR
202 MOVD gobuf_ret(R5), R3
203 MOVD gobuf_ctxt(R5), R12
204 MOVD $0, gobuf_sp(R5)
205 MOVD $0, gobuf_ret(R5)
206 MOVD $0, gobuf_lr(R5)
207 MOVD $0, gobuf_ctxt(R5)
208 CMP R0, R0 // set condition codes for == test, needed by stack split
209 MOVD gobuf_pc(R5), R6
210 BR (R6)
211
212// void mcall(fn func(*g))
213// Switch to m->g0's stack, call fn(g).
214// Fn must never return. It should gogo(&g->sched)
215// to keep running g.
216TEXT runtime·mcall(SB), NOSPLIT, $-8-8
217 // Save caller state in g->sched
218 MOVD R15, (g_sched+gobuf_sp)(g)
219 MOVD LR, (g_sched+gobuf_pc)(g)
Michael Mundayf1515a02016-09-12 13:33:00 -0400220 MOVD $0, (g_sched+gobuf_lr)(g)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400221 MOVD g, (g_sched+gobuf_g)(g)
222
223 // Switch to m->g0 & its stack, call fn.
224 MOVD g, R3
225 MOVD g_m(g), R8
226 MOVD m_g0(R8), g
227 BL runtime·save_g(SB)
228 CMP g, R3
229 BNE 2(PC)
230 BR runtime·badmcall(SB)
231 MOVD fn+0(FP), R12 // context
232 MOVD 0(R12), R4 // code pointer
233 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp
234 SUB $16, R15
235 MOVD R3, 8(R15)
236 MOVD $0, 0(R15)
237 BL (R4)
238 BR runtime·badmcall2(SB)
239
240// systemstack_switch is a dummy routine that systemstack leaves at the bottom
241// of the G stack. We need to distinguish the routine that
242// lives at the bottom of the G stack from the one that lives
243// at the top of the system stack because the one at the top of
244// the system stack terminates the stack walk (see topofstack()).
245TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
246 UNDEF
247 BL (LR) // make sure this function is not leaf
248 RET
249
250// func systemstack(fn func())
251TEXT runtime·systemstack(SB), NOSPLIT, $0-8
252 MOVD fn+0(FP), R3 // R3 = fn
253 MOVD R3, R12 // context
254 MOVD g_m(g), R4 // R4 = m
255
256 MOVD m_gsignal(R4), R5 // R5 = gsignal
257 CMPBEQ g, R5, noswitch
258
259 MOVD m_g0(R4), R5 // R5 = g0
260 CMPBEQ g, R5, noswitch
261
262 MOVD m_curg(R4), R6
263 CMPBEQ g, R6, switch
264
265 // Bad: g is not gsignal, not g0, not curg. What is it?
266 // Hide call from linker nosplit analysis.
267 MOVD $runtime·badsystemstack(SB), R3
268 BL (R3)
Austin Clements7f1b2732018-01-12 12:39:22 -0500269 BL runtime·abort(SB)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400270
271switch:
272 // save our state in g->sched. Pretend to
273 // be systemstack_switch if the G stack is scanned.
274 MOVD $runtime·systemstack_switch(SB), R6
275 ADD $16, R6 // get past prologue
276 MOVD R6, (g_sched+gobuf_pc)(g)
277 MOVD R15, (g_sched+gobuf_sp)(g)
Michael Mundayf1515a02016-09-12 13:33:00 -0400278 MOVD $0, (g_sched+gobuf_lr)(g)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400279 MOVD g, (g_sched+gobuf_g)(g)
280
281 // switch to g0
282 MOVD R5, g
283 BL runtime·save_g(SB)
284 MOVD (g_sched+gobuf_sp)(g), R3
285 // make it look like mstart called systemstack on g0, to stop traceback
286 SUB $8, R3
287 MOVD $runtime·mstart(SB), R4
288 MOVD R4, 0(R3)
289 MOVD R3, R15
290
291 // call target function
292 MOVD 0(R12), R3 // code pointer
293 BL (R3)
294
295 // switch back to g
296 MOVD g_m(g), R3
297 MOVD m_curg(R3), g
298 BL runtime·save_g(SB)
299 MOVD (g_sched+gobuf_sp)(g), R15
300 MOVD $0, (g_sched+gobuf_sp)(g)
301 RET
302
303noswitch:
304 // already on m stack, just call directly
Austin Clements15d6ab62017-10-27 15:20:21 -0400305 // Using a tail call here cleans up tracebacks since we won't stop
306 // at an intermediate systemstack.
Michael Mundaye6f36f02016-03-18 19:09:39 -0400307 MOVD 0(R12), R3 // code pointer
Austin Clements15d6ab62017-10-27 15:20:21 -0400308 MOVD 0(R15), LR // restore LR
309 ADD $8, R15
310 BR (R3)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400311
312/*
313 * support for morestack
314 */
315
316// Called during function prolog when more stack is needed.
317// Caller has already loaded:
318// R3: framesize, R4: argsize, R5: LR
319//
320// The traceback routines see morestack on a g0 as being
321// the top of a stack (for example, morestack calling newstack
322// calling the scheduler calling newm calling gc), so we must
323// record an argument size. For that purpose, it has no arguments.
324TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
325 // Cannot grow scheduler stack (m->g0).
326 MOVD g_m(g), R7
327 MOVD m_g0(R7), R8
Austin Clements687d9d52016-10-13 10:44:57 -0400328 CMPBNE g, R8, 3(PC)
329 BL runtime·badmorestackg0(SB)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400330 BL runtime·abort(SB)
331
332 // Cannot grow signal stack (m->gsignal).
333 MOVD m_gsignal(R7), R8
334 CMP g, R8
Austin Clements687d9d52016-10-13 10:44:57 -0400335 BNE 3(PC)
336 BL runtime·badmorestackgsignal(SB)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400337 BL runtime·abort(SB)
338
339 // Called from f.
340 // Set g->sched to context in f.
Michael Mundaye6f36f02016-03-18 19:09:39 -0400341 MOVD R15, (g_sched+gobuf_sp)(g)
342 MOVD LR, R8
343 MOVD R8, (g_sched+gobuf_pc)(g)
344 MOVD R5, (g_sched+gobuf_lr)(g)
Austin Clements3beaf262017-10-22 21:37:05 -0400345 MOVD R12, (g_sched+gobuf_ctxt)(g)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400346
347 // Called from f.
348 // Set m->morebuf to f's caller.
349 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
350 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
351 MOVD g, (m_morebuf+gobuf_g)(R7)
352
353 // Call newstack on m->g0's stack.
354 MOVD m_g0(R7), g
355 BL runtime·save_g(SB)
356 MOVD (g_sched+gobuf_sp)(g), R15
Austin Clementsbf9c71c2016-10-19 18:27:39 -0400357 // Create a stack frame on g0 to call newstack.
Austin Clements3beaf262017-10-22 21:37:05 -0400358 MOVD $0, -8(R15) // Zero saved LR in frame
359 SUB $8, R15
Michael Mundaye6f36f02016-03-18 19:09:39 -0400360 BL runtime·newstack(SB)
361
362 // Not reached, but make sure the return PC from the call to newstack
363 // is still in this function, and not the beginning of the next.
364 UNDEF
365
366TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
367 MOVD $0, R12
368 BR runtime·morestack(SB)
369
Michael Mundaye6f36f02016-03-18 19:09:39 -0400370// reflectcall: call a function with the given argument list
371// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
372// we don't have variable-sized frames, so we use a small number
373// of constant-sized-frame functions to encode a few bits of size in the pc.
374// Caution: ugly multiline assembly macros in your future!
375
376#define DISPATCH(NAME,MAXSIZE) \
377 MOVD $MAXSIZE, R4; \
378 CMP R3, R4; \
379 BGT 3(PC); \
380 MOVD $NAME(SB), R5; \
381 BR (R5)
382// Note: can't just "BR NAME(SB)" - bad inlining results.
383
Michael Mundaye6f36f02016-03-18 19:09:39 -0400384TEXT ·reflectcall(SB), NOSPLIT, $-8-32
385 MOVWZ argsize+24(FP), R3
Austin Clements40bff822020-10-03 20:40:49 -0400386 DISPATCH(runtime·call16, 16)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400387 DISPATCH(runtime·call32, 32)
388 DISPATCH(runtime·call64, 64)
389 DISPATCH(runtime·call128, 128)
390 DISPATCH(runtime·call256, 256)
391 DISPATCH(runtime·call512, 512)
392 DISPATCH(runtime·call1024, 1024)
393 DISPATCH(runtime·call2048, 2048)
394 DISPATCH(runtime·call4096, 4096)
395 DISPATCH(runtime·call8192, 8192)
396 DISPATCH(runtime·call16384, 16384)
397 DISPATCH(runtime·call32768, 32768)
398 DISPATCH(runtime·call65536, 65536)
399 DISPATCH(runtime·call131072, 131072)
400 DISPATCH(runtime·call262144, 262144)
401 DISPATCH(runtime·call524288, 524288)
402 DISPATCH(runtime·call1048576, 1048576)
403 DISPATCH(runtime·call2097152, 2097152)
404 DISPATCH(runtime·call4194304, 4194304)
405 DISPATCH(runtime·call8388608, 8388608)
406 DISPATCH(runtime·call16777216, 16777216)
407 DISPATCH(runtime·call33554432, 33554432)
408 DISPATCH(runtime·call67108864, 67108864)
409 DISPATCH(runtime·call134217728, 134217728)
410 DISPATCH(runtime·call268435456, 268435456)
411 DISPATCH(runtime·call536870912, 536870912)
412 DISPATCH(runtime·call1073741824, 1073741824)
413 MOVD $runtime·badreflectcall(SB), R5
414 BR (R5)
415
416#define CALLFN(NAME,MAXSIZE) \
417TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
418 NO_LOCAL_POINTERS; \
419 /* copy arguments to stack */ \
Michael Munday3436f072016-08-30 15:25:28 -0400420 MOVD arg+16(FP), R4; \
421 MOVWZ argsize+24(FP), R5; \
422 MOVD $stack-MAXSIZE(SP), R6; \
423loopArgs: /* copy 256 bytes at a time */ \
424 CMP R5, $256; \
425 BLT tailArgs; \
426 SUB $256, R5; \
427 MVC $256, 0(R4), 0(R6); \
428 MOVD $256(R4), R4; \
429 MOVD $256(R6), R6; \
430 BR loopArgs; \
431tailArgs: /* copy remaining bytes */ \
432 CMP R5, $0; \
433 BEQ callFunction; \
434 SUB $1, R5; \
435 EXRL $callfnMVC<>(SB), R5; \
436callFunction: \
Michael Mundaye6f36f02016-03-18 19:09:39 -0400437 MOVD f+8(FP), R12; \
438 MOVD (R12), R8; \
439 PCDATA $PCDATA_StackMapIndex, $0; \
440 BL (R8); \
441 /* copy return values back */ \
Austin Clements79561a82016-10-20 22:45:18 -0400442 MOVD argtype+0(FP), R7; \
Michael Munday3436f072016-08-30 15:25:28 -0400443 MOVD arg+16(FP), R6; \
444 MOVWZ n+24(FP), R5; \
445 MOVD $stack-MAXSIZE(SP), R4; \
446 MOVWZ retoffset+28(FP), R1; \
447 ADD R1, R4; \
448 ADD R1, R6; \
449 SUB R1, R5; \
Austin Clements79561a82016-10-20 22:45:18 -0400450 BL callRet<>(SB); \
451 RET
452
453// callRet copies return values back at the end of call*. This is a
454// separate function so it can allocate stack space for the arguments
455// to reflectcallmove. It does not follow the Go ABI; it expects its
456// arguments in registers.
457TEXT callRet<>(SB), NOSPLIT, $32-0
458 MOVD R7, 8(R15)
459 MOVD R6, 16(R15)
460 MOVD R4, 24(R15)
461 MOVD R5, 32(R15)
462 BL runtime·reflectcallmove(SB)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400463 RET
464
Austin Clements40bff822020-10-03 20:40:49 -0400465CALLFN(·call16, 16)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400466CALLFN(·call32, 32)
467CALLFN(·call64, 64)
468CALLFN(·call128, 128)
469CALLFN(·call256, 256)
470CALLFN(·call512, 512)
471CALLFN(·call1024, 1024)
472CALLFN(·call2048, 2048)
473CALLFN(·call4096, 4096)
474CALLFN(·call8192, 8192)
475CALLFN(·call16384, 16384)
476CALLFN(·call32768, 32768)
477CALLFN(·call65536, 65536)
478CALLFN(·call131072, 131072)
479CALLFN(·call262144, 262144)
480CALLFN(·call524288, 524288)
481CALLFN(·call1048576, 1048576)
482CALLFN(·call2097152, 2097152)
483CALLFN(·call4194304, 4194304)
484CALLFN(·call8388608, 8388608)
485CALLFN(·call16777216, 16777216)
486CALLFN(·call33554432, 33554432)
487CALLFN(·call67108864, 67108864)
488CALLFN(·call134217728, 134217728)
489CALLFN(·call268435456, 268435456)
490CALLFN(·call536870912, 536870912)
491CALLFN(·call1073741824, 1073741824)
492
Michael Munday3436f072016-08-30 15:25:28 -0400493// Not a function: target for EXRL (execute relative long) instruction.
494TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0
495 MVC $1, 0(R4), 0(R6)
496
Michael Mundaye6f36f02016-03-18 19:09:39 -0400497TEXT runtime·procyield(SB),NOSPLIT,$0-0
498 RET
499
500// void jmpdefer(fv, sp);
501// called from deferreturn.
502// 1. grab stored LR for caller
503// 2. sub 6 bytes to get back to BL deferreturn (size of BRASL instruction)
504// 3. BR to fn
505TEXT runtime·jmpdefer(SB),NOSPLIT|NOFRAME,$0-16
506 MOVD 0(R15), R1
507 SUB $6, R1, LR
508
509 MOVD fv+0(FP), R12
510 MOVD argp+8(FP), R15
511 SUB $8, R15
512 MOVD 0(R12), R3
513 BR (R3)
514
Austin Clements70c107c2016-10-19 15:49:31 -0400515// Save state of caller into g->sched. Smashes R1.
Michael Mundaye6f36f02016-03-18 19:09:39 -0400516TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0
517 MOVD LR, (g_sched+gobuf_pc)(g)
518 MOVD R15, (g_sched+gobuf_sp)(g)
519 MOVD $0, (g_sched+gobuf_lr)(g)
520 MOVD $0, (g_sched+gobuf_ret)(g)
Austin Clements70c107c2016-10-19 15:49:31 -0400521 // Assert ctxt is zero. See func save.
522 MOVD (g_sched+gobuf_ctxt)(g), R1
523 CMPBEQ R1, $0, 2(PC)
524 BL runtime·badctxt(SB)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400525 RET
526
527// func asmcgocall(fn, arg unsafe.Pointer) int32
528// Call fn(arg) on the scheduler stack,
529// aligned appropriately for the gcc ABI.
530// See cgocall.go for more details.
531TEXT ·asmcgocall(SB),NOSPLIT,$0-20
532 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
533 // C TLS base pointer in AR0:AR1
534 MOVD fn+0(FP), R3
535 MOVD arg+8(FP), R4
536
537 MOVD R15, R2 // save original stack pointer
538 MOVD g, R5
539
540 // Figure out if we need to switch to m->g0 stack.
541 // We get called to create new OS threads too, and those
542 // come in on the m->g0 stack already.
543 MOVD g_m(g), R6
544 MOVD m_g0(R6), R6
545 CMPBEQ R6, g, g0
546 BL gosave<>(SB)
547 MOVD R6, g
548 BL runtime·save_g(SB)
549 MOVD (g_sched+gobuf_sp)(g), R15
550
551 // Now on a scheduling stack (a pthread-created stack).
552g0:
553 // Save room for two of our pointers, plus 160 bytes of callee
554 // save area that lives on the caller stack.
555 SUB $176, R15
556 MOVD $~7, R6
557 AND R6, R15 // 8-byte alignment for gcc ABI
558 MOVD R5, 168(R15) // save old g on stack
559 MOVD (g_stack+stack_hi)(R5), R5
560 SUB R2, R5
561 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
Michael Mundayf1515a02016-09-12 13:33:00 -0400562 MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400563 MOVD R4, R2 // arg in R2
564 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15
565
566 XOR R0, R0 // set R0 back to 0.
567 // Restore g, stack pointer.
568 MOVD 168(R15), g
569 BL runtime·save_g(SB)
570 MOVD (g_stack+stack_hi)(g), R5
571 MOVD 160(R15), R6
572 SUB R6, R5
573 MOVD R5, R15
574
575 MOVW R2, ret+16(FP)
576 RET
577
Austin Clements30c18872020-10-01 17:22:38 -0400578// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400579// See cgocall.go for more details.
Austin Clements30c18872020-10-01 17:22:38 -0400580TEXT ·cgocallback(SB),NOSPLIT,$24-24
Michael Mundaye6f36f02016-03-18 19:09:39 -0400581 NO_LOCAL_POINTERS
582
583 // Load m and g from thread-local storage.
584 MOVB runtime·iscgo(SB), R3
585 CMPBEQ R3, $0, nocgo
586 BL runtime·load_g(SB)
587
588nocgo:
589 // If g is nil, Go did not create the current thread.
590 // Call needm to obtain one for temporary use.
591 // In this case, we're running on the thread stack, so there's
592 // lots of space, but the linker doesn't know. Hide the call from
593 // the linker analysis by using an indirect call.
594 CMPBEQ g, $0, needm
595
596 MOVD g_m(g), R8
597 MOVD R8, savedm-8(SP)
598 BR havem
599
600needm:
601 MOVD g, savedm-8(SP) // g is zero, so is m.
602 MOVD $runtime·needm(SB), R3
603 BL (R3)
604
605 // Set m->sched.sp = SP, so that if a panic happens
606 // during the function we are about to execute, it will
607 // have a valid SP to run on the g0 stack.
608 // The next few lines (after the havem label)
609 // will save this SP onto the stack and then write
610 // the same SP back to m->sched.sp. That seems redundant,
611 // but if an unrecovered panic happens, unwindm will
612 // restore the g->sched.sp from the stack location
613 // and then systemstack will try to use it. If we don't set it here,
614 // that restored SP will be uninitialized (typically 0) and
615 // will not be usable.
616 MOVD g_m(g), R8
617 MOVD m_g0(R8), R3
618 MOVD R15, (g_sched+gobuf_sp)(R3)
619
620havem:
621 // Now there's a valid m, and we're running on its m->g0.
622 // Save current m->g0->sched.sp on stack and then set it to SP.
623 // Save current sp in m->g0->sched.sp in preparation for
624 // switch back to m->curg stack.
625 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
626 MOVD m_g0(R8), R3
627 MOVD (g_sched+gobuf_sp)(R3), R4
Austin Clements30c18872020-10-01 17:22:38 -0400628 MOVD R4, savedsp-24(SP) // must match frame size
Michael Mundaye6f36f02016-03-18 19:09:39 -0400629 MOVD R15, (g_sched+gobuf_sp)(R3)
630
631 // Switch to m->curg stack and call runtime.cgocallbackg.
632 // Because we are taking over the execution of m->curg
633 // but *not* resuming what had been running, we need to
634 // save that information (m->curg->sched) so we can restore it.
635 // We can restore m->curg->sched.sp easily, because calling
636 // runtime.cgocallbackg leaves SP unchanged upon return.
Austin Clements30c18872020-10-01 17:22:38 -0400637 // To save m->curg->sched.pc, we push it onto the curg stack and
638 // open a frame the same size as cgocallback's g0 frame.
639 // Once we switch to the curg stack, the pushed PC will appear
640 // to be the return PC of cgocallback, so that the traceback
641 // will seamlessly trace back into the earlier calls.
Michael Mundaye6f36f02016-03-18 19:09:39 -0400642 MOVD m_curg(R8), g
643 BL runtime·save_g(SB)
644 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
645 MOVD (g_sched+gobuf_pc)(g), R5
Austin Clements30c18872020-10-01 17:22:38 -0400646 MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size
647 // Gather our arguments into registers.
648 MOVD fn+0(FP), R1
649 MOVD frame+8(FP), R2
650 MOVD ctxt+16(FP), R3
651 MOVD $-(24+8)(R4), R15 // switch stack; must match frame size
652 MOVD R1, 8(R15)
653 MOVD R2, 16(R15)
654 MOVD R3, 24(R15)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400655 BL runtime·cgocallbackg(SB)
656
657 // Restore g->sched (== m->curg->sched) from saved values.
658 MOVD 0(R15), R5
659 MOVD R5, (g_sched+gobuf_pc)(g)
Austin Clements30c18872020-10-01 17:22:38 -0400660 MOVD $(24+8)(R15), R4 // must match frame size
Michael Mundaye6f36f02016-03-18 19:09:39 -0400661 MOVD R4, (g_sched+gobuf_sp)(g)
662
663 // Switch back to m->g0's stack and restore m->g0->sched.sp.
664 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
665 // so we do not have to restore it.)
666 MOVD g_m(g), R8
667 MOVD m_g0(R8), g
668 BL runtime·save_g(SB)
669 MOVD (g_sched+gobuf_sp)(g), R15
Austin Clements30c18872020-10-01 17:22:38 -0400670 MOVD savedsp-24(SP), R4 // must match frame size
Michael Mundaye6f36f02016-03-18 19:09:39 -0400671 MOVD R4, (g_sched+gobuf_sp)(g)
672
673 // If the m on entry was nil, we called needm above to borrow an m
674 // for the duration of the call. Since the call is over, return it with dropm.
675 MOVD savedm-8(SP), R6
676 CMPBNE R6, $0, droppedm
677 MOVD $runtime·dropm(SB), R3
678 BL (R3)
679droppedm:
680
681 // Done!
682 RET
683
684// void setg(G*); set g. for use by needm.
685TEXT runtime·setg(SB), NOSPLIT, $0-8
686 MOVD gg+0(FP), g
687 // This only happens if iscgo, so jump straight to save_g
688 BL runtime·save_g(SB)
689 RET
690
691// void setg_gcc(G*); set g in C TLS.
692// Must obey the gcc calling convention.
693TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
694 // The standard prologue clobbers LR (R14), which is callee-save in
695 // the C ABI, so we have to use NOFRAME and save LR ourselves.
696 MOVD LR, R1
697 // Also save g, R10, and R11 since they're callee-save in C ABI
698 MOVD R10, R3
699 MOVD g, R4
700 MOVD R11, R5
701
702 MOVD R2, g
703 BL runtime·save_g(SB)
704
705 MOVD R5, R11
706 MOVD R4, g
707 MOVD R3, R10
708 MOVD R1, LR
709 RET
710
Michael Mundaye6f36f02016-03-18 19:09:39 -0400711TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
712 MOVW (R0), R0
713 UNDEF
714
715// int64 runtime·cputicks(void)
716TEXT runtime·cputicks(SB),NOSPLIT,$0-8
717 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals.
718 // This means that since about 1972 the msb has been set, making the
719 // result of a call to STORE CLOCK (stck) a negative number.
720 // We clear the msb to make it positive.
721 STCK ret+0(FP) // serialises before and after call
722 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043
723 SLD $1, R3
724 SRD $1, R3
725 MOVD R3, ret+0(FP)
726 RET
727
Michael Mundaye6f36f02016-03-18 19:09:39 -0400728// AES hashing not implemented for s390x
Keith Randallfbfb41e2019-08-20 11:03:13 -0700729TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
730 JMP runtime·memhashFallback(SB)
731TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
732 JMP runtime·strhashFallback(SB)
733TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
734 JMP runtime·memhash32Fallback(SB)
735TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
736 JMP runtime·memhash64Fallback(SB)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400737
Michael Mundaye6f36f02016-03-18 19:09:39 -0400738TEXT runtime·return0(SB), NOSPLIT, $0
739 MOVW $0, R3
740 RET
741
742// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
743// Must obey the gcc calling convention.
744TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
745 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them
746 MOVD g, R1
747 MOVD R10, R3
748 MOVD LR, R4
749 MOVD R11, R5
750
751 BL runtime·load_g(SB) // clobbers g (R13), R10, R11
752 MOVD g_m(g), R2
753 MOVD m_curg(R2), R2
754 MOVD (g_stack+stack_hi)(R2), R2
755
756 MOVD R1, g
757 MOVD R3, R10
758 MOVD R4, LR
759 MOVD R5, R11
760 RET
761
762// The top-most function running on a goroutine
763// returns to goexit+PCQuantum.
Michael Mundayaafe2572019-03-28 12:51:30 -0400764TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
Michael Mundaye6f36f02016-03-18 19:09:39 -0400765 BYTE $0x07; BYTE $0x00; // 2-byte nop
766 BL runtime·goexit1(SB) // does not return
767 // traceback from goexit1 must hit code range of goexit
768 BYTE $0x07; BYTE $0x00; // 2-byte nop
769
Michael Mundaye6f36f02016-03-18 19:09:39 -0400770TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
Russ Cox26a5f6a2019-06-05 13:57:47 -0400771 // Stores are already ordered on s390x, so this is just a
772 // compile barrier.
Michael Mundaye6f36f02016-03-18 19:09:39 -0400773 RET
774
Michael Mundaye6f36f02016-03-18 19:09:39 -0400775// This is called from .init_array and follows the platform, not Go, ABI.
776// We are overly conservative. We could only save the registers we use.
777// However, since this function is only called once per loaded module
778// performance is unimportant.
779TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
Michael Munday06fcc322016-11-23 14:54:12 -0500780 // Save R6-R15 in the register save area of the calling function.
781 // Don't bother saving F8-F15 as we aren't doing any calls.
Michael Mundaye6f36f02016-03-18 19:09:39 -0400782 STMG R6, R15, 48(R15)
Michael Mundaye6f36f02016-03-18 19:09:39 -0400783
784 // append the argument (passed in R2, as per the ELF ABI) to the
785 // moduledata linked list.
786 MOVD runtime·lastmoduledatap(SB), R1
787 MOVD R2, moduledata_next(R1)
788 MOVD R2, runtime·lastmoduledatap(SB)
789
Michael Munday06fcc322016-11-23 14:54:12 -0500790 // Restore R6-R15.
Michael Mundaye6f36f02016-03-18 19:09:39 -0400791 LMG 48(R15), R6, R15
Michael Mundaye6f36f02016-03-18 19:09:39 -0400792 RET
793
794TEXT ·checkASM(SB),NOSPLIT,$0-1
795 MOVB $1, ret+0(FP)
796 RET
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800797
798// gcWriteBarrier performs a heap pointer write and informs the GC.
799//
800// gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
801// - R2 is the destination of the write
802// - R3 is the value being written at R2.
Michael Munday5d9c7822018-04-30 16:55:13 +0100803// It clobbers R10 (the temp register).
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800804// It does not clobber any other general-purpose registers,
805// but may clobber others (e.g., floating point registers).
Michael Munday5d9c7822018-04-30 16:55:13 +0100806TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$104
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800807 // Save the registers clobbered by the fast path.
Michael Munday5d9c7822018-04-30 16:55:13 +0100808 MOVD R1, 96(R15)
809 MOVD R4, 104(R15)
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800810 MOVD g_m(g), R1
811 MOVD m_p(R1), R1
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800812 // Increment wbBuf.next position.
Michael Munday5d9c7822018-04-30 16:55:13 +0100813 MOVD $16, R4
814 ADD (p_wbBuf+wbBuf_next)(R1), R4
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800815 MOVD R4, (p_wbBuf+wbBuf_next)(R1)
816 MOVD (p_wbBuf+wbBuf_end)(R1), R1
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800817 // Record the write.
Michael Munday5d9c7822018-04-30 16:55:13 +0100818 MOVD R3, -16(R4) // Record value
819 MOVD (R2), R10 // TODO: This turns bad writes into bad reads.
820 MOVD R10, -8(R4) // Record *slot
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800821 // Is the buffer full?
Michael Munday5d9c7822018-04-30 16:55:13 +0100822 CMPBEQ R4, R1, flush
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800823ret:
Michael Munday5d9c7822018-04-30 16:55:13 +0100824 MOVD 96(R15), R1
825 MOVD 104(R15), R4
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800826 // Do the write.
827 MOVD R3, (R2)
828 RET
829
830flush:
831 // Save all general purpose registers since these could be
832 // clobbered by wbBufFlush and were not saved by the caller.
Michael Munday5d9c7822018-04-30 16:55:13 +0100833 STMG R2, R3, 8(R15) // set R2 and R3 as arguments for wbBufFlush
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800834 MOVD R0, 24(R15)
835 // R1 already saved.
836 // R4 already saved.
Michael Munday5d9c7822018-04-30 16:55:13 +0100837 STMG R5, R12, 32(R15) // save R5 - R12
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800838 // R13 is g.
839 // R14 is LR.
840 // R15 is SP.
841
842 // This takes arguments R2 and R3.
843 CALL runtime·wbBufFlush(SB)
844
Michael Munday5d9c7822018-04-30 16:55:13 +0100845 LMG 8(R15), R2, R3 // restore R2 - R3
846 MOVD 24(R15), R0 // restore R0
847 LMG 32(R15), R5, R12 // restore R5 - R12
Austin Clements2ae1e1a2017-11-15 14:54:24 -0800848 JMP ret
Keith Randall2c423f02019-02-06 14:12:36 -0800849
850// Note: these functions use a special calling convention to save generated code space.
851// Arguments are passed in registers, but the space for those arguments are allocated
852// in the caller's stack frame. These stubs write the args into that stack space and
853// then tail call to the corresponding runtime handler.
854// The tail call makes these stubs disappear in backtraces.
855TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
856 MOVD R0, x+0(FP)
857 MOVD R1, y+8(FP)
858 JMP runtime·goPanicIndex(SB)
859TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
860 MOVD R0, x+0(FP)
861 MOVD R1, y+8(FP)
862 JMP runtime·goPanicIndexU(SB)
863TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
864 MOVD R1, x+0(FP)
865 MOVD R2, y+8(FP)
866 JMP runtime·goPanicSliceAlen(SB)
867TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
868 MOVD R1, x+0(FP)
869 MOVD R2, y+8(FP)
870 JMP runtime·goPanicSliceAlenU(SB)
871TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
872 MOVD R1, x+0(FP)
873 MOVD R2, y+8(FP)
874 JMP runtime·goPanicSliceAcap(SB)
875TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
876 MOVD R1, x+0(FP)
877 MOVD R2, y+8(FP)
878 JMP runtime·goPanicSliceAcapU(SB)
879TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
880 MOVD R0, x+0(FP)
881 MOVD R1, y+8(FP)
882 JMP runtime·goPanicSliceB(SB)
883TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
884 MOVD R0, x+0(FP)
885 MOVD R1, y+8(FP)
886 JMP runtime·goPanicSliceBU(SB)
887TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
888 MOVD R2, x+0(FP)
889 MOVD R3, y+8(FP)
890 JMP runtime·goPanicSlice3Alen(SB)
891TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
892 MOVD R2, x+0(FP)
893 MOVD R3, y+8(FP)
894 JMP runtime·goPanicSlice3AlenU(SB)
895TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
896 MOVD R2, x+0(FP)
897 MOVD R3, y+8(FP)
898 JMP runtime·goPanicSlice3Acap(SB)
899TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
900 MOVD R2, x+0(FP)
901 MOVD R3, y+8(FP)
902 JMP runtime·goPanicSlice3AcapU(SB)
903TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
904 MOVD R1, x+0(FP)
905 MOVD R2, y+8(FP)
906 JMP runtime·goPanicSlice3B(SB)
907TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
908 MOVD R1, x+0(FP)
909 MOVD R2, y+8(FP)
910 JMP runtime·goPanicSlice3BU(SB)
911TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
912 MOVD R0, x+0(FP)
913 MOVD R1, y+8(FP)
914 JMP runtime·goPanicSlice3C(SB)
915TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
916 MOVD R0, x+0(FP)
917 MOVD R1, y+8(FP)
918 JMP runtime·goPanicSlice3CU(SB)