Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 1 | // Copyright 2016 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | #include "go_asm.h" |
| 6 | #include "go_tls.h" |
| 7 | #include "funcdata.h" |
| 8 | #include "textflag.h" |
| 9 | |
Bill O'Farrell | c2efb2f | 2017-10-31 19:18:48 -0400 | [diff] [blame] | 10 | // _rt0_s390x_lib is common startup code for s390x systems when |
| 11 | // using -buildmode=c-archive or -buildmode=c-shared. The linker will |
| 12 | // arrange to invoke this function as a global constructor (for |
| 13 | // c-archive) or when the shared library is loaded (for c-shared). |
| 14 | // We expect argc and argv to be passed in the usual C ABI registers |
| 15 | // R2 and R3. |
| 16 | TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0 |
| 17 | STMG R6, R15, 48(R15) |
| 18 | MOVD R2, _rt0_s390x_lib_argc<>(SB) |
| 19 | MOVD R3, _rt0_s390x_lib_argv<>(SB) |
| 20 | |
| 21 | // Save R6-R15 in the register save area of the calling function. |
| 22 | STMG R6, R15, 48(R15) |
| 23 | |
| 24 | // Allocate 80 bytes on the stack. |
| 25 | MOVD $-80(R15), R15 |
| 26 | |
| 27 | // Save F8-F15 in our stack frame. |
| 28 | FMOVD F8, 16(R15) |
| 29 | FMOVD F9, 24(R15) |
| 30 | FMOVD F10, 32(R15) |
| 31 | FMOVD F11, 40(R15) |
| 32 | FMOVD F12, 48(R15) |
| 33 | FMOVD F13, 56(R15) |
| 34 | FMOVD F14, 64(R15) |
| 35 | FMOVD F15, 72(R15) |
| 36 | |
| 37 | // Synchronous initialization. |
| 38 | MOVD $runtime·libpreinit(SB), R1 |
| 39 | BL R1 |
| 40 | |
| 41 | // Create a new thread to finish Go runtime initialization. |
| 42 | MOVD _cgo_sys_thread_create(SB), R1 |
| 43 | CMP R1, $0 |
| 44 | BEQ nocgo |
| 45 | MOVD $_rt0_s390x_lib_go(SB), R2 |
| 46 | MOVD $0, R3 |
| 47 | BL R1 |
| 48 | BR restore |
| 49 | |
| 50 | nocgo: |
| 51 | MOVD $0x800000, R1 // stacksize |
| 52 | MOVD R1, 0(R15) |
| 53 | MOVD $_rt0_s390x_lib_go(SB), R1 |
| 54 | MOVD R1, 8(R15) // fn |
| 55 | MOVD $runtime·newosproc(SB), R1 |
| 56 | BL R1 |
| 57 | |
| 58 | restore: |
| 59 | // Restore F8-F15 from our stack frame. |
| 60 | FMOVD 16(R15), F8 |
| 61 | FMOVD 24(R15), F9 |
| 62 | FMOVD 32(R15), F10 |
| 63 | FMOVD 40(R15), F11 |
| 64 | FMOVD 48(R15), F12 |
| 65 | FMOVD 56(R15), F13 |
| 66 | FMOVD 64(R15), F14 |
| 67 | FMOVD 72(R15), F15 |
| 68 | MOVD $80(R15), R15 |
| 69 | |
| 70 | // Restore R6-R15. |
| 71 | LMG 48(R15), R6, R15 |
| 72 | RET |
| 73 | |
| 74 | // _rt0_s390x_lib_go initializes the Go runtime. |
| 75 | // This is started in a separate thread by _rt0_s390x_lib. |
| 76 | TEXT _rt0_s390x_lib_go(SB), NOSPLIT|NOFRAME, $0 |
| 77 | MOVD _rt0_s390x_lib_argc<>(SB), R2 |
| 78 | MOVD _rt0_s390x_lib_argv<>(SB), R3 |
| 79 | MOVD $runtime·rt0_go(SB), R1 |
| 80 | BR R1 |
| 81 | |
| 82 | DATA _rt0_s390x_lib_argc<>(SB)/8, $0 |
| 83 | GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8 |
| 84 | DATA _rt0_s90x_lib_argv<>(SB)/8, $0 |
| 85 | GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8 |
| 86 | |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 87 | TEXT runtime·rt0_go(SB),NOSPLIT,$0 |
| 88 | // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer |
| 89 | // C TLS base pointer in AR0:AR1 |
| 90 | |
| 91 | // initialize essential registers |
| 92 | XOR R0, R0 |
| 93 | |
| 94 | SUB $24, R15 |
| 95 | MOVW R2, 8(R15) // argc |
| 96 | MOVD R3, 16(R15) // argv |
| 97 | |
| 98 | // create istack out of the given (operating system) stack. |
| 99 | // _cgo_init may update stackguard. |
| 100 | MOVD $runtime·g0(SB), g |
| 101 | MOVD R15, R11 |
| 102 | SUB $(64*1024), R11 |
| 103 | MOVD R11, g_stackguard0(g) |
| 104 | MOVD R11, g_stackguard1(g) |
| 105 | MOVD R11, (g_stack+stack_lo)(g) |
| 106 | MOVD R15, (g_stack+stack_hi)(g) |
| 107 | |
| 108 | // if there is a _cgo_init, call it using the gcc ABI. |
| 109 | MOVD _cgo_init(SB), R11 |
| 110 | CMPBEQ R11, $0, nocgo |
| 111 | MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR |
| 112 | SLD $32, R4, R4 |
| 113 | MOVW AR1, R4 // arg 2: TLS base pointer |
| 114 | MOVD $setg_gcc<>(SB), R3 // arg 1: setg |
| 115 | MOVD g, R2 // arg 0: G |
| 116 | // C functions expect 160 bytes of space on caller stack frame |
| 117 | // and an 8-byte aligned stack pointer |
| 118 | MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI) |
| 119 | SUB $160, R15 // reserve 160 bytes |
| 120 | MOVD $~7, R6 |
| 121 | AND R6, R15 // 8-byte align |
| 122 | BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14) |
| 123 | MOVD R9, R15 // restore stack |
| 124 | XOR R0, R0 // zero R0 |
| 125 | |
| 126 | nocgo: |
| 127 | // update stackguard after _cgo_init |
| 128 | MOVD (g_stack+stack_lo)(g), R2 |
| 129 | ADD $const__StackGuard, R2 |
| 130 | MOVD R2, g_stackguard0(g) |
| 131 | MOVD R2, g_stackguard1(g) |
| 132 | |
| 133 | // set the per-goroutine and per-mach "registers" |
| 134 | MOVD $runtime·m0(SB), R2 |
| 135 | |
| 136 | // save m->g0 = g0 |
| 137 | MOVD g, m_g0(R2) |
| 138 | // save m0 to g0->m |
| 139 | MOVD R2, g_m(g) |
| 140 | |
| 141 | BL runtime·check(SB) |
| 142 | |
| 143 | // argc/argv are already prepared on stack |
| 144 | BL runtime·args(SB) |
| 145 | BL runtime·osinit(SB) |
| 146 | BL runtime·schedinit(SB) |
| 147 | |
| 148 | // create a new goroutine to start program |
| 149 | MOVD $runtime·mainPC(SB), R2 // entry |
| 150 | SUB $24, R15 |
| 151 | MOVD R2, 16(R15) |
Michael Munday | f1515a0 | 2016-09-12 13:33:00 -0400 | [diff] [blame] | 152 | MOVD $0, 8(R15) |
| 153 | MOVD $0, 0(R15) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 154 | BL runtime·newproc(SB) |
| 155 | ADD $24, R15 |
| 156 | |
| 157 | // start this M |
| 158 | BL runtime·mstart(SB) |
| 159 | |
Michael Munday | f1515a0 | 2016-09-12 13:33:00 -0400 | [diff] [blame] | 160 | MOVD $0, 1(R0) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 161 | RET |
| 162 | |
| 163 | DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) |
| 164 | GLOBL runtime·mainPC(SB),RODATA,$8 |
| 165 | |
| 166 | TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 |
Michael Munday | f1515a0 | 2016-09-12 13:33:00 -0400 | [diff] [blame] | 167 | MOVD $0, 2(R0) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 168 | RET |
| 169 | |
| 170 | TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 |
| 171 | RET |
| 172 | |
| 173 | /* |
| 174 | * go-routine |
| 175 | */ |
| 176 | |
| 177 | // void gosave(Gobuf*) |
| 178 | // save state in Gobuf; setjmp |
| 179 | TEXT runtime·gosave(SB), NOSPLIT, $-8-8 |
| 180 | MOVD buf+0(FP), R3 |
| 181 | MOVD R15, gobuf_sp(R3) |
| 182 | MOVD LR, gobuf_pc(R3) |
| 183 | MOVD g, gobuf_g(R3) |
| 184 | MOVD $0, gobuf_lr(R3) |
| 185 | MOVD $0, gobuf_ret(R3) |
Austin Clements | 70c107c | 2016-10-19 15:49:31 -0400 | [diff] [blame] | 186 | // Assert ctxt is zero. See func save. |
| 187 | MOVD gobuf_ctxt(R3), R3 |
| 188 | CMPBEQ R3, $0, 2(PC) |
| 189 | BL runtime·badctxt(SB) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 190 | RET |
| 191 | |
| 192 | // void gogo(Gobuf*) |
| 193 | // restore state from Gobuf; longjmp |
Austin Clements | 70c107c | 2016-10-19 15:49:31 -0400 | [diff] [blame] | 194 | TEXT runtime·gogo(SB), NOSPLIT, $16-8 |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 195 | MOVD buf+0(FP), R5 |
| 196 | MOVD gobuf_g(R5), g // make sure g is not nil |
| 197 | BL runtime·save_g(SB) |
| 198 | |
| 199 | MOVD 0(g), R4 |
| 200 | MOVD gobuf_sp(R5), R15 |
| 201 | MOVD gobuf_lr(R5), LR |
| 202 | MOVD gobuf_ret(R5), R3 |
| 203 | MOVD gobuf_ctxt(R5), R12 |
| 204 | MOVD $0, gobuf_sp(R5) |
| 205 | MOVD $0, gobuf_ret(R5) |
| 206 | MOVD $0, gobuf_lr(R5) |
| 207 | MOVD $0, gobuf_ctxt(R5) |
| 208 | CMP R0, R0 // set condition codes for == test, needed by stack split |
| 209 | MOVD gobuf_pc(R5), R6 |
| 210 | BR (R6) |
| 211 | |
| 212 | // void mcall(fn func(*g)) |
| 213 | // Switch to m->g0's stack, call fn(g). |
| 214 | // Fn must never return. It should gogo(&g->sched) |
| 215 | // to keep running g. |
| 216 | TEXT runtime·mcall(SB), NOSPLIT, $-8-8 |
| 217 | // Save caller state in g->sched |
| 218 | MOVD R15, (g_sched+gobuf_sp)(g) |
| 219 | MOVD LR, (g_sched+gobuf_pc)(g) |
Michael Munday | f1515a0 | 2016-09-12 13:33:00 -0400 | [diff] [blame] | 220 | MOVD $0, (g_sched+gobuf_lr)(g) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 221 | MOVD g, (g_sched+gobuf_g)(g) |
| 222 | |
| 223 | // Switch to m->g0 & its stack, call fn. |
| 224 | MOVD g, R3 |
| 225 | MOVD g_m(g), R8 |
| 226 | MOVD m_g0(R8), g |
| 227 | BL runtime·save_g(SB) |
| 228 | CMP g, R3 |
| 229 | BNE 2(PC) |
| 230 | BR runtime·badmcall(SB) |
| 231 | MOVD fn+0(FP), R12 // context |
| 232 | MOVD 0(R12), R4 // code pointer |
| 233 | MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp |
| 234 | SUB $16, R15 |
| 235 | MOVD R3, 8(R15) |
| 236 | MOVD $0, 0(R15) |
| 237 | BL (R4) |
| 238 | BR runtime·badmcall2(SB) |
| 239 | |
| 240 | // systemstack_switch is a dummy routine that systemstack leaves at the bottom |
| 241 | // of the G stack. We need to distinguish the routine that |
| 242 | // lives at the bottom of the G stack from the one that lives |
| 243 | // at the top of the system stack because the one at the top of |
| 244 | // the system stack terminates the stack walk (see topofstack()). |
| 245 | TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 |
| 246 | UNDEF |
| 247 | BL (LR) // make sure this function is not leaf |
| 248 | RET |
| 249 | |
| 250 | // func systemstack(fn func()) |
| 251 | TEXT runtime·systemstack(SB), NOSPLIT, $0-8 |
| 252 | MOVD fn+0(FP), R3 // R3 = fn |
| 253 | MOVD R3, R12 // context |
| 254 | MOVD g_m(g), R4 // R4 = m |
| 255 | |
| 256 | MOVD m_gsignal(R4), R5 // R5 = gsignal |
| 257 | CMPBEQ g, R5, noswitch |
| 258 | |
| 259 | MOVD m_g0(R4), R5 // R5 = g0 |
| 260 | CMPBEQ g, R5, noswitch |
| 261 | |
| 262 | MOVD m_curg(R4), R6 |
| 263 | CMPBEQ g, R6, switch |
| 264 | |
| 265 | // Bad: g is not gsignal, not g0, not curg. What is it? |
| 266 | // Hide call from linker nosplit analysis. |
| 267 | MOVD $runtime·badsystemstack(SB), R3 |
| 268 | BL (R3) |
Austin Clements | 7f1b273 | 2018-01-12 12:39:22 -0500 | [diff] [blame] | 269 | BL runtime·abort(SB) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 270 | |
| 271 | switch: |
| 272 | // save our state in g->sched. Pretend to |
| 273 | // be systemstack_switch if the G stack is scanned. |
| 274 | MOVD $runtime·systemstack_switch(SB), R6 |
| 275 | ADD $16, R6 // get past prologue |
| 276 | MOVD R6, (g_sched+gobuf_pc)(g) |
| 277 | MOVD R15, (g_sched+gobuf_sp)(g) |
Michael Munday | f1515a0 | 2016-09-12 13:33:00 -0400 | [diff] [blame] | 278 | MOVD $0, (g_sched+gobuf_lr)(g) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 279 | MOVD g, (g_sched+gobuf_g)(g) |
| 280 | |
| 281 | // switch to g0 |
| 282 | MOVD R5, g |
| 283 | BL runtime·save_g(SB) |
| 284 | MOVD (g_sched+gobuf_sp)(g), R3 |
| 285 | // make it look like mstart called systemstack on g0, to stop traceback |
| 286 | SUB $8, R3 |
| 287 | MOVD $runtime·mstart(SB), R4 |
| 288 | MOVD R4, 0(R3) |
| 289 | MOVD R3, R15 |
| 290 | |
| 291 | // call target function |
| 292 | MOVD 0(R12), R3 // code pointer |
| 293 | BL (R3) |
| 294 | |
| 295 | // switch back to g |
| 296 | MOVD g_m(g), R3 |
| 297 | MOVD m_curg(R3), g |
| 298 | BL runtime·save_g(SB) |
| 299 | MOVD (g_sched+gobuf_sp)(g), R15 |
| 300 | MOVD $0, (g_sched+gobuf_sp)(g) |
| 301 | RET |
| 302 | |
| 303 | noswitch: |
| 304 | // already on m stack, just call directly |
Austin Clements | 15d6ab6 | 2017-10-27 15:20:21 -0400 | [diff] [blame] | 305 | // Using a tail call here cleans up tracebacks since we won't stop |
| 306 | // at an intermediate systemstack. |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 307 | MOVD 0(R12), R3 // code pointer |
Austin Clements | 15d6ab6 | 2017-10-27 15:20:21 -0400 | [diff] [blame] | 308 | MOVD 0(R15), LR // restore LR |
| 309 | ADD $8, R15 |
| 310 | BR (R3) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 311 | |
| 312 | /* |
| 313 | * support for morestack |
| 314 | */ |
| 315 | |
| 316 | // Called during function prolog when more stack is needed. |
| 317 | // Caller has already loaded: |
| 318 | // R3: framesize, R4: argsize, R5: LR |
| 319 | // |
| 320 | // The traceback routines see morestack on a g0 as being |
| 321 | // the top of a stack (for example, morestack calling newstack |
| 322 | // calling the scheduler calling newm calling gc), so we must |
| 323 | // record an argument size. For that purpose, it has no arguments. |
| 324 | TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 |
| 325 | // Cannot grow scheduler stack (m->g0). |
| 326 | MOVD g_m(g), R7 |
| 327 | MOVD m_g0(R7), R8 |
Austin Clements | 687d9d5 | 2016-10-13 10:44:57 -0400 | [diff] [blame] | 328 | CMPBNE g, R8, 3(PC) |
| 329 | BL runtime·badmorestackg0(SB) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 330 | BL runtime·abort(SB) |
| 331 | |
| 332 | // Cannot grow signal stack (m->gsignal). |
| 333 | MOVD m_gsignal(R7), R8 |
| 334 | CMP g, R8 |
Austin Clements | 687d9d5 | 2016-10-13 10:44:57 -0400 | [diff] [blame] | 335 | BNE 3(PC) |
| 336 | BL runtime·badmorestackgsignal(SB) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 337 | BL runtime·abort(SB) |
| 338 | |
| 339 | // Called from f. |
| 340 | // Set g->sched to context in f. |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 341 | MOVD R15, (g_sched+gobuf_sp)(g) |
| 342 | MOVD LR, R8 |
| 343 | MOVD R8, (g_sched+gobuf_pc)(g) |
| 344 | MOVD R5, (g_sched+gobuf_lr)(g) |
Austin Clements | 3beaf26 | 2017-10-22 21:37:05 -0400 | [diff] [blame] | 345 | MOVD R12, (g_sched+gobuf_ctxt)(g) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 346 | |
| 347 | // Called from f. |
| 348 | // Set m->morebuf to f's caller. |
| 349 | MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC |
| 350 | MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP |
| 351 | MOVD g, (m_morebuf+gobuf_g)(R7) |
| 352 | |
| 353 | // Call newstack on m->g0's stack. |
| 354 | MOVD m_g0(R7), g |
| 355 | BL runtime·save_g(SB) |
| 356 | MOVD (g_sched+gobuf_sp)(g), R15 |
Austin Clements | bf9c71c | 2016-10-19 18:27:39 -0400 | [diff] [blame] | 357 | // Create a stack frame on g0 to call newstack. |
Austin Clements | 3beaf26 | 2017-10-22 21:37:05 -0400 | [diff] [blame] | 358 | MOVD $0, -8(R15) // Zero saved LR in frame |
| 359 | SUB $8, R15 |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 360 | BL runtime·newstack(SB) |
| 361 | |
| 362 | // Not reached, but make sure the return PC from the call to newstack |
| 363 | // is still in this function, and not the beginning of the next. |
| 364 | UNDEF |
| 365 | |
| 366 | TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 |
| 367 | MOVD $0, R12 |
| 368 | BR runtime·morestack(SB) |
| 369 | |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 370 | // reflectcall: call a function with the given argument list |
| 371 | // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). |
| 372 | // we don't have variable-sized frames, so we use a small number |
| 373 | // of constant-sized-frame functions to encode a few bits of size in the pc. |
| 374 | // Caution: ugly multiline assembly macros in your future! |
| 375 | |
| 376 | #define DISPATCH(NAME,MAXSIZE) \ |
| 377 | MOVD $MAXSIZE, R4; \ |
| 378 | CMP R3, R4; \ |
| 379 | BGT 3(PC); \ |
| 380 | MOVD $NAME(SB), R5; \ |
| 381 | BR (R5) |
| 382 | // Note: can't just "BR NAME(SB)" - bad inlining results. |
| 383 | |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 384 | TEXT ·reflectcall(SB), NOSPLIT, $-8-32 |
| 385 | MOVWZ argsize+24(FP), R3 |
Austin Clements | 40bff82 | 2020-10-03 20:40:49 -0400 | [diff] [blame] | 386 | DISPATCH(runtime·call16, 16) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 387 | DISPATCH(runtime·call32, 32) |
| 388 | DISPATCH(runtime·call64, 64) |
| 389 | DISPATCH(runtime·call128, 128) |
| 390 | DISPATCH(runtime·call256, 256) |
| 391 | DISPATCH(runtime·call512, 512) |
| 392 | DISPATCH(runtime·call1024, 1024) |
| 393 | DISPATCH(runtime·call2048, 2048) |
| 394 | DISPATCH(runtime·call4096, 4096) |
| 395 | DISPATCH(runtime·call8192, 8192) |
| 396 | DISPATCH(runtime·call16384, 16384) |
| 397 | DISPATCH(runtime·call32768, 32768) |
| 398 | DISPATCH(runtime·call65536, 65536) |
| 399 | DISPATCH(runtime·call131072, 131072) |
| 400 | DISPATCH(runtime·call262144, 262144) |
| 401 | DISPATCH(runtime·call524288, 524288) |
| 402 | DISPATCH(runtime·call1048576, 1048576) |
| 403 | DISPATCH(runtime·call2097152, 2097152) |
| 404 | DISPATCH(runtime·call4194304, 4194304) |
| 405 | DISPATCH(runtime·call8388608, 8388608) |
| 406 | DISPATCH(runtime·call16777216, 16777216) |
| 407 | DISPATCH(runtime·call33554432, 33554432) |
| 408 | DISPATCH(runtime·call67108864, 67108864) |
| 409 | DISPATCH(runtime·call134217728, 134217728) |
| 410 | DISPATCH(runtime·call268435456, 268435456) |
| 411 | DISPATCH(runtime·call536870912, 536870912) |
| 412 | DISPATCH(runtime·call1073741824, 1073741824) |
| 413 | MOVD $runtime·badreflectcall(SB), R5 |
| 414 | BR (R5) |
| 415 | |
| 416 | #define CALLFN(NAME,MAXSIZE) \ |
| 417 | TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ |
| 418 | NO_LOCAL_POINTERS; \ |
| 419 | /* copy arguments to stack */ \ |
Michael Munday | 3436f07 | 2016-08-30 15:25:28 -0400 | [diff] [blame] | 420 | MOVD arg+16(FP), R4; \ |
| 421 | MOVWZ argsize+24(FP), R5; \ |
| 422 | MOVD $stack-MAXSIZE(SP), R6; \ |
| 423 | loopArgs: /* copy 256 bytes at a time */ \ |
| 424 | CMP R5, $256; \ |
| 425 | BLT tailArgs; \ |
| 426 | SUB $256, R5; \ |
| 427 | MVC $256, 0(R4), 0(R6); \ |
| 428 | MOVD $256(R4), R4; \ |
| 429 | MOVD $256(R6), R6; \ |
| 430 | BR loopArgs; \ |
| 431 | tailArgs: /* copy remaining bytes */ \ |
| 432 | CMP R5, $0; \ |
| 433 | BEQ callFunction; \ |
| 434 | SUB $1, R5; \ |
| 435 | EXRL $callfnMVC<>(SB), R5; \ |
| 436 | callFunction: \ |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 437 | MOVD f+8(FP), R12; \ |
| 438 | MOVD (R12), R8; \ |
| 439 | PCDATA $PCDATA_StackMapIndex, $0; \ |
| 440 | BL (R8); \ |
| 441 | /* copy return values back */ \ |
Austin Clements | 79561a8 | 2016-10-20 22:45:18 -0400 | [diff] [blame] | 442 | MOVD argtype+0(FP), R7; \ |
Michael Munday | 3436f07 | 2016-08-30 15:25:28 -0400 | [diff] [blame] | 443 | MOVD arg+16(FP), R6; \ |
| 444 | MOVWZ n+24(FP), R5; \ |
| 445 | MOVD $stack-MAXSIZE(SP), R4; \ |
| 446 | MOVWZ retoffset+28(FP), R1; \ |
| 447 | ADD R1, R4; \ |
| 448 | ADD R1, R6; \ |
| 449 | SUB R1, R5; \ |
Austin Clements | 79561a8 | 2016-10-20 22:45:18 -0400 | [diff] [blame] | 450 | BL callRet<>(SB); \ |
| 451 | RET |
| 452 | |
| 453 | // callRet copies return values back at the end of call*. This is a |
| 454 | // separate function so it can allocate stack space for the arguments |
| 455 | // to reflectcallmove. It does not follow the Go ABI; it expects its |
| 456 | // arguments in registers. |
| 457 | TEXT callRet<>(SB), NOSPLIT, $32-0 |
| 458 | MOVD R7, 8(R15) |
| 459 | MOVD R6, 16(R15) |
| 460 | MOVD R4, 24(R15) |
| 461 | MOVD R5, 32(R15) |
| 462 | BL runtime·reflectcallmove(SB) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 463 | RET |
| 464 | |
Austin Clements | 40bff82 | 2020-10-03 20:40:49 -0400 | [diff] [blame] | 465 | CALLFN(·call16, 16) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 466 | CALLFN(·call32, 32) |
| 467 | CALLFN(·call64, 64) |
| 468 | CALLFN(·call128, 128) |
| 469 | CALLFN(·call256, 256) |
| 470 | CALLFN(·call512, 512) |
| 471 | CALLFN(·call1024, 1024) |
| 472 | CALLFN(·call2048, 2048) |
| 473 | CALLFN(·call4096, 4096) |
| 474 | CALLFN(·call8192, 8192) |
| 475 | CALLFN(·call16384, 16384) |
| 476 | CALLFN(·call32768, 32768) |
| 477 | CALLFN(·call65536, 65536) |
| 478 | CALLFN(·call131072, 131072) |
| 479 | CALLFN(·call262144, 262144) |
| 480 | CALLFN(·call524288, 524288) |
| 481 | CALLFN(·call1048576, 1048576) |
| 482 | CALLFN(·call2097152, 2097152) |
| 483 | CALLFN(·call4194304, 4194304) |
| 484 | CALLFN(·call8388608, 8388608) |
| 485 | CALLFN(·call16777216, 16777216) |
| 486 | CALLFN(·call33554432, 33554432) |
| 487 | CALLFN(·call67108864, 67108864) |
| 488 | CALLFN(·call134217728, 134217728) |
| 489 | CALLFN(·call268435456, 268435456) |
| 490 | CALLFN(·call536870912, 536870912) |
| 491 | CALLFN(·call1073741824, 1073741824) |
| 492 | |
Michael Munday | 3436f07 | 2016-08-30 15:25:28 -0400 | [diff] [blame] | 493 | // Not a function: target for EXRL (execute relative long) instruction. |
| 494 | TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0 |
| 495 | MVC $1, 0(R4), 0(R6) |
| 496 | |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 497 | TEXT runtime·procyield(SB),NOSPLIT,$0-0 |
| 498 | RET |
| 499 | |
| 500 | // void jmpdefer(fv, sp); |
| 501 | // called from deferreturn. |
| 502 | // 1. grab stored LR for caller |
| 503 | // 2. sub 6 bytes to get back to BL deferreturn (size of BRASL instruction) |
| 504 | // 3. BR to fn |
| 505 | TEXT runtime·jmpdefer(SB),NOSPLIT|NOFRAME,$0-16 |
| 506 | MOVD 0(R15), R1 |
| 507 | SUB $6, R1, LR |
| 508 | |
| 509 | MOVD fv+0(FP), R12 |
| 510 | MOVD argp+8(FP), R15 |
| 511 | SUB $8, R15 |
| 512 | MOVD 0(R12), R3 |
| 513 | BR (R3) |
| 514 | |
Austin Clements | 70c107c | 2016-10-19 15:49:31 -0400 | [diff] [blame] | 515 | // Save state of caller into g->sched. Smashes R1. |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 516 | TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 |
| 517 | MOVD LR, (g_sched+gobuf_pc)(g) |
| 518 | MOVD R15, (g_sched+gobuf_sp)(g) |
| 519 | MOVD $0, (g_sched+gobuf_lr)(g) |
| 520 | MOVD $0, (g_sched+gobuf_ret)(g) |
Austin Clements | 70c107c | 2016-10-19 15:49:31 -0400 | [diff] [blame] | 521 | // Assert ctxt is zero. See func save. |
| 522 | MOVD (g_sched+gobuf_ctxt)(g), R1 |
| 523 | CMPBEQ R1, $0, 2(PC) |
| 524 | BL runtime·badctxt(SB) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 525 | RET |
| 526 | |
| 527 | // func asmcgocall(fn, arg unsafe.Pointer) int32 |
| 528 | // Call fn(arg) on the scheduler stack, |
| 529 | // aligned appropriately for the gcc ABI. |
| 530 | // See cgocall.go for more details. |
| 531 | TEXT ·asmcgocall(SB),NOSPLIT,$0-20 |
| 532 | // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer |
| 533 | // C TLS base pointer in AR0:AR1 |
| 534 | MOVD fn+0(FP), R3 |
| 535 | MOVD arg+8(FP), R4 |
| 536 | |
| 537 | MOVD R15, R2 // save original stack pointer |
| 538 | MOVD g, R5 |
| 539 | |
| 540 | // Figure out if we need to switch to m->g0 stack. |
| 541 | // We get called to create new OS threads too, and those |
| 542 | // come in on the m->g0 stack already. |
| 543 | MOVD g_m(g), R6 |
| 544 | MOVD m_g0(R6), R6 |
| 545 | CMPBEQ R6, g, g0 |
| 546 | BL gosave<>(SB) |
| 547 | MOVD R6, g |
| 548 | BL runtime·save_g(SB) |
| 549 | MOVD (g_sched+gobuf_sp)(g), R15 |
| 550 | |
| 551 | // Now on a scheduling stack (a pthread-created stack). |
| 552 | g0: |
| 553 | // Save room for two of our pointers, plus 160 bytes of callee |
| 554 | // save area that lives on the caller stack. |
| 555 | SUB $176, R15 |
| 556 | MOVD $~7, R6 |
| 557 | AND R6, R15 // 8-byte alignment for gcc ABI |
| 558 | MOVD R5, 168(R15) // save old g on stack |
| 559 | MOVD (g_stack+stack_hi)(R5), R5 |
| 560 | SUB R2, R5 |
| 561 | MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) |
Michael Munday | f1515a0 | 2016-09-12 13:33:00 -0400 | [diff] [blame] | 562 | MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 563 | MOVD R4, R2 // arg in R2 |
| 564 | BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15 |
| 565 | |
| 566 | XOR R0, R0 // set R0 back to 0. |
| 567 | // Restore g, stack pointer. |
| 568 | MOVD 168(R15), g |
| 569 | BL runtime·save_g(SB) |
| 570 | MOVD (g_stack+stack_hi)(g), R5 |
| 571 | MOVD 160(R15), R6 |
| 572 | SUB R6, R5 |
| 573 | MOVD R5, R15 |
| 574 | |
| 575 | MOVW R2, ret+16(FP) |
| 576 | RET |
| 577 | |
Austin Clements | 30c1887 | 2020-10-01 17:22:38 -0400 | [diff] [blame] | 578 | // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 579 | // See cgocall.go for more details. |
Austin Clements | 30c1887 | 2020-10-01 17:22:38 -0400 | [diff] [blame] | 580 | TEXT ·cgocallback(SB),NOSPLIT,$24-24 |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 581 | NO_LOCAL_POINTERS |
| 582 | |
| 583 | // Load m and g from thread-local storage. |
| 584 | MOVB runtime·iscgo(SB), R3 |
| 585 | CMPBEQ R3, $0, nocgo |
| 586 | BL runtime·load_g(SB) |
| 587 | |
| 588 | nocgo: |
| 589 | // If g is nil, Go did not create the current thread. |
| 590 | // Call needm to obtain one for temporary use. |
| 591 | // In this case, we're running on the thread stack, so there's |
| 592 | // lots of space, but the linker doesn't know. Hide the call from |
| 593 | // the linker analysis by using an indirect call. |
| 594 | CMPBEQ g, $0, needm |
| 595 | |
| 596 | MOVD g_m(g), R8 |
| 597 | MOVD R8, savedm-8(SP) |
| 598 | BR havem |
| 599 | |
| 600 | needm: |
| 601 | MOVD g, savedm-8(SP) // g is zero, so is m. |
| 602 | MOVD $runtime·needm(SB), R3 |
| 603 | BL (R3) |
| 604 | |
| 605 | // Set m->sched.sp = SP, so that if a panic happens |
| 606 | // during the function we are about to execute, it will |
| 607 | // have a valid SP to run on the g0 stack. |
| 608 | // The next few lines (after the havem label) |
| 609 | // will save this SP onto the stack and then write |
| 610 | // the same SP back to m->sched.sp. That seems redundant, |
| 611 | // but if an unrecovered panic happens, unwindm will |
| 612 | // restore the g->sched.sp from the stack location |
| 613 | // and then systemstack will try to use it. If we don't set it here, |
| 614 | // that restored SP will be uninitialized (typically 0) and |
| 615 | // will not be usable. |
| 616 | MOVD g_m(g), R8 |
| 617 | MOVD m_g0(R8), R3 |
| 618 | MOVD R15, (g_sched+gobuf_sp)(R3) |
| 619 | |
| 620 | havem: |
| 621 | // Now there's a valid m, and we're running on its m->g0. |
| 622 | // Save current m->g0->sched.sp on stack and then set it to SP. |
| 623 | // Save current sp in m->g0->sched.sp in preparation for |
| 624 | // switch back to m->curg stack. |
| 625 | // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). |
| 626 | MOVD m_g0(R8), R3 |
| 627 | MOVD (g_sched+gobuf_sp)(R3), R4 |
Austin Clements | 30c1887 | 2020-10-01 17:22:38 -0400 | [diff] [blame] | 628 | MOVD R4, savedsp-24(SP) // must match frame size |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 629 | MOVD R15, (g_sched+gobuf_sp)(R3) |
| 630 | |
| 631 | // Switch to m->curg stack and call runtime.cgocallbackg. |
| 632 | // Because we are taking over the execution of m->curg |
| 633 | // but *not* resuming what had been running, we need to |
| 634 | // save that information (m->curg->sched) so we can restore it. |
| 635 | // We can restore m->curg->sched.sp easily, because calling |
| 636 | // runtime.cgocallbackg leaves SP unchanged upon return. |
Austin Clements | 30c1887 | 2020-10-01 17:22:38 -0400 | [diff] [blame] | 637 | // To save m->curg->sched.pc, we push it onto the curg stack and |
| 638 | // open a frame the same size as cgocallback's g0 frame. |
| 639 | // Once we switch to the curg stack, the pushed PC will appear |
| 640 | // to be the return PC of cgocallback, so that the traceback |
| 641 | // will seamlessly trace back into the earlier calls. |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 642 | MOVD m_curg(R8), g |
| 643 | BL runtime·save_g(SB) |
| 644 | MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 |
| 645 | MOVD (g_sched+gobuf_pc)(g), R5 |
Austin Clements | 30c1887 | 2020-10-01 17:22:38 -0400 | [diff] [blame] | 646 | MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size |
| 647 | // Gather our arguments into registers. |
| 648 | MOVD fn+0(FP), R1 |
| 649 | MOVD frame+8(FP), R2 |
| 650 | MOVD ctxt+16(FP), R3 |
| 651 | MOVD $-(24+8)(R4), R15 // switch stack; must match frame size |
| 652 | MOVD R1, 8(R15) |
| 653 | MOVD R2, 16(R15) |
| 654 | MOVD R3, 24(R15) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 655 | BL runtime·cgocallbackg(SB) |
| 656 | |
| 657 | // Restore g->sched (== m->curg->sched) from saved values. |
| 658 | MOVD 0(R15), R5 |
| 659 | MOVD R5, (g_sched+gobuf_pc)(g) |
Austin Clements | 30c1887 | 2020-10-01 17:22:38 -0400 | [diff] [blame] | 660 | MOVD $(24+8)(R15), R4 // must match frame size |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 661 | MOVD R4, (g_sched+gobuf_sp)(g) |
| 662 | |
| 663 | // Switch back to m->g0's stack and restore m->g0->sched.sp. |
| 664 | // (Unlike m->curg, the g0 goroutine never uses sched.pc, |
| 665 | // so we do not have to restore it.) |
| 666 | MOVD g_m(g), R8 |
| 667 | MOVD m_g0(R8), g |
| 668 | BL runtime·save_g(SB) |
| 669 | MOVD (g_sched+gobuf_sp)(g), R15 |
Austin Clements | 30c1887 | 2020-10-01 17:22:38 -0400 | [diff] [blame] | 670 | MOVD savedsp-24(SP), R4 // must match frame size |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 671 | MOVD R4, (g_sched+gobuf_sp)(g) |
| 672 | |
| 673 | // If the m on entry was nil, we called needm above to borrow an m |
| 674 | // for the duration of the call. Since the call is over, return it with dropm. |
| 675 | MOVD savedm-8(SP), R6 |
| 676 | CMPBNE R6, $0, droppedm |
| 677 | MOVD $runtime·dropm(SB), R3 |
| 678 | BL (R3) |
| 679 | droppedm: |
| 680 | |
| 681 | // Done! |
| 682 | RET |
| 683 | |
| 684 | // void setg(G*); set g. for use by needm. |
| 685 | TEXT runtime·setg(SB), NOSPLIT, $0-8 |
| 686 | MOVD gg+0(FP), g |
| 687 | // This only happens if iscgo, so jump straight to save_g |
| 688 | BL runtime·save_g(SB) |
| 689 | RET |
| 690 | |
| 691 | // void setg_gcc(G*); set g in C TLS. |
| 692 | // Must obey the gcc calling convention. |
| 693 | TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0 |
| 694 | // The standard prologue clobbers LR (R14), which is callee-save in |
| 695 | // the C ABI, so we have to use NOFRAME and save LR ourselves. |
| 696 | MOVD LR, R1 |
| 697 | // Also save g, R10, and R11 since they're callee-save in C ABI |
| 698 | MOVD R10, R3 |
| 699 | MOVD g, R4 |
| 700 | MOVD R11, R5 |
| 701 | |
| 702 | MOVD R2, g |
| 703 | BL runtime·save_g(SB) |
| 704 | |
| 705 | MOVD R5, R11 |
| 706 | MOVD R4, g |
| 707 | MOVD R3, R10 |
| 708 | MOVD R1, LR |
| 709 | RET |
| 710 | |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 711 | TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 |
| 712 | MOVW (R0), R0 |
| 713 | UNDEF |
| 714 | |
| 715 | // int64 runtime·cputicks(void) |
| 716 | TEXT runtime·cputicks(SB),NOSPLIT,$0-8 |
| 717 | // The TOD clock on s390 counts from the year 1900 in ~250ps intervals. |
| 718 | // This means that since about 1972 the msb has been set, making the |
| 719 | // result of a call to STORE CLOCK (stck) a negative number. |
| 720 | // We clear the msb to make it positive. |
| 721 | STCK ret+0(FP) // serialises before and after call |
| 722 | MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043 |
| 723 | SLD $1, R3 |
| 724 | SRD $1, R3 |
| 725 | MOVD R3, ret+0(FP) |
| 726 | RET |
| 727 | |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 728 | // AES hashing not implemented for s390x |
Keith Randall | fbfb41e | 2019-08-20 11:03:13 -0700 | [diff] [blame] | 729 | TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32 |
| 730 | JMP runtime·memhashFallback(SB) |
| 731 | TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24 |
| 732 | JMP runtime·strhashFallback(SB) |
| 733 | TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24 |
| 734 | JMP runtime·memhash32Fallback(SB) |
| 735 | TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24 |
| 736 | JMP runtime·memhash64Fallback(SB) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 737 | |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 738 | TEXT runtime·return0(SB), NOSPLIT, $0 |
| 739 | MOVW $0, R3 |
| 740 | RET |
| 741 | |
| 742 | // Called from cgo wrappers, this function returns g->m->curg.stack.hi. |
| 743 | // Must obey the gcc calling convention. |
| 744 | TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0 |
| 745 | // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them |
| 746 | MOVD g, R1 |
| 747 | MOVD R10, R3 |
| 748 | MOVD LR, R4 |
| 749 | MOVD R11, R5 |
| 750 | |
| 751 | BL runtime·load_g(SB) // clobbers g (R13), R10, R11 |
| 752 | MOVD g_m(g), R2 |
| 753 | MOVD m_curg(R2), R2 |
| 754 | MOVD (g_stack+stack_hi)(R2), R2 |
| 755 | |
| 756 | MOVD R1, g |
| 757 | MOVD R3, R10 |
| 758 | MOVD R4, LR |
| 759 | MOVD R5, R11 |
| 760 | RET |
| 761 | |
| 762 | // The top-most function running on a goroutine |
| 763 | // returns to goexit+PCQuantum. |
Michael Munday | aafe257 | 2019-03-28 12:51:30 -0400 | [diff] [blame] | 764 | TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 765 | BYTE $0x07; BYTE $0x00; // 2-byte nop |
| 766 | BL runtime·goexit1(SB) // does not return |
| 767 | // traceback from goexit1 must hit code range of goexit |
| 768 | BYTE $0x07; BYTE $0x00; // 2-byte nop |
| 769 | |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 770 | TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 |
Russ Cox | 26a5f6a | 2019-06-05 13:57:47 -0400 | [diff] [blame] | 771 | // Stores are already ordered on s390x, so this is just a |
| 772 | // compile barrier. |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 773 | RET |
| 774 | |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 775 | // This is called from .init_array and follows the platform, not Go, ABI. |
| 776 | // We are overly conservative. We could only save the registers we use. |
| 777 | // However, since this function is only called once per loaded module |
| 778 | // performance is unimportant. |
| 779 | TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0 |
Michael Munday | 06fcc32 | 2016-11-23 14:54:12 -0500 | [diff] [blame] | 780 | // Save R6-R15 in the register save area of the calling function. |
| 781 | // Don't bother saving F8-F15 as we aren't doing any calls. |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 782 | STMG R6, R15, 48(R15) |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 783 | |
| 784 | // append the argument (passed in R2, as per the ELF ABI) to the |
| 785 | // moduledata linked list. |
| 786 | MOVD runtime·lastmoduledatap(SB), R1 |
| 787 | MOVD R2, moduledata_next(R1) |
| 788 | MOVD R2, runtime·lastmoduledatap(SB) |
| 789 | |
Michael Munday | 06fcc32 | 2016-11-23 14:54:12 -0500 | [diff] [blame] | 790 | // Restore R6-R15. |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 791 | LMG 48(R15), R6, R15 |
Michael Munday | e6f36f0 | 2016-03-18 19:09:39 -0400 | [diff] [blame] | 792 | RET |
| 793 | |
| 794 | TEXT ·checkASM(SB),NOSPLIT,$0-1 |
| 795 | MOVB $1, ret+0(FP) |
| 796 | RET |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 797 | |
| 798 | // gcWriteBarrier performs a heap pointer write and informs the GC. |
| 799 | // |
| 800 | // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments: |
| 801 | // - R2 is the destination of the write |
| 802 | // - R3 is the value being written at R2. |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 803 | // It clobbers R10 (the temp register). |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 804 | // It does not clobber any other general-purpose registers, |
| 805 | // but may clobber others (e.g., floating point registers). |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 806 | TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$104 |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 807 | // Save the registers clobbered by the fast path. |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 808 | MOVD R1, 96(R15) |
| 809 | MOVD R4, 104(R15) |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 810 | MOVD g_m(g), R1 |
| 811 | MOVD m_p(R1), R1 |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 812 | // Increment wbBuf.next position. |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 813 | MOVD $16, R4 |
| 814 | ADD (p_wbBuf+wbBuf_next)(R1), R4 |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 815 | MOVD R4, (p_wbBuf+wbBuf_next)(R1) |
| 816 | MOVD (p_wbBuf+wbBuf_end)(R1), R1 |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 817 | // Record the write. |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 818 | MOVD R3, -16(R4) // Record value |
| 819 | MOVD (R2), R10 // TODO: This turns bad writes into bad reads. |
| 820 | MOVD R10, -8(R4) // Record *slot |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 821 | // Is the buffer full? |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 822 | CMPBEQ R4, R1, flush |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 823 | ret: |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 824 | MOVD 96(R15), R1 |
| 825 | MOVD 104(R15), R4 |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 826 | // Do the write. |
| 827 | MOVD R3, (R2) |
| 828 | RET |
| 829 | |
| 830 | flush: |
| 831 | // Save all general purpose registers since these could be |
| 832 | // clobbered by wbBufFlush and were not saved by the caller. |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 833 | STMG R2, R3, 8(R15) // set R2 and R3 as arguments for wbBufFlush |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 834 | MOVD R0, 24(R15) |
| 835 | // R1 already saved. |
| 836 | // R4 already saved. |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 837 | STMG R5, R12, 32(R15) // save R5 - R12 |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 838 | // R13 is g. |
| 839 | // R14 is LR. |
| 840 | // R15 is SP. |
| 841 | |
| 842 | // This takes arguments R2 and R3. |
| 843 | CALL runtime·wbBufFlush(SB) |
| 844 | |
Michael Munday | 5d9c782 | 2018-04-30 16:55:13 +0100 | [diff] [blame] | 845 | LMG 8(R15), R2, R3 // restore R2 - R3 |
| 846 | MOVD 24(R15), R0 // restore R0 |
| 847 | LMG 32(R15), R5, R12 // restore R5 - R12 |
Austin Clements | 2ae1e1a | 2017-11-15 14:54:24 -0800 | [diff] [blame] | 848 | JMP ret |
Keith Randall | 2c423f0 | 2019-02-06 14:12:36 -0800 | [diff] [blame] | 849 | |
| 850 | // Note: these functions use a special calling convention to save generated code space. |
| 851 | // Arguments are passed in registers, but the space for those arguments are allocated |
| 852 | // in the caller's stack frame. These stubs write the args into that stack space and |
| 853 | // then tail call to the corresponding runtime handler. |
| 854 | // The tail call makes these stubs disappear in backtraces. |
| 855 | TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 |
| 856 | MOVD R0, x+0(FP) |
| 857 | MOVD R1, y+8(FP) |
| 858 | JMP runtime·goPanicIndex(SB) |
| 859 | TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 |
| 860 | MOVD R0, x+0(FP) |
| 861 | MOVD R1, y+8(FP) |
| 862 | JMP runtime·goPanicIndexU(SB) |
| 863 | TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 |
| 864 | MOVD R1, x+0(FP) |
| 865 | MOVD R2, y+8(FP) |
| 866 | JMP runtime·goPanicSliceAlen(SB) |
| 867 | TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 |
| 868 | MOVD R1, x+0(FP) |
| 869 | MOVD R2, y+8(FP) |
| 870 | JMP runtime·goPanicSliceAlenU(SB) |
| 871 | TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 |
| 872 | MOVD R1, x+0(FP) |
| 873 | MOVD R2, y+8(FP) |
| 874 | JMP runtime·goPanicSliceAcap(SB) |
| 875 | TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 |
| 876 | MOVD R1, x+0(FP) |
| 877 | MOVD R2, y+8(FP) |
| 878 | JMP runtime·goPanicSliceAcapU(SB) |
| 879 | TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 |
| 880 | MOVD R0, x+0(FP) |
| 881 | MOVD R1, y+8(FP) |
| 882 | JMP runtime·goPanicSliceB(SB) |
| 883 | TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 |
| 884 | MOVD R0, x+0(FP) |
| 885 | MOVD R1, y+8(FP) |
| 886 | JMP runtime·goPanicSliceBU(SB) |
| 887 | TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 |
| 888 | MOVD R2, x+0(FP) |
| 889 | MOVD R3, y+8(FP) |
| 890 | JMP runtime·goPanicSlice3Alen(SB) |
| 891 | TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 |
| 892 | MOVD R2, x+0(FP) |
| 893 | MOVD R3, y+8(FP) |
| 894 | JMP runtime·goPanicSlice3AlenU(SB) |
| 895 | TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 |
| 896 | MOVD R2, x+0(FP) |
| 897 | MOVD R3, y+8(FP) |
| 898 | JMP runtime·goPanicSlice3Acap(SB) |
| 899 | TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 |
| 900 | MOVD R2, x+0(FP) |
| 901 | MOVD R3, y+8(FP) |
| 902 | JMP runtime·goPanicSlice3AcapU(SB) |
| 903 | TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 |
| 904 | MOVD R1, x+0(FP) |
| 905 | MOVD R2, y+8(FP) |
| 906 | JMP runtime·goPanicSlice3B(SB) |
| 907 | TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 |
| 908 | MOVD R1, x+0(FP) |
| 909 | MOVD R2, y+8(FP) |
| 910 | JMP runtime·goPanicSlice3BU(SB) |
| 911 | TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 |
| 912 | MOVD R0, x+0(FP) |
| 913 | MOVD R1, y+8(FP) |
| 914 | JMP runtime·goPanicSlice3C(SB) |
| 915 | TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 |
| 916 | MOVD R0, x+0(FP) |
| 917 | MOVD R1, y+8(FP) |
| 918 | JMP runtime·goPanicSlice3CU(SB) |