blob: d7c45a183dca1cf8ae8dcce063aa8497657cbb65 [file] [log] [blame]
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "go_asm.h"
#include "funcdata.h"
#include "textflag.h"
// func rt0_go()
TEXT runtime·rt0_go(SB),NOSPLIT,$0
// X2 = stack; A0 = argc; A1 = argv
ADD $-24, X2
MOV A0, 8(X2) // argc
MOV A1, 16(X2) // argv
// create istack out of the given (operating system) stack.
// _cgo_init may update stackguard.
MOV $runtime·g0(SB), g
MOV $(-64*1024), T0
ADD T0, X2, T1
MOV T1, g_stackguard0(g)
MOV T1, g_stackguard1(g)
MOV T1, (g_stack+stack_lo)(g)
MOV X2, (g_stack+stack_hi)(g)
// if there is a _cgo_init, call it using the gcc ABI.
MOV _cgo_init(SB), T0
BEQ T0, ZERO, nocgo
MOV ZERO, A3 // arg 3: not used
MOV ZERO, A2 // arg 2: not used
MOV $setg_gcc<>(SB), A1 // arg 1: setg
MOV g, A0 // arg 0: G
// update stackguard after _cgo_init
MOV (g_stack+stack_lo)(g), T0
ADD $const__StackGuard, T0
MOV T0, g_stackguard0(g)
MOV T0, g_stackguard1(g)
// set the per-goroutine and per-mach "registers"
MOV $runtime·m0(SB), T0
// save m->g0 = g0
MOV g, m_g0(T0)
// save m0 to g0->m
MOV T0, g_m(g)
CALL runtime·check(SB)
// args are already prepared
CALL runtime·args(SB)
CALL runtime·osinit(SB)
CALL runtime·schedinit(SB)
// create a new goroutine to start program
MOV $runtime·mainPC(SB), T0 // entry
ADD $-24, X2
MOV T0, 16(X2)
CALL runtime·newproc(SB)
ADD $24, X2
// start this M
CALL runtime·mstart(SB)
WORD $0 // crash if reached
// void setg_gcc(G*); set g called from gcc with g in A0
TEXT setg_gcc<>(SB),NOSPLIT,$0-0
MOV A0, g
CALL runtime·save_g(SB)
// func cputicks() int64
TEXT runtime·cputicks(SB),NOSPLIT,$0-8
WORD $0xc0102573 // rdtime a0
MOV A0, ret+0(FP)
// systemstack_switch is a dummy routine that systemstack leaves at the bottom
// of the G stack. We need to distinguish the routine that
// lives at the bottom of the G stack from the one that lives
// at the top of the system stack because the one at the top of
// the system stack terminates the stack walk (see topofstack()).
TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
JALR RA, ZERO // make sure this function is not leaf
// func systemstack(fn func())
TEXT runtime·systemstack(SB), NOSPLIT, $0-8
MOV fn+0(FP), CTXT // CTXT = fn
MOV g_m(g), T0 // T0 = m
MOV m_gsignal(T0), T1 // T1 = gsignal
BEQ g, T1, noswitch
MOV m_g0(T0), T1 // T1 = g0
BEQ g, T1, noswitch
MOV m_curg(T0), T2
BEQ g, T2, switch
// Bad: g is not gsignal, not g0, not curg. What is it?
// Hide call from linker nosplit analysis.
MOV $runtime·badsystemstack(SB), T1
// save our state in g->sched. Pretend to
// be systemstack_switch if the G stack is scanned.
MOV $runtime·systemstack_switch(SB), T2
ADD $8, T2 // get past prologue
MOV T2, (g_sched+gobuf_pc)(g)
MOV X2, (g_sched+gobuf_sp)(g)
MOV ZERO, (g_sched+gobuf_lr)(g)
MOV g, (g_sched+gobuf_g)(g)
// switch to g0
MOV T1, g
CALL runtime·save_g(SB)
MOV (g_sched+gobuf_sp)(g), T0
// make it look like mstart called systemstack on g0, to stop traceback
ADD $-8, T0
MOV $runtime·mstart(SB), T1
MOV T1, 0(T0)
MOV T0, X2
// call target function
MOV 0(CTXT), T1 // code pointer
// switch back to g
MOV g_m(g), T0
MOV m_curg(T0), g
CALL runtime·save_g(SB)
MOV (g_sched+gobuf_sp)(g), X2
MOV ZERO, (g_sched+gobuf_sp)(g)
// already on m stack, just call directly
// Using a tail call here cleans up tracebacks since we won't stop
// at an intermediate systemstack.
MOV 0(CTXT), T1 // code pointer
ADD $8, X2
JMP (T1)
TEXT runtime·getcallerpc(SB),NOSPLIT|NOFRAME,$0-8
MOV 0(X2), T0 // LR saved by caller
MOV T0, ret+0(FP)
* support for morestack
// Called during function prolog when more stack is needed.
// Caller has already loaded:
// R1: framesize, R2: argsize, R3: LR
// The traceback routines see morestack on a g0 as being
// the top of a stack (for example, morestack calling newstack
// calling the scheduler calling newm calling gc), so we must
// record an argument size. For that purpose, it has no arguments.
// func morestack()
TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
// Cannot grow scheduler stack (m->g0).
MOV g_m(g), A0
MOV m_g0(A0), A1
BNE g, A1, 3(PC)
CALL runtime·badmorestackg0(SB)
CALL runtime·abort(SB)
// Cannot grow signal stack (m->gsignal).
MOV m_gsignal(A0), A1
BNE g, A1, 3(PC)
CALL runtime·badmorestackgsignal(SB)
CALL runtime·abort(SB)
// Called from f.
// Set g->sched to context in f.
MOV X2, (g_sched+gobuf_sp)(g)
MOV T0, (g_sched+gobuf_pc)(g)
MOV RA, (g_sched+gobuf_lr)(g)
MOV CTXT, (g_sched+gobuf_ctxt)(g)
// Called from f.
// Set m->morebuf to f's caller.
MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
MOV g, (m_morebuf+gobuf_g)(A0)
// Call newstack on m->g0's stack.
MOV m_g0(A0), g
CALL runtime·save_g(SB)
MOV (g_sched+gobuf_sp)(g), X2
// Create a stack frame on g0 to call newstack.
MOV ZERO, -8(X2) // Zero saved LR in frame
ADD $-8, X2
CALL runtime·newstack(SB)
// Not reached, but make sure the return PC from the call to newstack
// is still in this function, and not the beginning of the next.
// func morestack_noctxt()
TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
JMP runtime·morestack(SB)
// AES hashing not implemented for riscv64
TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
JMP runtime·memhashFallback(SB)
TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
JMP runtime·strhashFallback(SB)
TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
JMP runtime·memhash32Fallback(SB)
TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
JMP runtime·memhash64Fallback(SB)
// func return0()
TEXT runtime·return0(SB), NOSPLIT, $0
MOV $0, A0
// restore state from Gobuf; longjmp
// func gogo(buf *gobuf)
TEXT runtime·gogo(SB), NOSPLIT, $16-8
MOV buf+0(FP), T0
MOV gobuf_g(T0), g // make sure g is not nil
CALL runtime·save_g(SB)
MOV (g), ZERO // make sure g is not nil
MOV gobuf_sp(T0), X2
MOV gobuf_lr(T0), RA
MOV gobuf_ret(T0), A0
MOV gobuf_ctxt(T0), CTXT
MOV ZERO, gobuf_sp(T0)
MOV ZERO, gobuf_ret(T0)
MOV ZERO, gobuf_lr(T0)
MOV ZERO, gobuf_ctxt(T0)
MOV gobuf_pc(T0), T0
// func jmpdefer(fv *funcval, argp uintptr)
// called from deferreturn
// 1. grab stored return address from the caller's frame
// 2. sub 8 bytes to get back to JAL deferreturn
// 3. JMP to fn
TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16
MOV 0(X2), RA
ADD $-8, RA
MOV fv+0(FP), CTXT
MOV argp+8(FP), X2
ADD $-8, X2
// func procyield(cycles uint32)
TEXT runtime·procyield(SB),NOSPLIT,$0-0
// Switch to m->g0's stack, call fn(g).
// Fn must never return. It should gogo(&g->sched)
// to keep running g.
// func mcall(fn func(*g))
TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8
// Save caller state in g->sched
MOV X2, (g_sched+gobuf_sp)(g)
MOV RA, (g_sched+gobuf_pc)(g)
MOV ZERO, (g_sched+gobuf_lr)(g)
MOV g, (g_sched+gobuf_g)(g)
// Switch to m->g0 & its stack, call fn.
MOV g, T0
MOV g_m(g), T1
MOV m_g0(T1), g
CALL runtime·save_g(SB)
BNE g, T0, 2(PC)
JMP runtime·badmcall(SB)
MOV fn+0(FP), CTXT // context
MOV 0(CTXT), T1 // code pointer
MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
ADD $-16, X2
MOV T0, 8(X2)
JMP runtime·badmcall2(SB)
// func gosave(buf *gobuf)
// save state in Gobuf; setjmp
TEXT runtime·gosave(SB), NOSPLIT|NOFRAME, $0-8
MOV buf+0(FP), T1
MOV X2, gobuf_sp(T1)
MOV RA, gobuf_pc(T1)
MOV g, gobuf_g(T1)
MOV ZERO, gobuf_lr(T1)
MOV ZERO, gobuf_ret(T1)
// Assert ctxt is zero. See func save.
MOV gobuf_ctxt(T1), T1
CALL runtime·badctxt(SB)
// func asmcgocall(fn, arg unsafe.Pointer) int32
TEXT ·asmcgocall(SB),NOSPLIT,$0-20
// TODO(jsing): Add support for cgo - issue #36641.
WORD $0 // crash
// func asminit()
TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
// reflectcall: call a function with the given argument list
// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
BLTU T1, T0, 3(PC) \
MOV $NAME(SB), T2; \
// Note: can't just "BR NAME(SB)" - bad inlining results.
// func call(argtype *rtype, fn, arg unsafe.Pointer, n uint32, retoffset uint32)
TEXT reflect·call(SB), NOSPLIT, $0-0
JMP ·reflectcall(SB)
// func reflectcall(argtype *_type, fn, arg unsafe.Pointer, argsize uint32, retoffset uint32)
TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
MOVWU argsize+24(FP), T0
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
DISPATCH(runtime·call128, 128)
DISPATCH(runtime·call256, 256)
DISPATCH(runtime·call512, 512)
DISPATCH(runtime·call1024, 1024)
DISPATCH(runtime·call2048, 2048)
DISPATCH(runtime·call4096, 4096)
DISPATCH(runtime·call8192, 8192)
DISPATCH(runtime·call16384, 16384)
DISPATCH(runtime·call32768, 32768)
DISPATCH(runtime·call65536, 65536)
DISPATCH(runtime·call131072, 131072)
DISPATCH(runtime·call262144, 262144)
DISPATCH(runtime·call524288, 524288)
DISPATCH(runtime·call1048576, 1048576)
DISPATCH(runtime·call2097152, 2097152)
DISPATCH(runtime·call4194304, 4194304)
DISPATCH(runtime·call8388608, 8388608)
DISPATCH(runtime·call16777216, 16777216)
DISPATCH(runtime·call33554432, 33554432)
DISPATCH(runtime·call67108864, 67108864)
DISPATCH(runtime·call134217728, 134217728)
DISPATCH(runtime·call268435456, 268435456)
DISPATCH(runtime·call536870912, 536870912)
DISPATCH(runtime·call1073741824, 1073741824)
MOV $runtime·badreflectcall(SB), T2
/* copy arguments to stack */ \
MOV arg+16(FP), A1; \
MOVWU argsize+24(FP), A2; \
MOV X2, A3; \
ADD $8, A3; \
ADD A3, A2; \
BEQ A3, A2, 6(PC); \
MOVBU (A1), A4; \
ADD $1, A1; \
MOVB A4, (A3); \
ADD $1, A3; \
JMP -5(PC); \
/* call function */ \
MOV f+8(FP), CTXT; \
MOV (CTXT), A4; \
PCDATA $PCDATA_StackMapIndex, $0; \
JALR RA, A4; \
/* copy return values back */ \
MOV argtype+0(FP), A5; \
MOV arg+16(FP), A1; \
MOVWU n+24(FP), A2; \
MOVWU retoffset+28(FP), A4; \
ADD $8, X2, A3; \
ADD A4, A3; \
ADD A4, A1; \
SUB A4, A2; \
CALL callRet<>(SB); \
// callRet copies return values back at the end of call*. This is a
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
TEXT callRet<>(SB), NOSPLIT, $32-0
MOV A5, 8(X2)
MOV A1, 16(X2)
MOV A3, 24(X2)
MOV A2, 32(X2)
CALL runtime·reflectcallmove(SB)
CALLFN(·call16, 16)
CALLFN(·call32, 32)
CALLFN(·call64, 64)
CALLFN(·call128, 128)
CALLFN(·call256, 256)
CALLFN(·call512, 512)
CALLFN(·call1024, 1024)
CALLFN(·call2048, 2048)
CALLFN(·call4096, 4096)
CALLFN(·call8192, 8192)
CALLFN(·call16384, 16384)
CALLFN(·call32768, 32768)
CALLFN(·call65536, 65536)
CALLFN(·call131072, 131072)
CALLFN(·call262144, 262144)
CALLFN(·call524288, 524288)
CALLFN(·call1048576, 1048576)
CALLFN(·call2097152, 2097152)
CALLFN(·call4194304, 4194304)
CALLFN(·call8388608, 8388608)
CALLFN(·call16777216, 16777216)
CALLFN(·call33554432, 33554432)
CALLFN(·call67108864, 67108864)
CALLFN(·call134217728, 134217728)
CALLFN(·call268435456, 268435456)
CALLFN(·call536870912, 536870912)
CALLFN(·call1073741824, 1073741824)
// func goexit(neverCallThisFunction)
// The top-most function running on a goroutine
// returns to goexit+PCQuantum.
JMP runtime·goexit1(SB) // does not return
// traceback from goexit1 must hit code range of goexit
// func cgocallback_gofunc(fv uintptr, frame uintptr, framesize, ctxt uintptr)
TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32
// TODO(jsing): Add support for cgo - issue #36641.
WORD $0 // crash
TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
// void setg(G*); set g. for use by needm.
TEXT runtime·setg(SB), NOSPLIT, $0-8
MOV gg+0(FP), g
// This only happens if iscgo, so jump straight to save_g
CALL runtime·save_g(SB)
MOV $1, T0
MOV T0, ret+0(FP)
// gcWriteBarrier performs a heap pointer write and informs the GC.
// gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
// - T0 is the destination of the write
// - T1 is the value being written at T0.
// It clobbers R30 (the linker temp register - REG_TMP).
// The act of CALLing gcWriteBarrier will clobber RA (LR).
// It does not clobber any other general-purpose registers,
// but may clobber others (e.g., floating point registers).
TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$296
// Save the registers clobbered by the fast path.
MOV A0, 280(X2)
MOV A1, 288(X2)
MOV g_m(g), A0
MOV m_p(A0), A0
MOV (p_wbBuf+wbBuf_next)(A0), A1
// Increment position.
ADD $16, A1
MOV A1, (p_wbBuf+wbBuf_next)(A0)
MOV (p_wbBuf+wbBuf_end)(A0), A0
MOV A0, T6 // T6 is linker temp register (REG_TMP)
// Record the write.
MOV T1, -16(A1) // Record value
MOV (T0), A0 // TODO: This turns bad writes into bad reads.
MOV A0, -8(A1) // Record *slot
// Is the buffer full?
BEQ A1, T6, flush
MOV 280(X2), A0
MOV 288(X2), A1
// Do the write.
MOV T1, (T0)
// Save all general purpose registers since these could be
// clobbered by wbBufFlush and were not saved by the caller.
MOV T0, 8(X2) // Also first argument to wbBufFlush
MOV T1, 16(X2) // Also second argument to wbBufFlush
// TODO: Optimise
// R3 is g.
// R4 already saved (T0)
// R5 already saved (T1)
// R9 already saved (A0)
// R10 already saved (A1)
// R30 is tmp register.
MOV X0, 24(X2)
MOV X1, 32(X2)
MOV X2, 40(X2)
MOV X3, 48(X2)
MOV X4, 56(X2)
MOV X5, 64(X2)
MOV X6, 72(X2)
MOV X7, 80(X2)
MOV X8, 88(X2)
MOV X9, 96(X2)
MOV X10, 104(X2)
MOV X11, 112(X2)
MOV X12, 120(X2)
MOV X13, 128(X2)
MOV X14, 136(X2)
MOV X15, 144(X2)
MOV X16, 152(X2)
MOV X17, 160(X2)
MOV X18, 168(X2)
MOV X19, 176(X2)
MOV X20, 184(X2)
MOV X21, 192(X2)
MOV X22, 200(X2)
MOV X23, 208(X2)
MOV X24, 216(X2)
MOV X25, 224(X2)
MOV X26, 232(X2)
MOV X27, 240(X2)
MOV X28, 248(X2)
MOV X29, 256(X2)
MOV X30, 264(X2)
MOV X31, 272(X2)
// This takes arguments T0 and T1.
CALL runtime·wbBufFlush(SB)
MOV 24(X2), X0
MOV 32(X2), X1
MOV 40(X2), X2
MOV 48(X2), X3
MOV 56(X2), X4
MOV 64(X2), X5
MOV 72(X2), X6
MOV 80(X2), X7
MOV 88(X2), X8
MOV 96(X2), X9
MOV 104(X2), X10
MOV 112(X2), X11
MOV 120(X2), X12
MOV 128(X2), X13
MOV 136(X2), X14
MOV 144(X2), X15
MOV 152(X2), X16
MOV 160(X2), X17
MOV 168(X2), X18
MOV 176(X2), X19
MOV 184(X2), X20
MOV 192(X2), X21
MOV 200(X2), X22
MOV 208(X2), X23
MOV 216(X2), X24
MOV 224(X2), X25
MOV 232(X2), X26
MOV 240(X2), X27
MOV 248(X2), X28
MOV 256(X2), X29
MOV 264(X2), X30
MOV 272(X2), X31
JMP ret
// Note: these functions use a special calling convention to save generated code space.
// Arguments are passed in registers, but the space for those arguments are allocated
// in the caller's stack frame. These stubs write the args into that stack space and
// then tail call to the corresponding runtime handler.
// The tail call makes these stubs disappear in backtraces.
TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
MOV T0, x+0(FP)
MOV T1, y+8(FP)
JMP runtime·goPanicIndex(SB)
TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
MOV T0, x+0(FP)
MOV T1, y+8(FP)
JMP runtime·goPanicIndexU(SB)
TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
MOV T1, x+0(FP)
MOV T2, y+8(FP)
JMP runtime·goPanicSliceAlen(SB)
TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
MOV T1, x+0(FP)
MOV T2, y+8(FP)
JMP runtime·goPanicSliceAlenU(SB)
TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
MOV T1, x+0(FP)
MOV T2, y+8(FP)
JMP runtime·goPanicSliceAcap(SB)
TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
MOV T1, x+0(FP)
MOV T2, y+8(FP)
JMP runtime·goPanicSliceAcapU(SB)
TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
MOV T0, x+0(FP)
MOV T1, y+8(FP)
JMP runtime·goPanicSliceB(SB)
TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
MOV T0, x+0(FP)
MOV T1, y+8(FP)
JMP runtime·goPanicSliceBU(SB)
TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
MOV T2, x+0(FP)
MOV T3, y+8(FP)
JMP runtime·goPanicSlice3Alen(SB)
TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
MOV T2, x+0(FP)
MOV T3, y+8(FP)
JMP runtime·goPanicSlice3AlenU(SB)
TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
MOV T2, x+0(FP)
MOV T3, y+8(FP)
JMP runtime·goPanicSlice3Acap(SB)
TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
MOV T2, x+0(FP)
MOV T3, y+8(FP)
JMP runtime·goPanicSlice3AcapU(SB)
TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
MOV T1, x+0(FP)
MOV T2, y+8(FP)
JMP runtime·goPanicSlice3B(SB)
TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
MOV T1, x+0(FP)
MOV T2, y+8(FP)
JMP runtime·goPanicSlice3BU(SB)
TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
MOV T0, x+0(FP)
MOV T1, y+8(FP)
JMP runtime·goPanicSlice3C(SB)
TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
MOV T0, x+0(FP)
MOV T1, y+8(FP)
JMP runtime·goPanicSlice3CU(SB)
DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
GLOBL runtime·mainPC(SB),RODATA,$8