src/internal/runtime/atomic/atomic_arm64.s

// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "go_asm.h"
#include "textflag.h"

TEXT ·Casint32(SB), NOSPLIT, $0-17
	B	·Cas(SB)

TEXT ·Casint64(SB), NOSPLIT, $0-25
	B	·Cas64(SB)

TEXT ·Casuintptr(SB), NOSPLIT, $0-25
	B	·Cas64(SB)

TEXT ·CasRel(SB), NOSPLIT, $0-17
	B	·Cas(SB)

TEXT ·Loadint32(SB), NOSPLIT, $0-12
	B	·Load(SB)

TEXT ·Loadint64(SB), NOSPLIT, $0-16
	B	·Load64(SB)

TEXT ·Loaduintptr(SB), NOSPLIT, $0-16
	B	·Load64(SB)

TEXT ·Loaduint(SB), NOSPLIT, $0-16
	B	·Load64(SB)

TEXT ·Storeint32(SB), NOSPLIT, $0-12
	B	·Store(SB)

TEXT ·Storeint64(SB), NOSPLIT, $0-16
	B	·Store64(SB)

TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
	B	·Store64(SB)

TEXT ·Xaddint32(SB), NOSPLIT, $0-20
	B	·Xadd(SB)

TEXT ·Xaddint64(SB), NOSPLIT, $0-24
	B	·Xadd64(SB)

TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
	B	·Xadd64(SB)

TEXT ·Casp1(SB), NOSPLIT, $0-25
	B ·Cas64(SB)

// uint32 ·Load(uint32 volatile* addr)
TEXT ·Load(SB),NOSPLIT,$0-12
	MOVD	ptr+0(FP), R0
	LDARW	(R0), R0
	MOVW	R0, ret+8(FP)
	RET

// uint8 ·Load8(uint8 volatile* addr)
TEXT ·Load8(SB),NOSPLIT,$0-9
	MOVD	ptr+0(FP), R0
	LDARB	(R0), R0
	MOVB	R0, ret+8(FP)
	RET

// uint64 ·Load64(uint64 volatile* addr)
TEXT ·Load64(SB),NOSPLIT,$0-16
	MOVD	ptr+0(FP), R0
	LDAR	(R0), R0
	MOVD	R0, ret+8(FP)
	RET

// void *·Loadp(void *volatile *addr)
TEXT ·Loadp(SB),NOSPLIT,$0-16
	MOVD	ptr+0(FP), R0
	LDAR	(R0), R0
	MOVD	R0, ret+8(FP)
	RET

// uint32 ·LoadAcq(uint32 volatile* addr)
TEXT ·LoadAcq(SB),NOSPLIT,$0-12
	B	·Load(SB)

// uint64 ·LoadAcquintptr(uint64 volatile* addr)
TEXT ·LoadAcq64(SB),NOSPLIT,$0-16
	B	·Load64(SB)

// uintptr ·LoadAcq64(uintptr volatile* addr)
TEXT ·LoadAcquintptr(SB),NOSPLIT,$0-16
	B	·Load64(SB)

TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
	B	·Store64(SB)

TEXT ·StoreRel(SB), NOSPLIT, $0-12
	B	·Store(SB)

TEXT ·StoreRel64(SB), NOSPLIT, $0-16
	B	·Store64(SB)

TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
	B	·Store64(SB)

TEXT ·Store(SB), NOSPLIT, $0-12
	MOVD	ptr+0(FP), R0
	MOVW	val+8(FP), R1
	STLRW	R1, (R0)
	RET

TEXT ·Store8(SB), NOSPLIT, $0-9
	MOVD	ptr+0(FP), R0
	MOVB	val+8(FP), R1
	STLRB	R1, (R0)
	RET

TEXT ·Store64(SB), NOSPLIT, $0-16
	MOVD	ptr+0(FP), R0
	MOVD	val+8(FP), R1
	STLR	R1, (R0)
	RET

// uint8 Xchg(ptr *uint8, new uint8)
// Atomically:
//	old := *ptr;
//	*ptr = new;
//	return old;
TEXT ·Xchg8(SB), NOSPLIT, $0-17
	MOVD	ptr+0(FP), R0
	MOVB	new+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	SWPALB	R1, (R0), R2
	MOVB	R2, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRB	(R0), R2
	STLXRB	R1, (R0), R3
	CBNZ	R3, load_store_loop
	MOVB	R2, ret+16(FP)
	RET
#endif

// uint32 Xchg(ptr *uint32, new uint32)
// Atomically:
//	old := *ptr;
//	*ptr = new;
//	return old;
TEXT ·Xchg(SB), NOSPLIT, $0-20
	MOVD	ptr+0(FP), R0
	MOVW	new+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	SWPALW	R1, (R0), R2
	MOVW	R2, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRW	(R0), R2
	STLXRW	R1, (R0), R3
	CBNZ	R3, load_store_loop
	MOVW	R2, ret+16(FP)
	RET
#endif

// uint64 Xchg64(ptr *uint64, new uint64)
// Atomically:
//	old := *ptr;
//	*ptr = new;
//	return old;
TEXT ·Xchg64(SB), NOSPLIT, $0-24
	MOVD	ptr+0(FP), R0
	MOVD	new+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	SWPALD	R1, (R0), R2
	MOVD	R2, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXR	(R0), R2
	STLXR	R1, (R0), R3
	CBNZ	R3, load_store_loop
	MOVD	R2, ret+16(FP)
	RET
#endif

// func Cas(ptr *uint32, old, new uint32) bool
// Atomically:
//	if *ptr == old {
//		*ptr = new
//		return true
//	} else {
//		return false
// 	}
TEXT ·Cas(SB), NOSPLIT, $0-17
	MOVD	ptr+0(FP), R0
	MOVW	old+8(FP), R1
	MOVW	new+12(FP), R2
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	MOVD	R1, R3
	CASALW	R3, (R0), R2
	CMP 	R1, R3
	CSET	EQ, R0
	MOVB	R0, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRW	(R0), R3
	CMPW	R1, R3
	BNE	ok
	STLXRW	R2, (R0), R3
	CBNZ	R3, load_store_loop
ok:
	CSET	EQ, R0
	MOVB	R0, ret+16(FP)
	RET
#endif

// func Cas64(ptr *uint64, old, new uint64) bool
// Atomically:
//	if *ptr == old {
//		*ptr = new
//		return true
//	} else {
//		return false
//	}
TEXT ·Cas64(SB), NOSPLIT, $0-25
	MOVD	ptr+0(FP), R0
	MOVD	old+8(FP), R1
	MOVD	new+16(FP), R2
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	MOVD	R1, R3
	CASALD	R3, (R0), R2
	CMP 	R1, R3
	CSET	EQ, R0
	MOVB	R0, ret+24(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXR	(R0), R3
	CMP	R1, R3
	BNE	ok
	STLXR	R2, (R0), R3
	CBNZ	R3, load_store_loop
ok:
	CSET	EQ, R0
	MOVB	R0, ret+24(FP)
	RET
#endif

// uint32 xadd(uint32 volatile *ptr, int32 delta)
// Atomically:
//      *val += delta;
//      return *val;
TEXT ·Xadd(SB), NOSPLIT, $0-20
	MOVD	ptr+0(FP), R0
	MOVW	delta+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	LDADDALW	R1, (R0), R2
	ADD 	R1, R2
	MOVW	R2, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRW	(R0), R2
	ADDW	R2, R1, R2
	STLXRW	R2, (R0), R3
	CBNZ	R3, load_store_loop
	MOVW	R2, ret+16(FP)
	RET
#endif

// uint64 Xadd64(uint64 volatile *ptr, int64 delta)
// Atomically:
//      *val += delta;
//      return *val;
TEXT ·Xadd64(SB), NOSPLIT, $0-24
	MOVD	ptr+0(FP), R0
	MOVD	delta+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	LDADDALD	R1, (R0), R2
	ADD 	R1, R2
	MOVD	R2, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXR	(R0), R2
	ADD	R2, R1, R2
	STLXR	R2, (R0), R3
	CBNZ	R3, load_store_loop
	MOVD	R2, ret+16(FP)
	RET
#endif

TEXT ·Xchgint32(SB), NOSPLIT, $0-20
	B	·Xchg(SB)

TEXT ·Xchgint64(SB), NOSPLIT, $0-24
	B	·Xchg64(SB)

TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
	B	·Xchg64(SB)

TEXT ·And8(SB), NOSPLIT, $0-9
	MOVD	ptr+0(FP), R0
	MOVB	val+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	MVN 	R1, R2
	LDCLRALB	R2, (R0), R3
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRB	(R0), R2
	AND	R1, R2
	STLXRB	R2, (R0), R3
	CBNZ	R3, load_store_loop
	RET
#endif

TEXT ·Or8(SB), NOSPLIT, $0-9
	MOVD	ptr+0(FP), R0
	MOVB	val+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	LDORALB	R1, (R0), R2
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRB	(R0), R2
	ORR	R1, R2
	STLXRB	R2, (R0), R3
	CBNZ	R3, load_store_loop
	RET
#endif

// func And(addr *uint32, v uint32)
TEXT ·And(SB), NOSPLIT, $0-12
	MOVD	ptr+0(FP), R0
	MOVW	val+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	MVN 	R1, R2
	LDCLRALW	R2, (R0), R3
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRW	(R0), R2
	AND	R1, R2
	STLXRW	R2, (R0), R3
	CBNZ	R3, load_store_loop
	RET
#endif

// func Or(addr *uint32, v uint32)
TEXT ·Or(SB), NOSPLIT, $0-12
	MOVD	ptr+0(FP), R0
	MOVW	val+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	LDORALW	R1, (R0), R2
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRW	(R0), R2
	ORR	R1, R2
	STLXRW	R2, (R0), R3
	CBNZ	R3, load_store_loop
	RET
#endif

// func Or32(addr *uint32, v uint32) old uint32
TEXT ·Or32(SB), NOSPLIT, $0-20
	MOVD	ptr+0(FP), R0
	MOVW	val+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	LDORALW	R1, (R0), R2
	MOVD	R2, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRW	(R0), R2
	ORR	R1, R2, R3
	STLXRW	R3, (R0), R4
	CBNZ	R4, load_store_loop
	MOVD R2, ret+16(FP)
	RET
#endif

// func And32(addr *uint32, v uint32) old uint32
TEXT ·And32(SB), NOSPLIT, $0-20
	MOVD	ptr+0(FP), R0
	MOVW	val+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	MVN 	R1, R2
	LDCLRALW	R2, (R0), R3
	MOVD	R3, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRW	(R0), R2
	AND	R1, R2, R3
	STLXRW	R3, (R0), R4
	CBNZ	R4, load_store_loop
	MOVD R2, ret+16(FP)
	RET
#endif

// func Or64(addr *uint64, v uint64) old uint64
TEXT ·Or64(SB), NOSPLIT, $0-24
	MOVD	ptr+0(FP), R0
	MOVD	val+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	LDORALD	R1, (R0), R2
	MOVD	R2, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXR	(R0), R2
	ORR	R1, R2, R3
	STLXR	R3, (R0), R4
	CBNZ	R4, load_store_loop
	MOVD 	R2, ret+16(FP)
	RET
#endif

// func And64(addr *uint64, v uint64) old uint64
TEXT ·And64(SB), NOSPLIT, $0-24
	MOVD	ptr+0(FP), R0
	MOVD	val+8(FP), R1
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	MVN 	R1, R2
	LDCLRALD	R2, (R0), R3
	MOVD	R3, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXR	(R0), R2
	AND	R1, R2, R3
	STLXR	R3, (R0), R4
	CBNZ	R4, load_store_loop
	MOVD 	R2, ret+16(FP)
	RET
#endif

// func Anduintptr(addr *uintptr, v uintptr) old uintptr
TEXT ·Anduintptr(SB), NOSPLIT, $0-24
	B	·And64(SB)

// func Oruintptr(addr *uintptr, v uintptr) old uintptr
TEXT ·Oruintptr(SB), NOSPLIT, $0-24
	B	·Or64(SB)